diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
index 570bb2c..129ec26 100644
--- a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
@@ -16,7 +16,9 @@
 
 import java.util.Collection;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
@@ -38,7 +40,7 @@
     }
 
     @Override
-    public boolean rewriteWithRuleCollection(LogicalOperatorReference root, Collection<IAlgebraicRewriteRule> rules)
+    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
             throws AlgebricksException {
         boolean anyRuleFired = false;
         boolean anyChange = false;
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
index 4658041..8be65f4 100644
--- a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
@@ -16,7 +16,9 @@
 
 import java.util.Collection;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
@@ -40,7 +42,7 @@
     }
 
     @Override
-    public boolean rewriteWithRuleCollection(LogicalOperatorReference root,
+    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
             Collection<IAlgebraicRewriteRule> ruleCollection) throws AlgebricksException {
         boolean anyRuleFired = false;
         boolean anyChange = false;
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
index 69281a1..5cf62e0 100644
--- a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
@@ -16,7 +16,9 @@
 
 import java.util.Collection;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
@@ -31,7 +33,7 @@
     }
 
     @Override
-    public boolean rewriteWithRuleCollection(LogicalOperatorReference root, Collection<IAlgebraicRewriteRule> rules)
+    public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
             throws AlgebricksException {
         boolean fired = false;
         for (IAlgebraicRewriteRule rule : rules) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
index 2d19944..a54a88d 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
@@ -18,6 +18,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -62,7 +64,7 @@
      * @return true if the expression can be broken in at least two conjuncts,
      *         false otherwise.
      */
-    public boolean splitIntoConjuncts(List<LogicalExpressionReference> conjs);
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs);
 
     public abstract ILogicalExpression cloneExpression();
 }
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
index 198d762..719b5df 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
@@ -17,6 +17,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
@@ -30,7 +32,7 @@
 
 public interface ILogicalOperator {
 
-    public List<LogicalOperatorReference> getInputs();
+    public List<Mutable<ILogicalOperator>> getInputs();
 
     boolean hasInputs();
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
index bf0f548..c94a9c1 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
@@ -16,6 +16,8 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 public interface ILogicalPlan {
-    public List<LogicalOperatorReference> getRoots();
+    public List<Mutable<ILogicalOperator>> getRoots();
 }
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionReference.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionReference.java
deleted file mode 100644
index 6ab0cb9..0000000
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionReference.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-public final class LogicalExpressionReference {
-    private ILogicalExpression expression;
-
-    public LogicalExpressionReference(ILogicalExpression expression) {
-        this.expression = expression;
-    }
-
-    public ILogicalExpression getExpression() {
-        return expression;
-    }
-
-    public void setExpression(ILogicalExpression expression) {
-        this.expression = expression;
-    }
-
-    public String toString() {
-        return expression.toString();
-    }
-}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorReference.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorReference.java
deleted file mode 100644
index ce47ff8..0000000
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorReference.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.algebricks.core.algebra.base;
-
-public final class LogicalOperatorReference {
-    private ILogicalOperator operator;
-
-    public LogicalOperatorReference() {
-    }
-
-    public LogicalOperatorReference(ILogicalOperator operator) {
-        this.operator = operator;
-    }
-
-    public ILogicalOperator getOperator() {
-        return operator;
-    }
-
-    public void setOperator(ILogicalOperator operator) {
-        this.operator = operator;
-    }
-
-    @Override
-    public String toString() {
-        return operator == null ? "" : operator.toString();
-    }
-}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
index 6dcaa31..dae055c 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
@@ -21,9 +21,11 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
@@ -34,17 +36,20 @@
 public abstract class AbstractFunctionCallExpression extends AbstractLogicalExpression {
 
     public enum FunctionKind {
-        SCALAR, STATEFUL, AGGREGATE, UNNEST
+        SCALAR,
+        STATEFUL,
+        AGGREGATE,
+        UNNEST
     }
 
     protected IFunctionInfo finfo;
-    final private List<LogicalExpressionReference> arguments;
+    final private List<Mutable<ILogicalExpression>> arguments;
     private Object[] opaqueParameters;
     private final FunctionKind kind;
     private Map<Object, IExpressionAnnotation> annotationMap = new HashMap<Object, IExpressionAnnotation>();
 
     public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
-            List<LogicalExpressionReference> arguments) {
+            List<Mutable<ILogicalExpression>> arguments) {
         this.kind = kind;
         this.finfo = finfo;
         this.arguments = arguments;
@@ -53,13 +58,13 @@
     public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo) {
         this.kind = kind;
         this.finfo = finfo;
-        this.arguments = new ArrayList<LogicalExpressionReference>();
+        this.arguments = new ArrayList<Mutable<ILogicalExpression>>();
     }
 
     public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
-            LogicalExpressionReference... expressions) {
+            Mutable<ILogicalExpression>... expressions) {
         this(kind, finfo);
-        for (LogicalExpressionReference e : expressions) {
+        for (Mutable<ILogicalExpression> e : expressions) {
             this.arguments.add(e);
         }
     }
@@ -76,11 +81,11 @@
         return kind;
     }
 
-    protected List<LogicalExpressionReference> cloneArguments() {
-        List<LogicalExpressionReference> clonedArgs = new ArrayList<LogicalExpressionReference>(arguments.size());
-        for (LogicalExpressionReference e : arguments) {
-            ILogicalExpression e2 = ((AbstractLogicalExpression) e.getExpression()).cloneExpression();
-            clonedArgs.add(new LogicalExpressionReference(e2));
+    protected List<Mutable<ILogicalExpression>> cloneArguments() {
+        List<Mutable<ILogicalExpression>> clonedArgs = new ArrayList<Mutable<ILogicalExpression>>(arguments.size());
+        for (Mutable<ILogicalExpression> e : arguments) {
+            ILogicalExpression e2 = ((AbstractLogicalExpression) e.getValue()).cloneExpression();
+            clonedArgs.add(new MutableObject<ILogicalExpression>(e2));
         }
         return clonedArgs;
     }
@@ -97,7 +102,7 @@
         this.finfo = finfo;
     }
 
-    public List<LogicalExpressionReference> getArguments() {
+    public List<Mutable<ILogicalExpression>> getArguments() {
         return arguments;
     }
 
@@ -106,13 +111,13 @@
         sb.append("function-call: " + finfo.getFunctionIdentifier() + ", Args:[");
         // + arguments;
         boolean first = true;
-        for (LogicalExpressionReference ref : arguments) {
+        for (Mutable<ILogicalExpression> ref : arguments) {
             if (first) {
                 first = false;
             } else {
                 sb.append(", ");
             }
-            sb.append(ref.getExpression());
+            sb.append(ref.getValue());
         }
         sb.append("]");
         return sb.toString();
@@ -125,15 +130,15 @@
 
     @Override
     public void getUsedVariables(Collection<LogicalVariable> vars) {
-        for (LogicalExpressionReference arg : arguments) {
-            arg.getExpression().getUsedVariables(vars);
+        for (Mutable<ILogicalExpression> arg : arguments) {
+            arg.getValue().getUsedVariables(vars);
         }
     }
 
     @Override
     public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
-        for (LogicalExpressionReference arg : arguments) {
-            arg.getExpression().substituteVar(v1, v2);
+        for (Mutable<ILogicalExpression> arg : arguments) {
+            arg.getValue().substituteVar(v1, v2);
         }
     }
 
@@ -142,12 +147,12 @@
             Map<LogicalVariable, EquivalenceClass> equivClasses) {
         FunctionIdentifier funId = getFunctionIdentifier();
         if (funId == AlgebricksBuiltinFunctions.AND) {
-            for (LogicalExpressionReference a : arguments) {
-                a.getExpression().getConstraintsAndEquivClasses(fds, equivClasses);
+            for (Mutable<ILogicalExpression> a : arguments) {
+                a.getValue().getConstraintsAndEquivClasses(fds, equivClasses);
             }
         } else if (funId == AlgebricksBuiltinFunctions.EQ) {
-            ILogicalExpression opLeft = arguments.get(0).getExpression();
-            ILogicalExpression opRight = arguments.get(1).getExpression();
+            ILogicalExpression opLeft = arguments.get(0).getValue();
+            ILogicalExpression opRight = arguments.get(1).getValue();
             if (opLeft.getExpressionTag() == LogicalExpressionTag.CONSTANT
                     && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 ConstantExpression op1 = (ConstantExpression) opLeft;
@@ -166,12 +171,12 @@
     public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars) {
         FunctionIdentifier funId = getFunctionIdentifier();
         if (funId == AlgebricksBuiltinFunctions.AND) {
-            for (LogicalExpressionReference a : arguments) {
-                a.getExpression().getConstraintsForOuterJoin(fds, outerVars);
+            for (Mutable<ILogicalExpression> a : arguments) {
+                a.getValue().getConstraintsForOuterJoin(fds, outerVars);
             }
         } else if (funId == AlgebricksBuiltinFunctions.EQ) {
-            ILogicalExpression opLeft = arguments.get(0).getExpression();
-            ILogicalExpression opRight = arguments.get(1).getExpression();
+            ILogicalExpression opLeft = arguments.get(0).getValue();
+            ILogicalExpression opRight = arguments.get(1).getValue();
             if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
                     && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 LogicalVariable var1 = ((VariableReferenceExpression) opLeft).getVariableReference();
@@ -196,8 +201,8 @@
             if (!equal)
                 return false;
             for (int i = 0; i < arguments.size(); i++) {
-                ILogicalExpression argument = arguments.get(i).getExpression();
-                ILogicalExpression fceArgument = fce.getArguments().get(i).getExpression();
+                ILogicalExpression argument = arguments.get(i).getValue();
+                ILogicalExpression fceArgument = fce.getArguments().get(i).getValue();
                 if (!argument.equals(fceArgument))
                     return false;
             }
@@ -208,14 +213,14 @@
     @Override
     public int hashCode() {
         int h = finfo.hashCode();
-        for (LogicalExpressionReference e : arguments) {
-            h = h * 41 + e.getExpression().hashCode();
+        for (Mutable<ILogicalExpression> e : arguments) {
+            h = h * 41 + e.getValue().hashCode();
         }
         return h;
     }
 
     @Override
-    public boolean splitIntoConjuncts(List<LogicalExpressionReference> conjs) {
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
         if (getFunctionIdentifier() != AlgebricksBuiltinFunctions.AND || arguments.size() <= 1) {
             return false;
         } else {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
index 84a421b..ac77ee2 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
@@ -16,7 +16,9 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -47,13 +49,13 @@
     }
 
     public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
-            List<LogicalExpressionReference> arguments) {
+            List<Mutable<ILogicalExpression>> arguments) {
         super(FunctionKind.AGGREGATE, finfo, arguments);
         this.twoStep = isTwoStep;
     }
 
     public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
-            LogicalExpressionReference... expressions) {
+            Mutable<ILogicalExpression>... expressions) {
         super(FunctionKind.AGGREGATE, finfo, expressions);
         this.twoStep = isTwoStep;
     }
@@ -69,7 +71,7 @@
     @Override
     public AggregateFunctionCallExpression cloneExpression() {
         cloneAnnotations();
-        List<LogicalExpressionReference> clonedArgs = cloneArguments();
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
         AggregateFunctionCallExpression fun = new AggregateFunctionCallExpression(finfo, twoStep, clonedArgs);
         fun.setStepTwoAggregate(stepTwoAggregate);
         fun.setStepOneAggregate(stepOneAggregate);
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
index 800fee7..2fa0637 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
@@ -19,7 +19,9 @@
 import java.util.List;
 import java.util.Map;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
@@ -165,7 +167,7 @@
     }
 
     @Override
-    public boolean splitIntoConjuncts(List<LogicalExpressionReference> conjs) {
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
         return false;
     }
 }
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
index 9b1e2dc..fa8838b 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
@@ -16,7 +16,9 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -27,18 +29,18 @@
         super(FunctionKind.SCALAR, finfo);
     }
 
-    public ScalarFunctionCallExpression(IFunctionInfo finfo, List<LogicalExpressionReference> arguments) {
+    public ScalarFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
         super(FunctionKind.SCALAR, finfo, arguments);
     }
 
-    public ScalarFunctionCallExpression(IFunctionInfo finfo, LogicalExpressionReference... expressions) {
+    public ScalarFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
         super(FunctionKind.SCALAR, finfo, expressions);
     }
 
     @Override
     public ScalarFunctionCallExpression cloneExpression() {
         cloneAnnotations();
-        List<LogicalExpressionReference> clonedArgs = cloneArguments();
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
         return new ScalarFunctionCallExpression(finfo, clonedArgs);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
index 67c3682..c5cd897 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
@@ -16,7 +16,9 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPropertiesComputer;
@@ -33,13 +35,13 @@
     }
 
     public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
-            List<LogicalExpressionReference> arguments) {
+            List<Mutable<ILogicalExpression>> arguments) {
         super(FunctionKind.STATEFUL, finfo, arguments);
         this.propertiesComputer = propertiesComputer;
     }
 
     public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
-            LogicalExpressionReference... expressions) {
+            Mutable<ILogicalExpression>... expressions) {
         super(FunctionKind.STATEFUL, finfo, expressions);
         this.propertiesComputer = propertiesComputer;
     }
@@ -47,7 +49,7 @@
     @Override
     public StatefulFunctionCallExpression cloneExpression() {
         cloneAnnotations();
-        List<LogicalExpressionReference> clonedArgs = cloneArguments();
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
         return new StatefulFunctionCallExpression(finfo, propertiesComputer, clonedArgs);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
index 25fa964..5f3de4b 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
@@ -16,7 +16,9 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -29,18 +31,18 @@
         super(FunctionKind.UNNEST, finfo);
     }
 
-    public UnnestingFunctionCallExpression(IFunctionInfo finfo, List<LogicalExpressionReference> arguments) {
+    public UnnestingFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
         super(FunctionKind.UNNEST, finfo, arguments);
     }
 
-    public UnnestingFunctionCallExpression(IFunctionInfo finfo, LogicalExpressionReference... expressions) {
+    public UnnestingFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
         super(FunctionKind.UNNEST, finfo, expressions);
     }
 
     @Override
     public UnnestingFunctionCallExpression cloneExpression() {
         cloneAnnotations();
-        List<LogicalExpressionReference> clonedArgs = cloneArguments();
+        List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
         UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression(finfo, clonedArgs);
         ufce.setReturnsUniqueValues(returnsUniqueValues);
         return ufce;
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
index 07d423b..ecd993f 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
@@ -17,7 +17,9 @@
 import java.util.Collection;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
@@ -87,7 +89,7 @@
     }
 
     @Override
-    public boolean splitIntoConjuncts(List<LogicalExpressionReference> conjs) {
+    public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
         return false;
     }
 }
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
index 438386f..456010a 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
@@ -17,7 +17,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -28,15 +30,14 @@
  */
 public abstract class AbstractAssignOperator extends AbstractLogicalOperator {
     protected final List<LogicalVariable> variables;
-    protected final List<LogicalExpressionReference> expressions;
+    protected final List<Mutable<ILogicalExpression>> expressions;
 
     public AbstractAssignOperator() {
         this.variables = new ArrayList<LogicalVariable>();
-        this.expressions = new ArrayList<LogicalExpressionReference>();
+        this.expressions = new ArrayList<Mutable<ILogicalExpression>>();
     }
 
-    public AbstractAssignOperator(List<LogicalVariable> variables,
-            List<LogicalExpressionReference> expressions) {
+    public AbstractAssignOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
         this.variables = variables;
         this.expressions = expressions;
     }
@@ -45,14 +46,14 @@
         return variables;
     }
 
-    public List<LogicalExpressionReference> getExpressions() {
+    public List<Mutable<ILogicalExpression>> getExpressions() {
         return expressions;
     }
 
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
         schema.addAll(variables);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
index a6eb30f..c2911d4 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
@@ -16,34 +16,37 @@
 
 import java.util.ArrayList;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public abstract class AbstractBinaryJoinOperator extends AbstractLogicalOperator {
-    protected final LogicalExpressionReference condition;
+    protected final Mutable<ILogicalExpression> condition;
     protected JoinKind joinKind;
 
     public enum JoinKind {
-        INNER, LEFT_OUTER
+        INNER,
+        LEFT_OUTER
     }
 
-    public AbstractBinaryJoinOperator(JoinKind joinKind, LogicalExpressionReference condition) {
+    public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition) {
         this.joinKind = joinKind;
         this.condition = condition;
     }
 
-    public AbstractBinaryJoinOperator(JoinKind joinKind, LogicalExpressionReference condition, LogicalOperatorReference input1,
-            LogicalOperatorReference input2) {
+    public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition,
+            Mutable<ILogicalOperator> input1, Mutable<ILogicalOperator> input2) {
         this(joinKind, condition);
         inputs.add(input1);
         inputs.add(input2);
     }
 
-    public LogicalExpressionReference getCondition() {
+    public Mutable<ILogicalExpression> getCondition() {
         return condition;
     }
 
@@ -54,8 +57,8 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
-        schema.addAll(inputs.get(1).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
+        schema.addAll(inputs.get(1).getValue().getSchema());
     }
 
     @Override
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
index 9b1931d..88fd0d5 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
@@ -19,11 +19,12 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -60,12 +61,12 @@
     private boolean bJobGenEnabled = true;
     private Object hostQueryContext;
 
-    final protected List<LogicalOperatorReference> inputs;
+    final protected List<Mutable<ILogicalOperator>> inputs;
     // protected List<LogicalOperatorReference> outputs;
     protected List<LogicalVariable> schema;
 
     public AbstractLogicalOperator() {
-        inputs = new ArrayList<LogicalOperatorReference>();
+        inputs = new ArrayList<Mutable<ILogicalOperator>>();
         // outputs = new ArrayList<LogicalOperatorReference>();
     }
 
@@ -118,7 +119,7 @@
     }
 
     @Override
-    public final List<LogicalOperatorReference> getInputs() {
+    public final List<Mutable<ILogicalOperator>> getInputs() {
         return inputs;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
index ec4e6ef..ea4f3e0 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
@@ -19,8 +19,10 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 
 public abstract class AbstractOperatorWithNestedPlans extends AbstractLogicalOperator {
@@ -43,10 +45,10 @@
         return true;
     }
 
-    public LinkedList<LogicalOperatorReference> allRootsInReverseOrder() {
-        LinkedList<LogicalOperatorReference> allRoots = new LinkedList<LogicalOperatorReference>();
+    public LinkedList<Mutable<ILogicalOperator>> allRootsInReverseOrder() {
+        LinkedList<Mutable<ILogicalOperator>> allRoots = new LinkedList<Mutable<ILogicalOperator>>();
         for (ILogicalPlan p : nestedPlans) {
-            for (LogicalOperatorReference r : p.getRoots()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                 allRoots.addFirst(r);
             }
         }
@@ -68,10 +70,10 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
         for (ILogicalPlan p : nestedPlans) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                schema.addAll(r.getOperator().getSchema());
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                schema.addAll(r.getValue().getSchema());
             }
         }
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
index e5132ef..e4e7ec8 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
@@ -39,7 +39,7 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
         schema.addAll(variables);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
index 020d83c..22f73ff 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
@@ -16,21 +16,23 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public abstract class AbstractUnnestOperator extends AbstractScanOperator {
 
-    protected final LogicalExpressionReference expression;
+    protected final Mutable<ILogicalExpression> expression;
 
-    public AbstractUnnestOperator(List<LogicalVariable> variables, LogicalExpressionReference expression) {
+    public AbstractUnnestOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression) {
         super(variables);
         this.expression = expression;
     }
 
-    public LogicalExpressionReference getExpressionRef() {
+    public Mutable<ILogicalExpression> getExpressionRef() {
         return expression;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
index ba7c970..1555a76 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
@@ -3,7 +3,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -17,9 +19,9 @@
 
     // private ArrayList<AggregateFunctionCallExpression> expressions;
     // TODO type safe list of expressions
-    private List<LogicalExpressionReference> mergeExpressions;
+    private List<Mutable<ILogicalExpression>> mergeExpressions;
 
-    public AggregateOperator(List<LogicalVariable> variables, List<LogicalExpressionReference> expressions) {
+    public AggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
         super(variables, expressions);
     }
 
@@ -58,23 +60,23 @@
         schema.addAll(variables);
     }
 
-    public void setMergeExpressions(List<LogicalExpressionReference> merges) {
+    public void setMergeExpressions(List<Mutable<ILogicalExpression>> merges) {
         mergeExpressions = merges;
     }
 
-    public List<LogicalExpressionReference> getMergeExpressions() {
+    public List<Mutable<ILogicalExpression>> getMergeExpressions() {
         return mergeExpressions;
     }
 
     @Override
     public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx
-                .getMetadataProvider());
-        IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(inputs.get(0).getOperator());
+        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getMetadataProvider());
+        IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
         int n = variables.size();
         for (int i = 0; i < n; i++) {
-            Object t = ctx.getExpressionTypeComputer().getType(expressions.get(i).getExpression(), ctx.getMetadataProvider(),
-                    env2);
+            Object t = ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(),
+                    ctx.getMetadataProvider(), env2);
             env.setVarType(variables.get(i), t);
         }
         return env;
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
index db57c4b..eedd3a2 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
@@ -16,7 +16,9 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -35,11 +37,11 @@
 
 public class AssignOperator extends AbstractAssignOperator {
 
-    public AssignOperator(List<LogicalVariable> vars, List<LogicalExpressionReference> exprs) {
+    public AssignOperator(List<LogicalVariable> vars, List<Mutable<ILogicalExpression>> exprs) {
         super(vars, exprs);
     }
 
-    public AssignOperator(LogicalVariable var, LogicalExpressionReference expr) {
+    public AssignOperator(LogicalVariable var, Mutable<ILogicalExpression> expr) {
         super();
         this.variables.add(var);
         this.expressions.add(expr);
@@ -81,8 +83,10 @@
         IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
         int n = variables.size();
         for (int i = 0; i < n; i++) {
-            env.setVarType(variables.get(i), ctx.getExpressionTypeComputer().getType(
-                    expressions.get(i).getExpression(), ctx.getMetadataProvider(), env));
+            env.setVarType(
+                    variables.get(i),
+                    ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
+                            env));
         }
         return env;
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
index fe22219..2e330cf 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
@@ -16,8 +16,10 @@
 
 import java.util.ArrayList;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -29,20 +31,20 @@
 
 public class DieOperator extends AbstractLogicalOperator {
 
-    private final LogicalExpressionReference afterObjects; // mandatory
+    private final Mutable<ILogicalExpression> afterObjects; // mandatory
 
     public DieOperator(ILogicalExpression maxObjectsExpr) {
-        this.afterObjects = new LogicalExpressionReference(maxObjectsExpr);
+        this.afterObjects = new MutableObject<ILogicalExpression>(maxObjectsExpr);
     }
 
-    public LogicalExpressionReference getAfterObjects() {
+    public Mutable<ILogicalExpression> getAfterObjects() {
         return afterObjects;
     }
 
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
     }
 
     @Override
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
index 593e505..4d479e1 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
@@ -17,8 +17,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -31,9 +32,9 @@
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public class DistinctOperator extends AbstractLogicalOperator {
-    private final List<LogicalExpressionReference> expressions;
+    private final List<Mutable<ILogicalExpression>> expressions;
 
-    public DistinctOperator(List<LogicalExpressionReference> expressions) {
+    public DistinctOperator(List<Mutable<ILogicalExpression>> expressions) {
         this.expressions = expressions;
     }
 
@@ -42,13 +43,13 @@
         return LogicalOperatorTag.DISTINCT;
     }
 
-    public List<LogicalExpressionReference> getExpressions() {
+    public List<Mutable<ILogicalExpression>> getExpressions() {
         return expressions;
     }
 
     @Override
     public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getOperator().getSchema());
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
     }
 
     @Override
@@ -59,7 +60,7 @@
     @Override
     public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
         boolean changed = false;
-        for (LogicalExpressionReference e : expressions) {
+        for (Mutable<ILogicalExpression> e : expressions) {
             if (visitor.transform(e)) {
                 changed = true;
             }
@@ -79,8 +80,8 @@
 
     public List<LogicalVariable> getDistinctByVarList() {
         List<LogicalVariable> varList = new ArrayList<LogicalVariable>(expressions.size());
-        for (LogicalExpressionReference eRef : expressions) {
-            ILogicalExpression e = eRef.getExpression();
+        for (Mutable<ILogicalExpression> eRef : expressions) {
+            ILogicalExpression e = eRef.getValue();
             if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression v = (VariableReferenceExpression) e;
                 varList.add(v.getVariableReference());
@@ -90,8 +91,8 @@
     }
 
     public boolean isDistinctByVar(LogicalVariable var) {
-        for (LogicalExpressionReference eRef : expressions) {
-            ILogicalExpression e = eRef.getExpression();
+        for (Mutable<ILogicalExpression> eRef : expressions) {
+            ILogicalExpression e = eRef.getValue();
             if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression v = (VariableReferenceExpression) e;
                 if (v.getVariableReference() == var) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
index f0899c9..3e046b5 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
@@ -52,9 +52,9 @@
 
     @Override
     public void recomputeSchema() throws AlgebricksException {
-        AbstractLogicalOperator cld = (AbstractLogicalOperator) inputs.get(0).getOperator();
+        AbstractLogicalOperator cld = (AbstractLogicalOperator) inputs.get(0).getValue();
         OperatorPropertiesUtil.computeSchemaRecIfNull(cld);
-        List<LogicalVariable> inputSchema = inputs.get(0).getOperator().getSchema();
+        List<LogicalVariable> inputSchema = inputs.get(0).getValue().getSchema();
         schema = new ArrayList<LogicalVariable>(inputSchema);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
index 9bc37d4..762f034 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
@@ -18,12 +18,13 @@
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -42,33 +43,33 @@
 public class GroupByOperator extends AbstractOperatorWithNestedPlans {
     // If the LogicalVariable in a pair is null, it means that the GroupBy is
     // only grouping by the expression, without producing a new variable.
-    private final List<Pair<LogicalVariable, LogicalExpressionReference>> gByList;
-    private final List<Pair<LogicalVariable, LogicalExpressionReference>> decorList;
+    private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList;
+    private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList;
 
     // In decorList, if the variable (first member of the pair) is null, the
     // second member of the pair is variable reference which is propagated.
 
     public GroupByOperator() {
         super();
-        gByList = new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>();
-        decorList = new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>();
+        gByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+        decorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
     }
 
-    public GroupByOperator(List<Pair<LogicalVariable, LogicalExpressionReference>> groupByList,
-            List<Pair<LogicalVariable, LogicalExpressionReference>> decorList, List<ILogicalPlan> nestedPlans) {
+    public GroupByOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList, List<ILogicalPlan> nestedPlans) {
         super(nestedPlans);
         this.decorList = decorList;
         this.gByList = groupByList;
     }
 
     public void addGbyExpression(LogicalVariable variable, ILogicalExpression expression) {
-        this.gByList.add(new Pair<LogicalVariable, LogicalExpressionReference>(variable,
-                new LogicalExpressionReference(expression)));
+        this.gByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
+                new MutableObject<ILogicalExpression>(expression)));
     }
 
     public void addDecorExpression(LogicalVariable variable, ILogicalExpression expression) {
-        this.decorList.add(new Pair<LogicalVariable, LogicalExpressionReference>(variable,
-                new LogicalExpressionReference(expression)));
+        this.decorList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
+                new MutableObject<ILogicalExpression>(expression)));
     }
 
     @Override
@@ -76,7 +77,7 @@
         return LogicalOperatorTag.GROUP;
     }
 
-    public List<Pair<LogicalVariable, LogicalExpressionReference>> getGroupByList() {
+    public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getGroupByList() {
         return gByList;
     }
 
@@ -90,8 +91,8 @@
 
     public List<LogicalVariable> getGbyVarList() {
         List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size());
-        for (Pair<LogicalVariable, LogicalExpressionReference> ve : gByList) {
-            ILogicalExpression expr = ve.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) {
+            ILogicalExpression expr = ve.second.getValue();
             if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression v = (VariableReferenceExpression) expr;
                 varList.add(v.getVariableReference());
@@ -100,11 +101,11 @@
         return varList;
     }
 
-    public static String veListToString(List<Pair<LogicalVariable, LogicalExpressionReference>> vePairList) {
+    public static String veListToString(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList) {
         StringBuilder sb = new StringBuilder();
         sb.append("[");
         boolean fst = true;
-        for (Pair<LogicalVariable, LogicalExpressionReference> ve : vePairList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
             if (fst) {
                 fst = false;
             } else {
@@ -113,7 +114,7 @@
             if (ve.first != null) {
                 sb.append(ve.first + " := " + ve.second);
             } else {
-                sb.append(ve.second.getExpression());
+                sb.append(ve.second.getValue());
             }
         }
         sb.append("]");
@@ -123,10 +124,10 @@
     @Override
     public void recomputeSchema() {
         super.recomputeSchema();
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gByList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
             schema.add(p.first);
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : decorList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
             schema.add(getDecorVariable(p));
         }
     }
@@ -134,12 +135,12 @@
     @Override
     public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
         // super.getProducedVariables(vars);
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gByList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
             if (p.first != null) {
                 vars.add(p.first);
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : decorList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
             if (p.first != null) {
                 vars.add(p.first);
             }
@@ -148,11 +149,11 @@
 
     @Override
     public void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> g : gByList) {
-            g.second.getExpression().getUsedVariables(vars);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : gByList) {
+            g.second.getValue().getUsedVariables(vars);
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> g : decorList) {
-            g.second.getExpression().getUsedVariables(vars);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : decorList) {
+            g.second.getValue().getUsedVariables(vars);
         }
         // super.getUsedVariables(vars);
     }
@@ -164,8 +165,8 @@
             @Override
             public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
                     throws AlgebricksException {
-                for (Pair<LogicalVariable, LogicalExpressionReference> p : gByList) {
-                    ILogicalExpression expr = p.second.getExpression();
+                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+                    ILogicalExpression expr = p.second.getValue();
                     if (p.first != null) {
                         target.addVariable(p.first);
                     } else {
@@ -176,8 +177,8 @@
                         target.addVariable(v.getVariableReference());
                     }
                 }
-                for (Pair<LogicalVariable, LogicalExpressionReference> p : decorList) {
-                    ILogicalExpression expr = p.second.getExpression();
+                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+                    ILogicalExpression expr = p.second.getValue();
                     if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                         throw new AlgebricksException("pre-sorted group-by expects variable references.");
                     }
@@ -197,12 +198,12 @@
     @Override
     public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
         boolean b = false;
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gByList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
             if (visitor.transform(p.second)) {
                 b = true;
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : decorList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
             if (visitor.transform(p.second)) {
                 b = true;
             }
@@ -215,16 +216,16 @@
         return visitor.visitGroupByOperator(this, arg);
     }
 
-    public static LogicalVariable getDecorVariable(Pair<LogicalVariable, LogicalExpressionReference> p) {
+    public static LogicalVariable getDecorVariable(Pair<LogicalVariable, Mutable<ILogicalExpression>> p) {
         if (p.first != null) {
             return p.first;
         } else {
-            VariableReferenceExpression e = (VariableReferenceExpression) p.second.getExpression();
+            VariableReferenceExpression e = (VariableReferenceExpression) p.second.getValue();
             return e.getVariableReference();
         }
     }
 
-    public List<Pair<LogicalVariable, LogicalExpressionReference>> getDecorList() {
+    public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getDecorList() {
         return decorList;
     }
 
@@ -237,17 +238,17 @@
         ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
         int i = 0;
         for (ILogicalPlan p : nestedPlans) {
-            for (LogicalOperatorReference r : p.getRoots()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                 envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
                 i++;
             }
         }
-        IVariableTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx
-                .getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
-        ILogicalOperator child = inputs.get(0).getOperator();
+        IVariableTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+        ILogicalOperator child = inputs.get(0).getValue();
         IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(child);
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : getGroupByList()) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getGroupByList()) {
+            ILogicalExpression expr = p.second.getValue();
             if (p.first != null) {
                 env.setVarType(p.first, env2.getType(expr));
                 if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
@@ -255,17 +256,17 @@
                     env.setVarType(v1, env2.getVarType(v1));
                 }
             } else {
-                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getExpression();
+                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
                 LogicalVariable v2 = vre.getVariableReference();
                 env.setVarType(v2, env2.getVarType(v2));
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : getDecorList()) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
             if (p.first != null) {
                 env.setVarType(p.first, env2.getType(expr));
             } else {
-                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getExpression();
+                VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
                 LogicalVariable v2 = vre.getVariableReference();
                 env.setVarType(v2, env2.getVarType(v2));
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
index ca032a2..7264763 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
@@ -3,7 +3,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -18,12 +20,12 @@
 public class IndexInsertDeleteOperator extends AbstractLogicalOperator {
 
     private final IDataSourceIndex<?, ?> dataSourceIndex;
-    private final List<LogicalExpressionReference> primaryKeyExprs;
-    private final List<LogicalExpressionReference> secondaryKeyExprs;
+    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
+    private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
     private final Kind operation;
 
     public IndexInsertDeleteOperator(IDataSourceIndex<?, ?> dataSourceIndex,
-            List<LogicalExpressionReference> primaryKeyExprs, List<LogicalExpressionReference> secondaryKeyExprs,
+            List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
             Kind operation) {
         this.dataSourceIndex = dataSourceIndex;
         this.primaryKeyExprs = primaryKeyExprs;
@@ -34,7 +36,7 @@
     @Override
     public void recomputeSchema() throws AlgebricksException {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
     }
 
     @Override
@@ -78,7 +80,7 @@
         return createPropagatingAllInputsTypeEnvironment(ctx);
     }
 
-    public List<LogicalExpressionReference> getPrimaryKeyExpressions() {
+    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
         return primaryKeyExprs;
     }
 
@@ -86,7 +88,7 @@
         return dataSourceIndex;
     }
 
-    public List<LogicalExpressionReference> getSecondaryKeyExpressions() {
+    public List<Mutable<ILogicalExpression>> getSecondaryKeyExpressions() {
         return secondaryKeyExprs;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
index e9d8c2c..c2f0161 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
@@ -14,8 +14,10 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
@@ -24,12 +26,12 @@
 
 public class InnerJoinOperator extends AbstractBinaryJoinOperator {
 
-    public InnerJoinOperator(LogicalExpressionReference condition) {
+    public InnerJoinOperator(Mutable<ILogicalExpression> condition) {
         super(JoinKind.INNER, condition);
     }
 
-    public InnerJoinOperator(LogicalExpressionReference condition, LogicalOperatorReference input1,
-            LogicalOperatorReference input2) {
+    public InnerJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
+            Mutable<ILogicalOperator> input2) {
         super(JoinKind.INNER, condition, input1, input2);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
index be69e96..6699228 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
@@ -3,7 +3,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -17,16 +19,17 @@
 public class InsertDeleteOperator extends AbstractLogicalOperator {
 
     public enum Kind {
-        INSERT, DELETE
+        INSERT,
+        DELETE
     }
 
     private final IDataSource<?> dataSource;
-    private final LogicalExpressionReference payloadExpr;
-    private final List<LogicalExpressionReference> primaryKeyExprs;
+    private final Mutable<ILogicalExpression> payloadExpr;
+    private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
     private final Kind operation;
 
-    public InsertDeleteOperator(IDataSource<?> dataSource, LogicalExpressionReference payload,
-            List<LogicalExpressionReference> primaryKeyExprs, Kind operation) {
+    public InsertDeleteOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
+            List<Mutable<ILogicalExpression>> primaryKeyExprs, Kind operation) {
         this.dataSource = dataSource;
         this.payloadExpr = payload;
         this.primaryKeyExprs = primaryKeyExprs;
@@ -36,7 +39,7 @@
     @Override
     public void recomputeSchema() throws AlgebricksException {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
     }
 
     @Override
@@ -76,7 +79,7 @@
         return createPropagatingAllInputsTypeEnvironment(ctx);
     }
 
-    public List<LogicalExpressionReference> getPrimaryKeyExpressions() {
+    public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
         return primaryKeyExprs;
     }
 
@@ -84,7 +87,7 @@
         return dataSource;
     }
 
-    public LogicalExpressionReference getPayloadExpression() {
+    public Mutable<ILogicalExpression> getPayloadExpression() {
         return payloadExpr;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
index 8166b1b..7c5be7e 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
@@ -14,8 +14,10 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
@@ -28,12 +30,12 @@
 
 public class LeftOuterJoinOperator extends AbstractBinaryJoinOperator {
 
-    public LeftOuterJoinOperator(LogicalExpressionReference condition) {
+    public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition) {
         super(JoinKind.LEFT_OUTER, condition);
     }
 
-    public LeftOuterJoinOperator(LogicalExpressionReference condition, LogicalOperatorReference input1,
-            LogicalOperatorReference input2) {
+    public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
+            Mutable<ILogicalOperator> input2) {
         super(JoinKind.LEFT_OUTER, condition, input1, input2);
     }
 
@@ -54,8 +56,8 @@
         for (int i = 0; i < n; i++) {
             envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
         }
-        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(), ctx
-                .getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
+        return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+                ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
     }
 
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
index 5fedfdc..21738f6 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
@@ -16,8 +16,10 @@
 
 import java.util.ArrayList;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -29,13 +31,13 @@
 
 public class LimitOperator extends AbstractLogicalOperator {
 
-    private final LogicalExpressionReference maxObjects; // mandatory
-    private final LogicalExpressionReference offset; // optional
+    private final Mutable<ILogicalExpression> maxObjects; // mandatory
+    private final Mutable<ILogicalExpression> offset; // optional
     private boolean topmost;
 
     public LimitOperator(ILogicalExpression maxObjectsExpr, ILogicalExpression offsetExpr, boolean topmost) {
-        this.maxObjects = new LogicalExpressionReference(maxObjectsExpr);
-        this.offset = new LogicalExpressionReference(offsetExpr);
+        this.maxObjects = new MutableObject<ILogicalExpression>(maxObjectsExpr);
+        this.offset = new MutableObject<ILogicalExpression>(offsetExpr);
         this.topmost = topmost;
     }
 
@@ -51,11 +53,11 @@
         this(maxObjects, null, true);
     }
 
-    public LogicalExpressionReference getMaxObjects() {
+    public Mutable<ILogicalExpression> getMaxObjects() {
         return maxObjects;
     }
 
-    public LogicalExpressionReference getOffset() {
+    public Mutable<ILogicalExpression> getOffset() {
         return offset;
     }
 
@@ -66,7 +68,7 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
     }
 
     @Override
@@ -80,7 +82,7 @@
         if (visitor.transform(maxObjects)) {
             b = true;
         }
-        if (offset.getExpression() != null) {
+        if (offset.getValue() != null) {
             if (visitor.transform(offset)) {
                 b = true;
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
index fc17ce4..189a410 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
@@ -16,8 +16,9 @@
 
 import java.util.ArrayList;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -31,14 +32,14 @@
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public class NestedTupleSourceOperator extends AbstractLogicalOperator {
-    private final LogicalOperatorReference dataSourceReference;
+    private final Mutable<ILogicalOperator> dataSourceReference;
 
-    public NestedTupleSourceOperator(LogicalOperatorReference dataSourceReference) {
+    public NestedTupleSourceOperator(Mutable<ILogicalOperator> dataSourceReference) {
         this.dataSourceReference = dataSourceReference;
     }
 
     public ILogicalOperator getSourceOperator() {
-        return dataSourceReference.getOperator().getInputs().get(0).getOperator();
+        return dataSourceReference.getValue().getInputs().get(0).getValue();
     }
 
     @Override
@@ -46,16 +47,16 @@
         return LogicalOperatorTag.NESTEDTUPLESOURCE;
     }
 
-    public LogicalOperatorReference getDataSourceReference() {
+    public Mutable<ILogicalOperator> getDataSourceReference() {
         return dataSourceReference;
     }
 
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        ILogicalOperator topOp = dataSourceReference.getOperator();
-        for (LogicalOperatorReference i : topOp.getInputs()) {
-            schema.addAll(i.getOperator().getSchema());
+        ILogicalOperator topOp = dataSourceReference.getValue();
+        for (Mutable<ILogicalOperator> i : topOp.getInputs()) {
+            schema.addAll(i.getValue().getSchema());
         }
     }
 
@@ -87,7 +88,7 @@
 
             @Override
             public IVariableTypeEnvironment getTypeEnv() {
-                ILogicalOperator op = dataSourceReference.getOperator().getInputs().get(0).getOperator();
+                ILogicalOperator op = dataSourceReference.getValue().getInputs().get(0).getValue();
                 return ctx.getOutputTypeEnvironment(op);
             }
         };
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
index 9c3cef5..3a5cdde 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
@@ -17,7 +17,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -38,7 +40,7 @@
             DESC
         };
 
-        public LogicalExpressionReference getExpressionRef();
+        public Mutable<ILogicalExpression> getExpressionRef();
 
         public OrderKind getKind();
     }
@@ -46,7 +48,7 @@
     public static IOrder ASC_ORDER = new IOrder() {
 
         @Override
-        public LogicalExpressionReference getExpressionRef() {
+        public Mutable<ILogicalExpression> getExpressionRef() {
             return null;
         }
 
@@ -60,7 +62,7 @@
     public static IOrder DESC_ORDER = new IOrder() {
 
         @Override
-        public LogicalExpressionReference getExpressionRef() {
+        public Mutable<ILogicalExpression> getExpressionRef() {
             return null;
         }
 
@@ -71,14 +73,14 @@
     };
 
     public class FunOrder implements IOrder {
-        private final LogicalExpressionReference f;
+        private final Mutable<ILogicalExpression> f;
 
-        public FunOrder(LogicalExpressionReference f) {
+        public FunOrder(Mutable<ILogicalExpression> f) {
             this.f = f;
         }
 
         @Override
-        public LogicalExpressionReference getExpressionRef() {
+        public Mutable<ILogicalExpression> getExpressionRef() {
             return f;
         }
 
@@ -89,17 +91,17 @@
 
     };
 
-    private final List<Pair<IOrder, LogicalExpressionReference>> orderExpressions;
+    private final List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions;
 
     // These are pairs of type (comparison, expr) where comparison is
     // ASC or DESC or a boolean function of arity 2 that can take as
     // arguments results of expr.
 
     public OrderOperator() {
-        orderExpressions = new ArrayList<Pair<IOrder, LogicalExpressionReference>>();
+        orderExpressions = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
     }
 
-    public OrderOperator(List<Pair<IOrder, LogicalExpressionReference>> orderExpressions) {
+    public OrderOperator(List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions) {
         this.orderExpressions = orderExpressions;
     }
 
@@ -108,13 +110,13 @@
         return LogicalOperatorTag.ORDER;
     }
 
-    public List<Pair<IOrder, LogicalExpressionReference>> getOrderExpressions() {
+    public List<Pair<IOrder, Mutable<ILogicalExpression>>> getOrderExpressions() {
         return orderExpressions;
     }
 
     @Override
     public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getOperator().getSchema());
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
     }
 
     @Override
@@ -125,10 +127,10 @@
     @Override
     public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
         boolean b = false;
-        for (Pair<IOrder, LogicalExpressionReference> p : orderExpressions) {
+        for (Pair<IOrder, Mutable<ILogicalExpression>> p : orderExpressions) {
             if (p.first.getKind() == OrderKind.FUNCTIONCALL) {
                 FunOrder fo = (FunOrder) p.first;
-                LogicalExpressionReference r1 = fo.getExpressionRef();
+                Mutable<ILogicalExpression> r1 = fo.getExpressionRef();
                 if (visitor.transform(r1)) {
                     b = true;
                 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
index dc0e4df..a76237b 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
@@ -16,7 +16,9 @@
 
 import java.util.ArrayList;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -28,15 +30,15 @@
 
 public class PartitioningSplitOperator extends AbstractLogicalOperator {
 
-    private LogicalExpressionReference[] expressions;
+    private Mutable<ILogicalExpression>[] expressions;
     private boolean hasDefault;
 
-    public PartitioningSplitOperator(LogicalExpressionReference[] exprList, boolean defaultBranchDefined) {
+    public PartitioningSplitOperator(Mutable<ILogicalExpression>[] exprList, boolean defaultBranchDefined) {
         expressions = exprList;
         hasDefault = defaultBranchDefined;
     }
 
-    public LogicalExpressionReference[] getExpressions() {
+    public Mutable<ILogicalExpression>[] getExpressions() {
         return expressions;
     }
 
@@ -52,7 +54,7 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
     }
 
     @Override
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
index 0f0c624..f594373 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
@@ -60,7 +60,7 @@
 
     @Override
     public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getOperator().getSchema());
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
     }
 
     public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
index d9be7d4..798597e 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
@@ -16,7 +16,9 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -27,7 +29,7 @@
 
 public class RunningAggregateOperator extends AbstractAssignOperator {
 
-    public RunningAggregateOperator(List<LogicalVariable> variables, List<LogicalExpressionReference> expressions) {
+    public RunningAggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
         super(variables, expressions);
     }
 
@@ -68,8 +70,8 @@
         for (int i = 0; i < n; i++) {
             env.setVarType(
                     variables.get(i),
-                    ctx.getExpressionTypeComputer().getType(expressions.get(i).getExpression(),
-                            ctx.getMetadataProvider(), env));
+                    ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
+                            env));
         }
         return env;
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
index 93db0e9..c1e038e 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
@@ -16,8 +16,9 @@
 
 import java.util.ArrayList;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
@@ -36,9 +37,9 @@
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public class SelectOperator extends AbstractLogicalOperator {
-    private final LogicalExpressionReference condition;
+    private final Mutable<ILogicalExpression> condition;
 
-    public SelectOperator(LogicalExpressionReference condition) {
+    public SelectOperator(Mutable<ILogicalExpression> condition) {
         this.condition = condition;
     }
 
@@ -47,13 +48,13 @@
         return LogicalOperatorTag.SELECT;
     }
 
-    public LogicalExpressionReference getCondition() {
+    public Mutable<ILogicalExpression> getCondition() {
         return condition;
     }
 
     @Override
     public void recomputeSchema() {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getOperator().getSchema());
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
     }
 
     @Override
@@ -82,14 +83,14 @@
         envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
         PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
                 ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
-        if (condition.getExpression().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) condition.getExpression();
+        if (condition.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+            AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) condition.getValue();
             if (f1.getFunctionIdentifier() == AlgebricksBuiltinFunctions.NOT) {
-                ILogicalExpression a1 = f1.getArguments().get(0).getExpression();
+                ILogicalExpression a1 = f1.getArguments().get(0).getValue();
                 if (a1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
                     AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
                     if (f2.getFunctionIdentifier() == AlgebricksBuiltinFunctions.IS_NULL) {
-                        ILogicalExpression a2 = f2.getArguments().get(0).getExpression();
+                        ILogicalExpression a2 = f2.getArguments().get(0).getValue();
                         if (a2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                             LogicalVariable var = ((VariableReferenceExpression) a2).getVariableReference();
                             env.getNonNullVariables().add(var);
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
index a40df28..a75a9a9 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
@@ -19,7 +19,7 @@
 
     @Override
     public void recomputeSchema() throws AlgebricksException {
-        schema = new ArrayList<LogicalVariable>(inputs.get(0).getOperator().getSchema());
+        schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
     }
 
     @Override
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
index a60d58d..0d1704b 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
@@ -18,9 +18,11 @@
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -46,12 +48,12 @@
     }
 
     public SubplanOperator(ILogicalOperator planRoot) {
-        ArrayList<LogicalOperatorReference> roots = new ArrayList<LogicalOperatorReference>(1);
-        roots.add(new LogicalOperatorReference(planRoot));
+        ArrayList<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>(1);
+        roots.add(new MutableObject<ILogicalOperator>(planRoot));
         nestedPlans.add(new ALogicalPlanImpl(roots));
     }
 
-    public void setRootOp(LogicalOperatorReference opRef) {
+    public void setRootOp(Mutable<ILogicalOperator> opRef) {
         ILogicalPlan p = new ALogicalPlanImpl(opRef);
         nestedPlans.add(p);
     }
@@ -97,7 +99,7 @@
         envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
         int i = 1;
         for (ILogicalPlan p : nestedPlans) {
-            for (LogicalOperatorReference r : p.getRoots()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                 envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
                 i++;
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
index 9b78ae3..cec7d6f 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
@@ -74,7 +74,7 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        for (LogicalVariable v1 : inputs.get(0).getOperator().getSchema()) {
+        for (LogicalVariable v1 : inputs.get(0).getValue().getSchema()) {
             for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
                 if (t.first.equals(v1)) {
                     schema.add(t.third);
@@ -83,7 +83,7 @@
                 }
             }
         }
-        for (LogicalVariable v2 : inputs.get(1).getOperator().getSchema()) {
+        for (LogicalVariable v2 : inputs.get(1).getValue().getSchema()) {
             for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
                 if (t.second.equals(v2)) {
                     schema.add(t.third);
@@ -101,9 +101,9 @@
 
     @Override
     public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
-        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx
-                .getMetadataProvider());
-        IVariableTypeEnvironment envLeft = ctx.getOutputTypeEnvironment(inputs.get(0).getOperator());
+        IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+                ctx.getMetadataProvider());
+        IVariableTypeEnvironment envLeft = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
         if (envLeft == null) {
             throw new AlgebricksException("Left input types for union operator are not computed.");
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
index 7056d83..c4f5cb0 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -16,7 +16,9 @@
 
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -36,7 +38,7 @@
 
     // deprecate UnnestMap
 
-    public UnnestMapOperator(List<LogicalVariable> variables, LogicalExpressionReference expression,
+    public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
             List<Object> variableTypes) {
         super(variables, expression);
         this.variableTypes = variableTypes;
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
index 613c83b..f85cabb 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
@@ -16,7 +16,9 @@
 
 import java.util.ArrayList;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -29,11 +31,11 @@
     private LogicalVariable positionalVariable;
     private Object positionalVariableType;
 
-    public UnnestOperator(LogicalVariable variable, LogicalExpressionReference expression) {
+    public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression) {
         super(makeSingletonList(variable), expression);
     }
 
-    public UnnestOperator(LogicalVariable variable, LogicalExpressionReference expression,
+    public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression,
             LogicalVariable positionalVariable, Object positionalVariableType) {
         this(variable, expression);
         this.setPositionalVariable(positionalVariable);
@@ -79,7 +81,7 @@
     @Override
     public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
         IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
-        Object t = env.getType(expression.getExpression());
+        Object t = env.getType(expression.getValue());
         env.setVarType(variables.get(0), t);
         if (positionalVariable != null) {
             env.setVarType(positionalVariable, positionalVariableType);
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
index c3559f1..fdcdfd8 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
@@ -17,7 +17,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -29,15 +31,15 @@
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public class WriteOperator extends AbstractLogicalOperator {
-    private List<LogicalExpressionReference> expressions;
+    private List<Mutable<ILogicalExpression>> expressions;
     private IDataSink dataSink;
 
-    public WriteOperator(List<LogicalExpressionReference> expressions, IDataSink dataSink) {
+    public WriteOperator(List<Mutable<ILogicalExpression>> expressions, IDataSink dataSink) {
         this.expressions = expressions;
         this.dataSink = dataSink;
     }
 
-    public List<LogicalExpressionReference> getExpressions() {
+    public List<Mutable<ILogicalExpression>> getExpressions() {
         return expressions;
     }
 
@@ -80,7 +82,7 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
     }
 
     @Override
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
index e8150d2..6d05c76 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
@@ -17,7 +17,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -31,11 +33,11 @@
 public class WriteResultOperator extends AbstractLogicalOperator {
 
     private IDataSource<?> dataSource;
-    private LogicalExpressionReference payloadExpr;
-    private List<LogicalExpressionReference> keyExprs;
+    private Mutable<ILogicalExpression> payloadExpr;
+    private List<Mutable<ILogicalExpression>> keyExprs;
 
-    public WriteResultOperator(IDataSource<?> dataSource, LogicalExpressionReference payload,
-            List<LogicalExpressionReference> keyExprs) {
+    public WriteResultOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
+            List<Mutable<ILogicalExpression>> keyExprs) {
         this.dataSource = dataSource;
         this.payloadExpr = payload;
         this.keyExprs = keyExprs;
@@ -45,11 +47,11 @@
         return dataSource;
     }
 
-    public LogicalExpressionReference getPayloadExpression() {
+    public Mutable<ILogicalExpression> getPayloadExpression() {
         return payloadExpr;
     }
 
-    public List<LogicalExpressionReference> getKeyExpressions() {
+    public List<Mutable<ILogicalExpression>> getKeyExpressions() {
         return keyExprs;
     }
 
@@ -87,7 +89,7 @@
     @Override
     public void recomputeSchema() {
         schema = new ArrayList<LogicalVariable>();
-        schema.addAll(inputs.get(0).getOperator().getSchema());
+        schema.addAll(inputs.get(0).getValue().getSchema());
     }
 
     @Override
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
index 8ea552a..0c0f257 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
@@ -23,20 +23,20 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
@@ -86,7 +86,7 @@
 
     @Override
     public Void visitAssignOperator(AssignOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
         Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
         ctx.putEquivalenceClassMap(op, eqClasses);
         List<LogicalVariable> used = new ArrayList<LogicalVariable>();
@@ -110,7 +110,7 @@
 
     @Override
     public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
         Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
         ctx.putEquivalenceClassMap(op, eqClasses);
         List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
@@ -121,7 +121,7 @@
 
     @Override
     public Void visitDistinctOperator(DistinctOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator op0 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
         List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
         ctx.putFDList(op, functionalDependencies);
         for (FunctionalDependency inherited : getOrComputeFDs(op0, ctx)) {
@@ -146,9 +146,9 @@
             }
         }
         Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
-        List<LogicalExpressionReference> expressions = op.getExpressions();
-        for (LogicalExpressionReference pRef : expressions) {
-            ILogicalExpression p = pRef.getExpression();
+        List<Mutable<ILogicalExpression>> expressions = op.getExpressions();
+        for (Mutable<ILogicalExpression> pRef : expressions) {
+            ILogicalExpression p = pRef.getValue();
             if (p.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression v = (VariableReferenceExpression) p;
                 gbySet.add(v.getVariableReference());
@@ -161,9 +161,9 @@
 
         lgp.normalizeGroupingColumns(equivalenceClasses, functionalDependencies);
         Set<LogicalVariable> normSet = lgp.getColumnSet();
-        List<LogicalExpressionReference> newDistinctByList = new ArrayList<LogicalExpressionReference>();
-        for (LogicalExpressionReference p2Ref : expressions) {
-            ILogicalExpression p2 = p2Ref.getExpression();
+        List<Mutable<ILogicalExpression>> newDistinctByList = new ArrayList<Mutable<ILogicalExpression>>();
+        for (Mutable<ILogicalExpression> p2Ref : expressions) {
+            ILogicalExpression p2 = p2Ref.getValue();
             if (p2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression var2 = (VariableReferenceExpression) p2;
                 LogicalVariable v2 = var2.getVariableReference();
@@ -202,14 +202,14 @@
 
         List<FunctionalDependency> inheritedFDs = new ArrayList<FunctionalDependency>();
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                ILogicalOperator op2 = r.getOperator();
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                ILogicalOperator op2 = r.getValue();
                 equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
                 inheritedFDs.addAll(getOrComputeFDs(op2, ctx));
             }
         }
 
-        ILogicalOperator op0 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
         inheritedFDs.addAll(getOrComputeFDs(op0, ctx));
         Map<LogicalVariable, EquivalenceClass> inheritedEcs = getOrComputeEqClasses(op0, ctx);
         for (FunctionalDependency inherited : inheritedFDs) {
@@ -243,15 +243,15 @@
         }
 
         List<LogicalVariable> premiseGby = new LinkedList<LogicalVariable>();
-        List<Pair<LogicalVariable, LogicalExpressionReference>> gByList = op.getGroupByList();
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gByList) {
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList = op.getGroupByList();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
             premiseGby.add(p.first);
         }
 
-        List<Pair<LogicalVariable, LogicalExpressionReference>> decorList = op.getDecorList();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = op.getDecorList();
 
         LinkedList<LogicalVariable> conclDecor = new LinkedList<LogicalVariable>();
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : decorList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
             conclDecor.add(GroupByOperator.getDecorVariable(p));
         }
         if (!conclDecor.isEmpty()) {
@@ -259,8 +259,8 @@
         }
 
         Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gByList) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            ILogicalExpression expr = p.second.getValue();
             if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression v = (VariableReferenceExpression) expr;
                 gbySet.add(v.getVariableReference());
@@ -269,10 +269,10 @@
         LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
         lgp.normalizeGroupingColumns(inheritedEcs, inheritedFDs);
         Set<LogicalVariable> normSet = lgp.getColumnSet();
-        List<Pair<LogicalVariable, LogicalExpressionReference>> newGbyList = new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newGbyList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
         boolean changed = false;
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gByList) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+            ILogicalExpression expr = p.second.getValue();
             if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                 LogicalVariable v2 = varRef.getVariableReference();
@@ -309,13 +309,13 @@
         List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
         ctx.putEquivalenceClassMap(op, equivalenceClasses);
         ctx.putFDList(op, functionalDependencies);
-        ILogicalOperator op0 = op.getInputs().get(0).getOperator();
-        ILogicalOperator op1 = op.getInputs().get(1).getOperator();
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
+        ILogicalOperator op1 = op.getInputs().get(1).getValue();
         functionalDependencies.addAll(getOrComputeFDs(op0, ctx));
         functionalDependencies.addAll(getOrComputeFDs(op1, ctx));
         equivalenceClasses.putAll(getOrComputeEqClasses(op0, ctx));
         equivalenceClasses.putAll(getOrComputeEqClasses(op1, ctx));
-        ILogicalExpression expr = op.getCondition().getExpression();
+        ILogicalExpression expr = op.getCondition().getValue();
         expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
         return null;
     }
@@ -327,8 +327,8 @@
         List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
         ctx.putEquivalenceClassMap(op, equivalenceClasses);
         ctx.putFDList(op, functionalDependencies);
-        ILogicalOperator opLeft = op.getInputs().get(0).getOperator();
-        ILogicalOperator opRight = op.getInputs().get(1).getOperator();
+        ILogicalOperator opLeft = op.getInputs().get(0).getValue();
+        ILogicalOperator opRight = op.getInputs().get(1).getValue();
         functionalDependencies.addAll(getOrComputeFDs(opLeft, ctx));
         functionalDependencies.addAll(getOrComputeFDs(opRight, ctx));
         equivalenceClasses.putAll(getOrComputeEqClasses(opLeft, ctx));
@@ -343,7 +343,7 @@
         } else {
             leftSideVars = opLeft.getSchema();
         }
-        ILogicalExpression expr = op.getCondition().getExpression();
+        ILogicalExpression expr = op.getCondition().getValue();
         expr.getConstraintsForOuterJoin(functionalDependencies, leftSideVars);
         return null;
     }
@@ -363,8 +363,8 @@
     @Override
     public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, IOptimizationContext ctx)
             throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getDataSourceReference().getOperator();
-        ILogicalOperator inp1 = op1.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getDataSourceReference().getValue();
+        ILogicalOperator inp1 = op1.getInputs().get(0).getValue();
         Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
         ctx.putEquivalenceClassMap(op, eqClasses);
         List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
@@ -425,10 +425,10 @@
         List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
         ctx.putEquivalenceClassMap(op, equivalenceClasses);
         ctx.putFDList(op, functionalDependencies);
-        ILogicalOperator op0 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
         functionalDependencies.addAll(getOrComputeFDs(op0, ctx));
         equivalenceClasses.putAll(getOrComputeEqClasses(op0, ctx));
-        ILogicalExpression expr = op.getCondition().getExpression();
+        ILogicalExpression expr = op.getCondition().getValue();
         expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
         return null;
     }
@@ -440,8 +440,8 @@
         ctx.putEquivalenceClassMap(op, equivalenceClasses);
         ctx.putFDList(op, functionalDependencies);
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                ILogicalOperator op2 = r.getOperator();
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                ILogicalOperator op2 = r.getValue();
                 equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
                 functionalDependencies.addAll(getOrComputeFDs(op2, ctx));
             }
@@ -501,7 +501,7 @@
     }
 
     private void propagateFDsAndEquivClasses(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
         Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
         ctx.putEquivalenceClassMap(op, eqClasses);
         List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
@@ -530,7 +530,7 @@
 
     private void propagateFDsAndEquivClassesForUsedVars(ILogicalOperator op, IOptimizationContext ctx,
             List<LogicalVariable> usedVariables) throws AlgebricksException {
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         Map<LogicalVariable, EquivalenceClass> eqClasses = new HashMap<LogicalVariable, EquivalenceClass>();
         ctx.putEquivalenceClassMap(op, eqClasses);
         List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
@@ -584,13 +584,13 @@
 
     private void fdsEqClassesForAbstractUnnestOperator(AbstractUnnestOperator op, IOptimizationContext ctx)
             throws AlgebricksException {
-        ILogicalOperator inp1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator inp1 = op.getInputs().get(0).getValue();
         Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
         ctx.putEquivalenceClassMap(op, eqClasses);
         List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
         ctx.putFDList(op, fds);
 
-        ILogicalExpression expr = op.getExpressionRef().getExpression();
+        ILogicalExpression expr = op.getExpressionRef().getValue();
         if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
             AbstractFunctionCallExpression afe = (AbstractFunctionCallExpression) expr;
             if (afe.getKind() == FunctionKind.UNNEST && ((UnnestingFunctionCallExpression) afe).returnsUniqueValues()) {
@@ -612,8 +612,8 @@
     }
 
     private LogicalVariable getNewGbyVar(GroupByOperator g, LogicalVariable v) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : g.getGroupByList()) {
-            ILogicalExpression e = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getGroupByList()) {
+            ILogicalExpression e = p.second.getValue();
             if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
                 if (v2 == v) {
@@ -625,8 +625,8 @@
     }
 
     private LogicalVariable getNewDecorVar(GroupByOperator g, LogicalVariable v) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : g.getDecorList()) {
-            ILogicalExpression e = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
+            ILogicalExpression e = p.second.getValue();
             if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
                 if (v2 == v) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index 694324c..5d9e6b9 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -21,11 +21,12 @@
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
@@ -45,6 +46,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
@@ -58,7 +60,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
@@ -117,17 +118,17 @@
                 || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag())
             return Boolean.FALSE;
 
-        List<Pair<LogicalVariable, LogicalExpressionReference>> keyLists = op.getGroupByList();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyLists = op.getGroupByList();
         GroupByOperator gbyOpArg = (GroupByOperator) copyAndSubstituteVar(op, arg);
-        List<Pair<LogicalVariable, LogicalExpressionReference>> keyListsArg = gbyOpArg.getGroupByList();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyListsArg = gbyOpArg.getGroupByList();
 
         List<Pair<LogicalVariable, ILogicalExpression>> listLeft = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
         List<Pair<LogicalVariable, ILogicalExpression>> listRight = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
 
-        for (Pair<LogicalVariable, LogicalExpressionReference> pair : keyLists)
-            listLeft.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getExpression()));
-        for (Pair<LogicalVariable, LogicalExpressionReference> pair : keyListsArg)
-            listRight.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getExpression()));
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyLists)
+            listLeft.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyListsArg)
+            listRight.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
 
         boolean isomorphic = VariableUtilities.varListEqualUnordered(listLeft, listRight);
 
@@ -142,13 +143,13 @@
         List<ILogicalPlan> plans = op.getNestedPlans();
         List<ILogicalPlan> plansArg = argOp.getNestedPlans();
         for (int i = 0; i < plans.size(); i++) {
-            List<LogicalOperatorReference> roots = plans.get(i).getRoots();
-            List<LogicalOperatorReference> rootsArg = plansArg.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
             if (roots.size() != rootsArg.size())
                 return Boolean.FALSE;
             for (int j = 0; j < roots.size(); j++) {
-                ILogicalOperator topOp1 = roots.get(j).getOperator();
-                ILogicalOperator topOp2 = rootsArg.get(j).getOperator();
+                ILogicalOperator topOp1 = roots.get(j).getValue();
+                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
                 isomorphic = this.checkBottomUp(topOp1, topOp2);
                 if (!isomorphic)
                     return Boolean.FALSE;
@@ -165,7 +166,7 @@
         LimitOperator limitOpArg = (LimitOperator) copyAndSubstituteVar(op, arg);
         if (op.getOffset() != limitOpArg.getOffset())
             return Boolean.FALSE;
-        boolean isomorphic = op.getMaxObjects().getExpression().equals(limitOpArg.getMaxObjects().getExpression());
+        boolean isomorphic = op.getMaxObjects().getValue().equals(limitOpArg.getMaxObjects().getValue());
         return isomorphic;
     }
 
@@ -175,7 +176,7 @@
         if (aop.getOperatorTag() != LogicalOperatorTag.DIE)
             return Boolean.FALSE;
         DieOperator dieOpArg = (DieOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getAfterObjects().getExpression().equals(dieOpArg.getAfterObjects().getExpression());
+        boolean isomorphic = op.getAfterObjects().getValue().equals(dieOpArg.getAfterObjects().getValue());
         return isomorphic;
     }
 
@@ -185,7 +186,7 @@
         if (aop.getOperatorTag() != LogicalOperatorTag.INNERJOIN)
             return Boolean.FALSE;
         InnerJoinOperator joinOpArg = (InnerJoinOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getCondition().getExpression().equals(joinOpArg.getCondition().getExpression());
+        boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
         return isomorphic;
     }
 
@@ -196,7 +197,7 @@
         if (aop.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN)
             return Boolean.FALSE;
         LeftOuterJoinOperator joinOpArg = (LeftOuterJoinOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getCondition().getExpression().equals(joinOpArg.getCondition().getExpression());
+        boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
         return isomorphic;
     }
 
@@ -237,7 +238,7 @@
         if (aop.getOperatorTag() != LogicalOperatorTag.SELECT)
             return Boolean.FALSE;
         SelectOperator selectOpArg = (SelectOperator) copyAndSubstituteVar(op, arg);
-        boolean isomorphic = op.getCondition().getExpression().equals(selectOpArg.getCondition().getExpression());
+        boolean isomorphic = op.getCondition().getValue().equals(selectOpArg.getCondition().getValue());
         return isomorphic;
     }
 
@@ -290,13 +291,13 @@
         List<ILogicalPlan> plans = op.getNestedPlans();
         List<ILogicalPlan> plansArg = subplanOpArg.getNestedPlans();
         for (int i = 0; i < plans.size(); i++) {
-            List<LogicalOperatorReference> roots = plans.get(i).getRoots();
-            List<LogicalOperatorReference> rootsArg = plansArg.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
             if (roots.size() == rootsArg.size())
                 return Boolean.FALSE;
             for (int j = 0; j < roots.size(); j++) {
-                ILogicalOperator topOp1 = roots.get(j).getOperator();
-                ILogicalOperator topOp2 = rootsArg.get(j).getOperator();
+                ILogicalOperator topOp1 = roots.get(j).getValue();
+                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
                 boolean isomorphic = this.checkBottomUp(topOp1, topOp2);
                 if (!isomorphic)
                     return Boolean.FALSE;
@@ -328,7 +329,7 @@
                 && variableEqual(op.getPositionalVariable(), unnestOpArg.getPositionalVariable());
         if (!isomorphic)
             return Boolean.FALSE;
-        isomorphic = op.getExpressionRef().getExpression().equals(unnestOpArg.getExpressionRef().getExpression());
+        isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
         return isomorphic;
     }
 
@@ -341,7 +342,7 @@
         boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables());
         if (!isomorphic)
             return Boolean.FALSE;
-        isomorphic = op.getExpressionRef().getExpression().equals(unnestOpArg.getExpressionRef().getExpression());
+        isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
         return isomorphic;
     }
 
@@ -462,27 +463,27 @@
         return true;
     }
 
-    private Boolean compareExpressions(List<LogicalExpressionReference> opExprs,
-            List<LogicalExpressionReference> argExprs) {
+    private Boolean compareExpressions(List<Mutable<ILogicalExpression>> opExprs,
+            List<Mutable<ILogicalExpression>> argExprs) {
         if (opExprs.size() != argExprs.size())
             return Boolean.FALSE;
         for (int i = 0; i < opExprs.size(); i++) {
-            boolean isomorphic = opExprs.get(i).getExpression().equals(argExprs.get(i).getExpression());
+            boolean isomorphic = opExprs.get(i).getValue().equals(argExprs.get(i).getValue());
             if (!isomorphic)
                 return Boolean.FALSE;
         }
         return Boolean.TRUE;
     }
 
-    private Boolean compareIOrderAndExpressions(List<Pair<IOrder, LogicalExpressionReference>> opOrderExprs,
-            List<Pair<IOrder, LogicalExpressionReference>> argOrderExprs) {
+    private Boolean compareIOrderAndExpressions(List<Pair<IOrder, Mutable<ILogicalExpression>>> opOrderExprs,
+            List<Pair<IOrder, Mutable<ILogicalExpression>>> argOrderExprs) {
         if (opOrderExprs.size() != argOrderExprs.size())
             return Boolean.FALSE;
         for (int i = 0; i < opOrderExprs.size(); i++) {
             boolean isomorphic = opOrderExprs.get(i).first.equals(argOrderExprs.get(i).first);
             if (!isomorphic)
                 return Boolean.FALSE;
-            isomorphic = opOrderExprs.get(i).second.getExpression().equals(argOrderExprs.get(i).second.getExpression());
+            isomorphic = opOrderExprs.get(i).second.getValue().equals(argOrderExprs.get(i).second.getValue());
             if (!isomorphic)
                 return Boolean.FALSE;
         }
@@ -490,13 +491,13 @@
     }
 
     private Boolean checkBottomUp(ILogicalOperator op1, ILogicalOperator op2) throws AlgebricksException {
-        List<LogicalOperatorReference> inputs1 = op1.getInputs();
-        List<LogicalOperatorReference> inputs2 = op2.getInputs();
+        List<Mutable<ILogicalOperator>> inputs1 = op1.getInputs();
+        List<Mutable<ILogicalOperator>> inputs2 = op2.getInputs();
         if (inputs1.size() != inputs2.size())
             return Boolean.FALSE;
         for (int i = 0; i < inputs1.size(); i++) {
-            ILogicalOperator input1 = inputs1.get(i).getOperator();
-            ILogicalOperator input2 = inputs2.get(i).getOperator();
+            ILogicalOperator input1 = inputs1.get(i).getValue();
+            ILogicalOperator input2 = inputs2.get(i).getValue();
             boolean isomorphic = checkBottomUp(input1, input2);
             if (!isomorphic)
                 return Boolean.FALSE;
@@ -513,7 +514,7 @@
         List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
         if (argOp.getInputs().size() > 0)
             for (int i = 0; i < argOp.getInputs().size(); i++)
-                VariableUtilities.getLiveVariables(argOp.getInputs().get(i).getOperator(), liveVars);
+                VariableUtilities.getLiveVariables(argOp.getInputs().get(i).getValue(), liveVars);
         List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
         VariableUtilities.getProducedVariables(argOp, producedVars);
         List<LogicalVariable> producedVarsNew = new ArrayList<LogicalVariable>();
@@ -532,12 +533,12 @@
     }
 
     public List<Pair<LogicalVariable, ILogicalExpression>> getPairList(List<LogicalVariable> vars,
-            List<LogicalExpressionReference> exprs) throws AlgebricksException {
+            List<Mutable<ILogicalExpression>> exprs) throws AlgebricksException {
         List<Pair<LogicalVariable, ILogicalExpression>> list = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
         if (vars.size() != exprs.size())
             throw new AlgebricksException("variable list size does not equal to expression list size ");
         for (int i = 0; i < vars.size(); i++) {
-            list.add(new Pair<LogicalVariable, ILogicalExpression>(vars.get(i), exprs.get(i).getExpression()));
+            list.add(new Pair<LogicalVariable, ILogicalExpression>(vars.get(i), exprs.get(i).getValue()));
         }
         return list;
     }
@@ -548,18 +549,18 @@
     }
 
     private static ILogicalPlan deepCopy(ILogicalPlan plan) throws AlgebricksException {
-        List<LogicalOperatorReference> roots = plan.getRoots();
-        List<LogicalOperatorReference> newRoots = new ArrayList<LogicalOperatorReference>();
-        for (LogicalOperatorReference opRef : roots)
-            newRoots.add(new LogicalOperatorReference(bottomUpCopyOperators(opRef.getOperator())));
+        List<Mutable<ILogicalOperator>> roots = plan.getRoots();
+        List<Mutable<ILogicalOperator>> newRoots = new ArrayList<Mutable<ILogicalOperator>>();
+        for (Mutable<ILogicalOperator> opRef : roots)
+            newRoots.add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(opRef.getValue())));
         return new ALogicalPlanImpl(newRoots);
     }
 
     private static ILogicalOperator bottomUpCopyOperators(ILogicalOperator op) throws AlgebricksException {
         ILogicalOperator newOp = deepCopy(op);
         newOp.getInputs().clear();
-        for (LogicalOperatorReference child : op.getInputs())
-            newOp.getInputs().add(new LogicalOperatorReference(bottomUpCopyOperators(child.getOperator())));
+        for (Mutable<ILogicalOperator> child : op.getInputs())
+            newOp.getInputs().add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(child.getValue())));
         return newOp;
     }
 
@@ -577,7 +578,7 @@
         @Override
         public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
             ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
-            ArrayList<LogicalExpressionReference> newExpressions = new ArrayList<LogicalExpressionReference>();
+            ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             newList.addAll(op.getVariables());
             deepCopyExpressionRefs(newExpressions, op.getExpressions());
             return new AggregateOperator(newList, newExpressions);
@@ -587,7 +588,7 @@
         public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
                 throws AlgebricksException {
             ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
-            ArrayList<LogicalExpressionReference> newExpressions = new ArrayList<LogicalExpressionReference>();
+            ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             newList.addAll(op.getVariables());
             deepCopyExpressionRefs(newExpressions, op.getExpressions());
             return new RunningAggregateOperator(newList, newExpressions);
@@ -601,14 +602,14 @@
 
         @Override
         public ILogicalOperator visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
-            List<Pair<LogicalVariable, LogicalExpressionReference>> groupByList = new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>();
-            List<Pair<LogicalVariable, LogicalExpressionReference>> decoList = new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>();
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decoList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
             ArrayList<ILogicalPlan> newSubplans = new ArrayList<ILogicalPlan>();
-            for (Pair<LogicalVariable, LogicalExpressionReference> pair : op.getGroupByList())
-                groupByList.add(new Pair<LogicalVariable, LogicalExpressionReference>(pair.first,
+            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getGroupByList())
+                groupByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
                         deepCopyExpressionRef(pair.second)));
-            for (Pair<LogicalVariable, LogicalExpressionReference> pair : op.getDecorList())
-                decoList.add(new Pair<LogicalVariable, LogicalExpressionReference>(pair.first,
+            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getDecorList())
+                decoList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
                         deepCopyExpressionRef(pair.second)));
             for (ILogicalPlan plan : op.getNestedPlans()) {
                 newSubplans.add(IsomorphismOperatorVisitor.deepCopy(plan));
@@ -618,13 +619,13 @@
 
         @Override
         public ILogicalOperator visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
-            return new LimitOperator(deepCopyExpressionRef(op.getMaxObjects()).getExpression(), deepCopyExpressionRef(
-                    op.getOffset()).getExpression(), op.isTopmostLimitOp());
+            return new LimitOperator(deepCopyExpressionRef(op.getMaxObjects()).getValue(), deepCopyExpressionRef(
+                    op.getOffset()).getValue(), op.isTopmostLimitOp());
         }
 
         @Override
         public ILogicalOperator visitDieOperator(DieOperator op, Void arg) throws AlgebricksException {
-            return new DieOperator(deepCopyExpressionRef(op.getAfterObjects()).getExpression());
+            return new DieOperator(deepCopyExpressionRef(op.getAfterObjects()).getValue());
         }
 
         @Override
@@ -654,7 +655,7 @@
         @Override
         public ILogicalOperator visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
             ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
-            ArrayList<LogicalExpressionReference> newExpressions = new ArrayList<LogicalExpressionReference>();
+            ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             newList.addAll(op.getVariables());
             deepCopyExpressionRefs(newExpressions, op.getExpressions());
             return new AssignOperator(newList, newExpressions);
@@ -675,9 +676,9 @@
         @Override
         public ILogicalOperator visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg)
                 throws AlgebricksException {
-            ArrayList<LogicalExpressionReference> newExpressions = new ArrayList<LogicalExpressionReference>();
+            ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             deepCopyExpressionRefs(newExpressions, Arrays.asList(op.getExpressions()));
-            return new PartitioningSplitOperator(newExpressions.toArray(new LogicalExpressionReference[0]),
+            return new PartitioningSplitOperator(newExpressions.toArray(new Mutable[0]),
                     op.hasDefault());
         }
 
@@ -737,7 +738,7 @@
 
         @Override
         public ILogicalOperator visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
-            ArrayList<LogicalExpressionReference> newExpressions = new ArrayList<LogicalExpressionReference>();
+            ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             deepCopyExpressionRefs(newExpressions, op.getExpressions());
             return new DistinctOperator(newExpressions);
         }
@@ -749,14 +750,14 @@
 
         @Override
         public ILogicalOperator visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
-            ArrayList<LogicalExpressionReference> newExpressions = new ArrayList<LogicalExpressionReference>();
+            ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             deepCopyExpressionRefs(newExpressions, op.getExpressions());
             return new WriteOperator(newExpressions, op.getDataSink());
         }
 
         @Override
         public ILogicalOperator visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
-            ArrayList<LogicalExpressionReference> newKeyExpressions = new ArrayList<LogicalExpressionReference>();
+            ArrayList<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             deepCopyExpressionRefs(newKeyExpressions, op.getKeyExpressions());
             return new WriteResultOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()),
                     newKeyExpressions);
@@ -764,7 +765,7 @@
 
         @Override
         public ILogicalOperator visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
-            List<LogicalExpressionReference> newKeyExpressions = new ArrayList<LogicalExpressionReference>();
+            List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions());
             return new InsertDeleteOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()),
                     newKeyExpressions, op.getOperation());
@@ -773,9 +774,9 @@
         @Override
         public ILogicalOperator visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg)
                 throws AlgebricksException {
-            List<LogicalExpressionReference> newPrimaryKeyExpressions = new ArrayList<LogicalExpressionReference>();
+            List<Mutable<ILogicalExpression>> newPrimaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             deepCopyExpressionRefs(newPrimaryKeyExpressions, op.getPrimaryKeyExpressions());
-            List<LogicalExpressionReference> newSecondaryKeyExpressions = new ArrayList<LogicalExpressionReference>();
+            List<Mutable<ILogicalExpression>> newSecondaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
             deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
             return new IndexInsertDeleteOperator(op.getDataSourceIndex(), newPrimaryKeyExpressions,
                     newSecondaryKeyExpressions, op.getOperation());
@@ -786,23 +787,23 @@
             return new SinkOperator();
         }
 
-        private void deepCopyExpressionRefs(List<LogicalExpressionReference> newExprs,
-                List<LogicalExpressionReference> oldExprs) {
-            for (LogicalExpressionReference oldExpr : oldExprs)
-                newExprs.add(new LogicalExpressionReference(((AbstractLogicalExpression) oldExpr.getExpression())
+        private void deepCopyExpressionRefs(List<Mutable<ILogicalExpression>> newExprs,
+                List<Mutable<ILogicalExpression>> oldExprs) {
+            for (Mutable<ILogicalExpression> oldExpr : oldExprs)
+                newExprs.add(new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue())
                         .cloneExpression()));
         }
 
-        private LogicalExpressionReference deepCopyExpressionRef(LogicalExpressionReference oldExpr) {
-            return new LogicalExpressionReference(
-                    ((AbstractLogicalExpression) oldExpr.getExpression()).cloneExpression());
+        private Mutable<ILogicalExpression> deepCopyExpressionRef(Mutable<ILogicalExpression> oldExpr) {
+            return new MutableObject<ILogicalExpression>(
+                    ((AbstractLogicalExpression) oldExpr.getValue()).cloneExpression());
         }
 
-        private List<Pair<IOrder, LogicalExpressionReference>> deepCopyOrderAndExpression(
-                List<Pair<IOrder, LogicalExpressionReference>> ordersAndExprs) {
-            List<Pair<IOrder, LogicalExpressionReference>> newOrdersAndExprs = new ArrayList<Pair<IOrder, LogicalExpressionReference>>();
-            for (Pair<IOrder, LogicalExpressionReference> pair : ordersAndExprs)
-                newOrdersAndExprs.add(new Pair<IOrder, LogicalExpressionReference>(pair.first,
+        private List<Pair<IOrder, Mutable<ILogicalExpression>>> deepCopyOrderAndExpression(
+                List<Pair<IOrder, Mutable<ILogicalExpression>>> ordersAndExprs) {
+            List<Pair<IOrder, Mutable<ILogicalExpression>>> newOrdersAndExprs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+            for (Pair<IOrder, Mutable<ILogicalExpression>> pair : ordersAndExprs)
+                newOrdersAndExprs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(pair.first,
                         deepCopyExpressionRef(pair.second)));
             return newOrdersAndExprs;
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
index 214d6da..340b855 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
@@ -19,11 +19,12 @@
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
@@ -253,13 +254,13 @@
     }
 
     private void mapChildren(ILogicalOperator op, ILogicalOperator opArg) throws AlgebricksException {
-        List<LogicalOperatorReference> inputs = op.getInputs();
-        List<LogicalOperatorReference> inputsArg = opArg.getInputs();
+        List<Mutable<ILogicalOperator>> inputs = op.getInputs();
+        List<Mutable<ILogicalOperator>> inputsArg = opArg.getInputs();
         if (inputs.size() != inputsArg.size())
             throw new AlgebricksException("children are not isomoprhic");
         for (int i = 0; i < inputs.size(); i++) {
-            ILogicalOperator input = inputs.get(i).getOperator();
-            ILogicalOperator inputArg = inputsArg.get(i).getOperator();
+            ILogicalOperator input = inputs.get(i).getValue();
+            ILogicalOperator inputArg = inputsArg.get(i).getValue();
             input.accept(this, inputArg);
         }
     }
@@ -298,23 +299,23 @@
     private void mapVariablesForGroupBy(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
         GroupByOperator leftOp = (GroupByOperator) left;
         GroupByOperator rightOp = (GroupByOperator) right;
-        List<Pair<LogicalVariable, LogicalExpressionReference>> leftPairs = leftOp.getGroupByList();
-        List<Pair<LogicalVariable, LogicalExpressionReference>> rightPairs = rightOp.getGroupByList();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs = leftOp.getGroupByList();
+        List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs = rightOp.getGroupByList();
         mapVarExprPairList(leftPairs, rightPairs);
         leftPairs = leftOp.getDecorList();
         rightPairs = rightOp.getDecorList();
         mapVarExprPairList(leftPairs, rightPairs);
     }
 
-    private void mapVarExprPairList(List<Pair<LogicalVariable, LogicalExpressionReference>> leftPairs,
-            List<Pair<LogicalVariable, LogicalExpressionReference>> rightPairs) {
+    private void mapVarExprPairList(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs,
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs) {
         if (leftPairs.size() != rightPairs.size())
             return;
         for (int i = 0; i < leftPairs.size(); i++) {
-            ILogicalExpression exprLeft = leftPairs.get(i).second.getExpression();
+            ILogicalExpression exprLeft = leftPairs.get(i).second.getValue();
             LogicalVariable leftVar = leftPairs.get(i).first;
             for (int j = 0; j < leftPairs.size(); j++) {
-                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(rightPairs.get(j).second).getExpression();
+                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(rightPairs.get(j).second).getValue();
                 if (exprLeft.equals(exprRight)) {
                     LogicalVariable rightVar = rightPairs.get(j).first;
                     if (rightVar != null && leftVar != null) {
@@ -327,16 +328,16 @@
     }
 
     private void mapVariablesForAbstractAssign(List<LogicalVariable> variablesLeft,
-            List<LogicalExpressionReference> exprsLeft, List<LogicalVariable> variablesRight,
-            List<LogicalExpressionReference> exprsRight) {
+            List<Mutable<ILogicalExpression>> exprsLeft, List<LogicalVariable> variablesRight,
+            List<Mutable<ILogicalExpression>> exprsRight) {
         if (variablesLeft.size() != variablesRight.size())
             return;
         int size = variablesLeft.size();
         for (int i = 0; i < size; i++) {
-            ILogicalExpression exprLeft = exprsLeft.get(i).getExpression();
+            ILogicalExpression exprLeft = exprsLeft.get(i).getValue();
             LogicalVariable left = variablesLeft.get(i);
             for (int j = 0; j < size; j++) {
-                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(exprsRight.get(j)).getExpression();
+                ILogicalExpression exprRight = copyExpressionAndSubtituteVars(exprsRight.get(j)).getValue();
                 if (exprLeft.equals(exprRight)) {
                     LogicalVariable right = variablesRight.get(j);
                     variableMapping.put(right, left);
@@ -354,13 +355,13 @@
         if (plans.size() != plansArg.size())
             return;
         for (int i = 0; i < plans.size(); i++) {
-            List<LogicalOperatorReference> roots = plans.get(i).getRoots();
-            List<LogicalOperatorReference> rootsArg = plansArg.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+            List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
             if (roots.size() != rootsArg.size())
                 return;
             for (int j = 0; j < roots.size(); j++) {
-                ILogicalOperator topOp1 = roots.get(j).getOperator();
-                ILogicalOperator topOp2 = rootsArg.get(j).getOperator();
+                ILogicalOperator topOp1 = roots.get(j).getValue();
+                ILogicalOperator topOp2 = rootsArg.get(j).getValue();
                 topOp1.accept(this, topOp2);
             }
         }
@@ -371,11 +372,11 @@
         mapVariables(op, arg);
     }
 
-    private LogicalExpressionReference copyExpressionAndSubtituteVars(LogicalExpressionReference expr) {
-        ILogicalExpression copy = ((AbstractLogicalExpression) expr.getExpression()).cloneExpression();
+    private Mutable<ILogicalExpression> copyExpressionAndSubtituteVars(Mutable<ILogicalExpression> expr) {
+        ILogicalExpression copy = ((AbstractLogicalExpression) expr.getValue()).cloneExpression();
         for (Entry<LogicalVariable, LogicalVariable> entry : variableMapping.entrySet())
             copy.substituteVar(entry.getKey(), entry.getValue());
-        return new LogicalExpressionReference(copy);
+        return new MutableObject<ILogicalExpression>(copy);
     }
 
     private void mapVariablesForUnion(ILogicalOperator op, ILogicalOperator arg) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
index 950772d..30ad3ab 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
@@ -14,10 +14,11 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
@@ -66,8 +67,8 @@
 
     private static void computeLogicalPropertiesRec(ILogicalOperator op, LogicalPropertiesVisitor visitor,
             IOptimizationContext context) throws AlgebricksException {
-        for (LogicalOperatorReference ref : op.getInputs()) {
-            computeLogicalPropertiesRec(ref.getOperator(), visitor, context);
+        for (Mutable<ILogicalOperator> ref : op.getInputs()) {
+            computeLogicalPropertiesRec(ref.getValue(), visitor, context);
         }
         op.accept(visitor, context);
         if (AlgebricksConfig.DEBUG) {
@@ -263,7 +264,7 @@
     }
 
     private LogicalPropertiesVectorImpl propagateCardinality(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op0 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
         ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
         if (v0 == null) {
             return null;
@@ -280,13 +281,13 @@
             IVariableEvalSizeEnvironment varSizeEnv = context.getVariableEvalSizeEnvironment();
             IExpressionEvalSizeComputer evalSize = context.getExpressionEvalSizeComputer();
             if (evalSize != null) {
-                ILogicalOperator op0 = op.getInputs().get(0).getOperator();
+                ILogicalOperator op0 = op.getInputs().get(0).getValue();
                 ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
                 if (v0 != null) {
                     long frames0 = v0.getMaxOutputFrames();
                     long overhead = 0; // added per tuple
-                    for (LogicalExpressionReference exprRef : op.getExpressions()) {
-                        int sz = evalSize.getEvalSize(exprRef.getExpression(), varSizeEnv);
+                    for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+                        int sz = evalSize.getEvalSize(exprRef.getValue(), varSizeEnv);
                         if (sz == -1) {
                             return;
                         }
@@ -310,7 +311,7 @@
             throws AlgebricksException {
         LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
         // propagate also max number of frames (conservatively)
-        ILogicalOperator op0 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op0 = op.getInputs().get(0).getValue();
         ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
         if (v0 != null) {
             v.setMaxOutputFrames(v0.getMaxOutputFrames());
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
index 1e991f1..cd05624 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
@@ -18,9 +18,11 @@
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
@@ -99,16 +101,16 @@
     @Override
     public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getOperator(), producedVariables);
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : op.getGroupByList()) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
             if (p.first != null) {
                 producedVariables.add(p.first);
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : op.getDecorList()) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
             if (p.first != null) {
                 producedVariables.add(p.first);
             }
@@ -182,8 +184,8 @@
     @Override
     public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getOperator(), producedVariables);
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
             }
         }
         return null;
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
index be03961..6aa3796 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
@@ -16,12 +16,12 @@
 
 import java.util.Collection;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
@@ -103,20 +103,20 @@
     @Override
     public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getOperator(), schemaVariables);
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : op.getGroupByList()) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
             if (p.first != null) {
                 schemaVariables.add(p.first);
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : op.getDecorList()) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
             if (p.first != null) {
                 schemaVariables.add(p.first);
             } else {
-                ILogicalExpression e = p.second.getExpression();
+                ILogicalExpression e = p.second.getValue();
                 if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                     schemaVariables.add(((VariableReferenceExpression) e).getVariableReference());
                 }
@@ -194,13 +194,13 @@
 
     @Override
     public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
-        for (LogicalOperatorReference c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getOperator(), schemaVariables);
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
         }
         VariableUtilities.getProducedVariables(op, schemaVariables);
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                VariableUtilities.getLiveVariables(r.getOperator(), schemaVariables);
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
             }
         }
         return null;
@@ -237,8 +237,8 @@
     }
 
     private void standardLayout(ILogicalOperator op) throws AlgebricksException {
-        for (LogicalOperatorReference c : op.getInputs()) {
-            VariableUtilities.getLiveVariables(c.getOperator(), schemaVariables);
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
         }
         VariableUtilities.getProducedVariables(op, schemaVariables);
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
index d463805..b85c798 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -17,11 +17,11 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -40,6 +40,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
@@ -53,7 +54,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
@@ -80,7 +80,7 @@
             if (variables.get(i).equals(pair.first)) {
                 variables.set(i, pair.second);
             } else {
-                op.getExpressions().get(i).getExpression().substituteVar(pair.first, pair.second);
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
             }
         }
         substVarTypes(op, pair);
@@ -96,7 +96,7 @@
             if (variables.get(i).equals(pair.first)) {
                 variables.set(i, pair.second);
             } else {
-                op.getExpressions().get(i).getExpression().substituteVar(pair.first, pair.second);
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
             }
         }
         substVarTypes(op, pair);
@@ -120,8 +120,8 @@
     @Override
     public Void visitDistinctOperator(DistinctOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        for (LogicalExpressionReference eRef : op.getExpressions()) {
-            eRef.getExpression().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+            eRef.getValue().substituteVar(pair.first, pair.second);
         }
         substVarTypes(op, pair);
         return null;
@@ -145,8 +145,8 @@
         subst(pair.first, pair.second, op.getGroupByList());
         subst(pair.first, pair.second, op.getDecorList());
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getOperator(), pair.first,
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
                         pair.second, goThroughNts, ctx);
             }
         }
@@ -157,7 +157,7 @@
     @Override
     public Void visitInnerJoinOperator(InnerJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        op.getCondition().getExpression().substituteVar(pair.first, pair.second);
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
         substVarTypes(op, pair);
         return null;
     }
@@ -165,7 +165,7 @@
     @Override
     public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        op.getCondition().getExpression().substituteVar(pair.first, pair.second);
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
         substVarTypes(op, pair);
         return null;
     }
@@ -173,8 +173,8 @@
     @Override
     public Void visitLimitOperator(LimitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        op.getMaxObjects().getExpression().substituteVar(pair.first, pair.second);
-        ILogicalExpression offset = op.getOffset().getExpression();
+        op.getMaxObjects().getValue().substituteVar(pair.first, pair.second);
+        ILogicalExpression offset = op.getOffset().getValue();
         if (offset != null) {
             offset.substituteVar(pair.first, pair.second);
         }
@@ -185,7 +185,7 @@
     @Override
     public Void visitDieOperator(DieOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        op.getAfterObjects().getExpression().substituteVar(pair.first, pair.second);
+        op.getAfterObjects().getValue().substituteVar(pair.first, pair.second);
         substVarTypes(op, pair);
         return null;
     }
@@ -199,8 +199,8 @@
     @Override
     public Void visitOrderOperator(OrderOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        for (Pair<IOrder, LogicalExpressionReference> oe : op.getOrderExpressions()) {
-            oe.second.getExpression().substituteVar(pair.first, pair.second);
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+            oe.second.getValue().substituteVar(pair.first, pair.second);
         }
         substVarTypes(op, pair);
         return null;
@@ -209,8 +209,8 @@
     @Override
     public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        for (LogicalExpressionReference e : op.getExpressions()) {
-            e.getExpression().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
         }
         substVarTypes(op, pair);
         return null;
@@ -240,7 +240,7 @@
             if (variables.get(i).equals(pair.first)) {
                 variables.set(i, pair.second);
             } else {
-                op.getExpressions().get(i).getExpression().substituteVar(pair.first, pair.second);
+                op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
             }
         }
         substVarTypes(op, pair);
@@ -258,7 +258,7 @@
 
     @Override
     public Void visitSelectOperator(SelectOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
-        op.getCondition().getExpression().substituteVar(pair.first, pair.second);
+        op.getCondition().getValue().substituteVar(pair.first, pair.second);
         return null;
     }
 
@@ -266,8 +266,8 @@
     public Void visitSubplanOperator(SubplanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getOperator(), pair.first,
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
                         pair.second, goThroughNts, ctx);
             }
         }
@@ -303,7 +303,7 @@
                 return null;
             }
         }
-        op.getExpressionRef().getExpression().substituteVar(pair.first, pair.second);
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
         substVarTypes(op, pair);
         return null;
     }
@@ -318,7 +318,7 @@
                 return null;
             }
         }
-        op.getExpressionRef().getExpression().substituteVar(pair.first, pair.second);
+        op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
         substVarTypes(op, pair);
         return null;
     }
@@ -326,8 +326,8 @@
     @Override
     public Void visitWriteOperator(WriteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        for (LogicalExpressionReference e : op.getExpressions()) {
-            e.getExpression().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
         }
         substVarTypes(op, pair);
         return null;
@@ -336,22 +336,22 @@
     @Override
     public Void visitWriteResultOperator(WriteResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        op.getPayloadExpression().getExpression().substituteVar(pair.first, pair.second);
-        for (LogicalExpressionReference e : op.getKeyExpressions()) {
-            e.getExpression().substituteVar(pair.first, pair.second);
+        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
         }
         substVarTypes(op, pair);
         return null;
     }
 
     private void subst(LogicalVariable v1, LogicalVariable v2,
-            List<Pair<LogicalVariable, LogicalExpressionReference>> varExprPairList) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> ve : varExprPairList) {
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> varExprPairList) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : varExprPairList) {
             if (ve.first != null && ve.first.equals(v1)) {
                 ve.first = v2;
                 return;
             }
-            ve.second.getExpression().substituteVar(v1, v2);
+            ve.second.getValue().substituteVar(v1, v2);
         }
     }
 
@@ -374,9 +374,9 @@
     @Override
     public Void visitInsertDeleteOperator(InsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        op.getPayloadExpression().getExpression().substituteVar(pair.first, pair.second);
-        for (LogicalExpressionReference e : op.getPrimaryKeyExpressions()) {
-            e.getExpression().substituteVar(pair.first, pair.second);
+        op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
         }
         substVarTypes(op, pair);
         return null;
@@ -385,11 +385,11 @@
     @Override
     public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
             throws AlgebricksException {
-        for (LogicalExpressionReference e : op.getPrimaryKeyExpressions()) {
-            e.getExpression().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
         }
-        for (LogicalExpressionReference e : op.getSecondaryKeyExpressions()) {
-            e.getExpression().substituteVar(pair.first, pair.second);
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().substituteVar(pair.first, pair.second);
         }
         substVarTypes(op, pair);
         return null;
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
index 08512d8..d9d6a6a 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -17,10 +17,11 @@
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
@@ -37,6 +38,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
@@ -50,7 +52,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
@@ -66,16 +67,16 @@
 
     @Override
     public Void visitAggregateOperator(AggregateOperator op, Void arg) {
-        for (LogicalExpressionReference exprRef : op.getExpressions()) {
-            exprRef.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
 
     @Override
     public Void visitAssignOperator(AssignOperator op, Void arg) {
-        for (LogicalExpressionReference exprRef : op.getExpressions()) {
-            exprRef.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
@@ -88,8 +89,8 @@
 
     @Override
     public Void visitDistinctOperator(DistinctOperator op, Void arg) {
-        for (LogicalExpressionReference eRef : op.getExpressions()) {
-            eRef.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+            eRef.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
@@ -109,35 +110,35 @@
     @Override
     public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getOperator(), usedVariables);
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
             }
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> g : op.getGroupByList()) {
-            g.second.getExpression().getUsedVariables(usedVariables);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getGroupByList()) {
+            g.second.getValue().getUsedVariables(usedVariables);
         }
-        for (Pair<LogicalVariable, LogicalExpressionReference> g : op.getDecorList()) {
-            g.second.getExpression().getUsedVariables(usedVariables);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getDecorList()) {
+            g.second.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
 
     @Override
     public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) {
-        op.getCondition().getExpression().getUsedVariables(usedVariables);
+        op.getCondition().getValue().getUsedVariables(usedVariables);
         return null;
     }
 
     @Override
     public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) {
-        op.getCondition().getExpression().getUsedVariables(usedVariables);
+        op.getCondition().getValue().getUsedVariables(usedVariables);
         return null;
     }
 
     @Override
     public Void visitLimitOperator(LimitOperator op, Void arg) {
-        op.getMaxObjects().getExpression().getUsedVariables(usedVariables);
-        ILogicalExpression offsetExpr = op.getOffset().getExpression();
+        op.getMaxObjects().getValue().getUsedVariables(usedVariables);
+        ILogicalExpression offsetExpr = op.getOffset().getValue();
         if (offsetExpr != null) {
             offsetExpr.getUsedVariables(usedVariables);
         }
@@ -146,7 +147,7 @@
 
     @Override
     public Void visitDieOperator(DieOperator op, Void arg) {
-        op.getAfterObjects().getExpression().getUsedVariables(usedVariables);
+        op.getAfterObjects().getValue().getUsedVariables(usedVariables);
         return null;
     }
 
@@ -158,16 +159,16 @@
 
     @Override
     public Void visitOrderOperator(OrderOperator op, Void arg) {
-        for (Pair<IOrder, LogicalExpressionReference> oe : op.getOrderExpressions()) {
-            oe.second.getExpression().getUsedVariables(usedVariables);
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+            oe.second.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
 
     @Override
     public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) {
-        for (LogicalExpressionReference e : op.getExpressions()) {
-            e.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
@@ -185,8 +186,8 @@
 
     @Override
     public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) {
-        for (LogicalExpressionReference exprRef : op.getExpressions()) {
-            exprRef.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+            exprRef.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
@@ -204,15 +205,15 @@
 
     @Override
     public Void visitSelectOperator(SelectOperator op, Void arg) {
-        op.getCondition().getExpression().getUsedVariables(usedVariables);
+        op.getCondition().getValue().getUsedVariables(usedVariables);
         return null;
     }
 
     @Override
     public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getOperator(), usedVariables);
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
             }
         }
         return null;
@@ -233,49 +234,49 @@
 
     @Override
     public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
-        op.getExpressionRef().getExpression().getUsedVariables(usedVariables);
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
         return null;
     }
 
     @Override
     public Void visitUnnestOperator(UnnestOperator op, Void arg) {
-        op.getExpressionRef().getExpression().getUsedVariables(usedVariables);
+        op.getExpressionRef().getValue().getUsedVariables(usedVariables);
         return null;
     }
 
     @Override
     public Void visitWriteOperator(WriteOperator op, Void arg) {
-        for (LogicalExpressionReference expr : op.getExpressions()) {
-            expr.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
+            expr.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
 
     @Override
     public Void visitWriteResultOperator(WriteResultOperator op, Void arg) {
-        op.getPayloadExpression().getExpression().getUsedVariables(usedVariables);
-        for (LogicalExpressionReference e : op.getKeyExpressions()) {
-            e.getExpression().getUsedVariables(usedVariables);
+        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
 
     @Override
     public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) {
-        op.getPayloadExpression().getExpression().getUsedVariables(usedVariables);
-        for (LogicalExpressionReference e : op.getPrimaryKeyExpressions()) {
-            e.getExpression().getUsedVariables(usedVariables);
+        op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
 
     @Override
     public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) {
-        for (LogicalExpressionReference e : op.getPrimaryKeyExpressions()) {
-            e.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
         }
-        for (LogicalExpressionReference e : op.getSecondaryKeyExpressions()) {
-            e.getExpression().getUsedVariables(usedVariables);
+        for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+            e.getValue().getUsedVariables(usedVariables);
         }
         return null;
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
index a918bed..8e9c5b8 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
@@ -19,8 +19,9 @@
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
@@ -51,8 +52,8 @@
             throws AlgebricksException {
         // DFS traversal
         VariableUtilities.getUsedVariables(op, vars);
-        for (LogicalOperatorReference c : op.getInputs()) {
-            getUsedVariablesInDescendantsAndSelf(c.getOperator(), vars);
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            getUsedVariablesInDescendantsAndSelf(c.getValue(), vars);
         }
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
index 766237c..761c7f2 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
@@ -15,8 +15,8 @@
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
 
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -31,7 +31,7 @@
         Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op,
                 opSchema, context);
         builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second);
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
index 61422af..b9352f3 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
@@ -23,8 +23,8 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
@@ -64,9 +64,9 @@
         AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
 
         if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
-            AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+            AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
             IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
-            AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getOperator();
+            AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
             IPhysicalPropertiesVector pv1 = op1.getPhysicalOperator().getDeliveredProperties();
 
             if (pv0 == null || pv1 == null) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
index 038793e..620c161 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
@@ -21,9 +21,9 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
@@ -105,7 +105,7 @@
             throw new NotImplementedException("Nested plans with several roots are not supported.");
         }
         JobSpecification nestedJob = pc.compilePlan(p, outerPlanSchema);
-        ILogicalOperator topOpInSubplan = p.getRoots().get(0).getOperator();
+        ILogicalOperator topOpInSubplan = p.getRoots().get(0).getValue();
         JobGenContext context = pc.getContext();
         IOperatorSchema topOpInSubplanScm = context.getSchema(topOpInSubplan);
         opSchema.addAllVariables(topOpInSubplanScm);
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
index 1aee87b..84982aa 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
@@ -7,23 +7,24 @@
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
@@ -34,7 +35,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
 import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
 
 public abstract class AbstractPreclusteredGroupByPOperator extends AbstractPhysicalOperator {
@@ -64,7 +64,7 @@
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
         List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
         GroupByOperator gby = (GroupByOperator) op;
-        ILogicalOperator op2 = gby.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = gby.getInputs().get(0).getValue();
         IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
         IPartitioningProperty pp = childProp.getPartitioningProperty();
         List<ILocalStructuralProperty> childLocals = childProp.getLocalProperties();
@@ -126,10 +126,10 @@
         for (ILogicalPlan p : gby.getNestedPlans()) {
             // try to propagate secondary order requirements from nested
             // groupings
-            for (LogicalOperatorReference r : p.getRoots()) {
-                AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getOperator();
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
                 if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
-                    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getOperator();
+                    AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
                     IPhysicalOperator pop2 = op2.getPhysicalOperator();
                     if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
                         List<LogicalVariable> sndOrder = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
@@ -160,7 +160,7 @@
                     }
                     LocalOrderProperty lop = (LocalOrderProperty) prop;
                     LogicalVariable ord = lop.getColumn();
-                    Pair<LogicalVariable, LogicalExpressionReference> p = getGbyPairByRhsVar(gby, ord);
+                    Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord);
                     if (p == null) {
                         p = getDecorPairByRhsVar(gby, ord);
                         if (p == null) {
@@ -168,7 +168,7 @@
                             break;
                         }
                     }
-                    ILogicalExpression e = p.second.getExpression();
+                    ILogicalExpression e = p.second.getValue();
                     if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                         throw new IllegalStateException(
                                 "Right hand side of group-by assignment should have been normalized to a variable reference.");
@@ -177,10 +177,10 @@
                     props.add(new LocalOrderProperty(new OrderColumn(v, lop.getOrder())));
                 }
                 List<FunctionalDependency> fdList = new ArrayList<FunctionalDependency>();
-                for (Pair<LogicalVariable, LogicalExpressionReference> decorPair : gby.getDecorList()) {
+                for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
                     List<LogicalVariable> hd = gby.getGbyVarList();
                     List<LogicalVariable> tl = new ArrayList<LogicalVariable>(1);
-                    tl.add(((VariableReferenceExpression) decorPair.second.getExpression()).getVariableReference());
+                    tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
                     fdList.add(new FunctionalDependency(hd, tl));
                 }
                 if (allOk
@@ -200,9 +200,9 @@
         return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
     }
 
-    private static Pair<LogicalVariable, LogicalExpressionReference> getGbyPairByRhsVar(GroupByOperator gby,
+    private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getGbyPairByRhsVar(GroupByOperator gby,
             LogicalVariable var) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> ve : gby.getGroupByList()) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
             if (ve.first == var) {
                 return ve;
             }
@@ -210,9 +210,9 @@
         return null;
     }
 
-    private static Pair<LogicalVariable, LogicalExpressionReference> getDecorPairByRhsVar(GroupByOperator gby,
+    private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getDecorPairByRhsVar(GroupByOperator gby,
             LogicalVariable var) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> ve : gby.getDecorList()) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getDecorList()) {
             if (ve.first == var) {
                 return ve;
             }
@@ -221,8 +221,8 @@
     }
 
     private static LogicalVariable getLhsGbyVar(GroupByOperator gby, LogicalVariable var) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> ve : gby.getGroupByList()) {
-            ILogicalExpression e = ve.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
+            ILogicalExpression e = ve.second.getValue();
             if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 throw new IllegalStateException(
                         "Right hand side of group by assignment should have been normalized to a variable reference.");
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
index 2974753..224d94c 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
@@ -27,7 +27,7 @@
 public abstract class AbstractPropagatePropertiesForUsedVariablesPOperator extends AbstractPhysicalOperator {
 
     public void computeDeliveredPropertiesForUsedVariables(ILogicalOperator op, List<LogicalVariable> usedVariables) {
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
         List<ILocalStructuralProperty> downPropsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
         List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
index 227ed73..ca09b77 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
@@ -17,10 +17,11 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
@@ -57,7 +58,7 @@
         // invalidate cache
         computeLocalProperties(op);
         // }
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         StructuralPropertiesVector childProp = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties();
         deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), orderProps);
     }
@@ -82,8 +83,8 @@
         orderProps = new LinkedList<ILocalStructuralProperty>();
 
         OrderOperator ord = (OrderOperator) op;
-        for (Pair<IOrder, LogicalExpressionReference> p : ord.getOrderExpressions()) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<IOrder, Mutable<ILogicalExpression>> p : ord.getOrderExpressions()) {
+            ILogicalExpression expr = p.second.getValue();
             if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                 LogicalVariable var = varRef.getVariableReference();
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
index c7215fe..47038ad 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
@@ -17,10 +17,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
@@ -51,7 +53,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         IPhysicalPropertiesVector childProps = op2.getDeliveredPhysicalProperties();
         deliveredProperties = new StructuralPropertiesVector(childProps.getPartitioningProperty(),
                 new ArrayList<ILocalStructuralProperty>(0));
@@ -69,7 +71,7 @@
             throws AlgebricksException {
         AggregateOperator aggOp = (AggregateOperator) op;
         List<LogicalVariable> variables = aggOp.getVariables();
-        List<LogicalExpressionReference> expressions = aggOp.getExpressions();
+        List<Mutable<ILogicalExpression>> expressions = aggOp.getExpressions();
         int[] outColumns = new int[variables.size()];
         for (int i = 0; i < outColumns.length; i++) {
             outColumns[i] = opSchema.findVariable(variables.get(i));
@@ -78,9 +80,9 @@
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
         for (int i = 0; i < aggFactories.length; i++) {
             AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expressions.get(i)
-                    .getExpression();
-            aggFactories[i] = exprJobGen.createAggregateFunctionFactory(aggFun, context.getTypeEnvironment(op
-                    .getInputs().get(0).getOperator()), inputSchemas, context);
+                    .getValue();
+            aggFactories[i] = exprJobGen.createAggregateFunctionFactory(aggFun,
+                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
         }
 
         AggregateRuntimeFactory runtime = new AggregateRuntimeFactory(aggFactories);
@@ -89,7 +91,7 @@
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
         builder.contributeMicroOperator(aggOp, runtime, recDesc);
         // and contribute one edge from its child
-        ILogicalOperator src = aggOp.getInputs().get(0).getOperator();
+        ILogicalOperator src = aggOp.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, aggOp, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
index 276ada9..861765d 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
@@ -16,10 +16,12 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
@@ -43,7 +45,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -59,7 +61,7 @@
             throws AlgebricksException {
         AssignOperator assign = (AssignOperator) op;
         List<LogicalVariable> variables = assign.getVariables();
-        List<LogicalExpressionReference> expressions = assign.getExpressions();
+        List<Mutable<ILogicalExpression>> expressions = assign.getExpressions();
         int[] outColumns = new int[variables.size()];
         for (int i = 0; i < outColumns.length; i++) {
             outColumns[i] = opSchema.findVariable(variables.get(i));
@@ -67,8 +69,8 @@
         IEvaluatorFactory[] evalFactories = new IEvaluatorFactory[expressions.size()];
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
         for (int i = 0; i < evalFactories.length; i++) {
-            evalFactories[i] = exprJobGen.createEvaluatorFactory(expressions.get(i).getExpression(), context
-                    .getTypeEnvironment(op.getInputs().get(0).getOperator()), inputSchemas, context);
+            evalFactories[i] = exprJobGen.createEvaluatorFactory(expressions.get(i).getValue(),
+                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
         }
 
         // TODO push projections into the operator
@@ -80,7 +82,7 @@
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
         builder.contributeMicroOperator(assign, runtime, recDesc);
         // and contribute one edge from its child
-        ILogicalOperator src = assign.getInputs().get(0).getOperator();
+        ILogicalOperator src = assign.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, assign, 0);
 
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
index 65b6452..0c780d5 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
@@ -14,10 +14,10 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
 
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
@@ -48,7 +48,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         IPartitioningProperty pp = new BroadcastPartitioningProperty(domain);
         this.deliveredProperties = new StructuralPropertiesVector(pp, op2.getDeliveredPhysicalProperties()
                 .getLocalProperties());
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
index 7bf4e49..68a6a24 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -76,7 +76,7 @@
             builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
         }
 
-        ILogicalOperator srcExchange = scan.getInputs().get(0).getOperator();
+        ILogicalOperator srcExchange = scan.getInputs().get(0).getValue();
         builder.contributeGraphEdge(srcExchange, 0, scan, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
index a983ce7..c0c369e 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
@@ -5,14 +5,14 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
@@ -21,10 +21,10 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
@@ -56,17 +56,17 @@
     private int frameLimit = 0;
     private List<LogicalVariable> columnSet = new ArrayList<LogicalVariable>();
 
-    public ExternalGroupByPOperator(List<Pair<LogicalVariable, LogicalExpressionReference>> gbyList, int frameLimit,
+    public ExternalGroupByPOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList, int frameLimit,
             int tableSize) {
         this.tableSize = tableSize;
         this.frameLimit = frameLimit;
         computeColumnSet(gbyList);
     }
 
-    public void computeColumnSet(List<Pair<LogicalVariable, LogicalExpressionReference>> gbyList) {
+    public void computeColumnSet(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList) {
         columnSet.clear();
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gbyList) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
+            ILogicalExpression expr = p.second.getValue();
             if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression v = (VariableReferenceExpression) expr;
                 columnSet.add(v.getVariableReference());
@@ -104,13 +104,13 @@
             propsLocal.add(new LocalGroupingProperty(columnSet));
         }
         for (ILogicalPlan p : gOp.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                ILogicalOperator rOp = r.getOperator();
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                ILogicalOperator rOp = r.getValue();
                 propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
             }
         }
 
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
         deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
     }
@@ -139,8 +139,8 @@
         int numFds = gby.getDecorList().size();
         int fdColumns[] = new int[numFds];
         int j = 0;
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gby.getDecorList()) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
             if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 throw new AlgebricksException("pre-sorted group-by expects variable references.");
             }
@@ -160,8 +160,8 @@
                     "External group-by currently works only for one nested plan with one root containing"
                             + "an aggregate and a nested-tuple-source.");
         }
-        LogicalOperatorReference r0 = p0.getRoots().get(0);
-        AggregateOperator aggOp = (AggregateOperator) r0.getOperator();
+        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
 
         IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
         List<Object> intermediateTypes = new ArrayList<Object>();
@@ -169,10 +169,10 @@
         ISerializableAggregateFunctionFactory[] aff = new ISerializableAggregateFunctionFactory[n];
         int i = 0;
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
-        IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getOperator());
+        IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
         IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
-        for (LogicalExpressionReference exprRef : aggOp.getExpressions()) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getExpression();
+        for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
             aff[i++] = exprJobGen.createSerializableAggregateFunctionFactory(aggFun, aggOpInputEnv, inputSchemas,
                     context);
             intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv,
@@ -188,9 +188,9 @@
         }
 
         List<LogicalVariable> keyAndDecVariables = new ArrayList<LogicalVariable>();
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gby.getGroupByList())
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList())
             keyAndDecVariables.add(p.first);
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gby.getDecorList())
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList())
             keyAndDecVariables.add(GroupByOperator.getDecorVariable(p));
 
         for (LogicalVariable var : keyAndDecVariables)
@@ -210,7 +210,7 @@
         localInputSchemas[0] = new OperatorSchemaImpl();
         for (i = 0; i < n; i++) {
             AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
-                    .get(i).getExpression();
+                    .get(i).getValue();
             aggFun.getUsedVariables(usedVars);
         }
         i = 0;
@@ -223,7 +223,7 @@
             localInputSchemas[0].addVariable(usedVar);
         for (i = 0; i < n; i++) {
             AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
-                    .get(i).getExpression();
+                    .get(i).getValue();
             merges[i] = exprJobGen.createSerializableAggregateFunctionFactory(mergeFun, aggOpInputEnv,
                     localInputSchemas, context);
         }
@@ -238,7 +238,7 @@
                 recordDescriptor, new HashSpillableGroupingTableFactory(tpcf, tableSize), false);
 
         contributeOpDesc(builder, gby, gbyOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
index 0a44b0a..a54f116 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
@@ -19,11 +19,11 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -31,6 +31,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
@@ -40,7 +41,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
@@ -78,7 +78,7 @@
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
         IPartitioningProperty p = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(partitionFields),
                 domain);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
         List<ILocalStructuralProperty> locals = new ArrayList<ILocalStructuralProperty>();
         for (ILocalStructuralProperty prop : op2Locals) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
index 378852b..2d3724a 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
@@ -24,9 +24,9 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
@@ -128,9 +128,9 @@
         }
         contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
 
-        ILogicalOperator src1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getOperator();
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
         builder.contributeGraphEdge(src2, 0, op, 1);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
index e1606f0..08406d4 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
@@ -24,9 +24,9 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
@@ -112,15 +112,15 @@
         }
         contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
 
-        ILogicalOperator src1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getOperator();
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
         builder.contributeGraphEdge(src2, 0, op, 1);
     }
 
     @Override
     protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
         List<ILocalStructuralProperty> lp0 = pv0.getLocalProperties();
         if (lp0 != null) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
index 47b5cc4..8b8138f 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
@@ -76,7 +76,7 @@
 
         IPushRuntimeFactory runtime = new InMemorySortRuntimeFactory(sortFields, nkcf, comps, null);
         builder.contributeMicroOperator(op, runtime, recDescriptor);
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
index 6e22ac3..276122f 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
@@ -53,7 +53,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -83,8 +83,8 @@
         IMetadataProvider mp = context.getMetadataProvider();
 
         JobSpecification spec = builder.getJobSpec();
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getOperator(),
-                inputSchemas[0], context);
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+                context);
 
         Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
         if (insertDeleteOp.getOperation() == Kind.INSERT)
@@ -96,7 +96,7 @@
 
         builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
         builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getOperator();
+        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
index 44b54c8..2863aa6 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
@@ -47,7 +47,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -71,8 +71,8 @@
         InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op;
         IMetadataProvider mp = context.getMetadataProvider();
         JobSpecification spec = builder.getJobSpec();
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getOperator(),
-                inputSchemas[0], context);
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+                context);
 
         Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
         if (insertDeleteOp.getOperation() == Kind.INSERT)
@@ -84,7 +84,7 @@
 
         builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
         builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getOperator();
+        ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
index 8d424d1..bf59352 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
@@ -2,10 +2,11 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -50,8 +51,8 @@
         int fdColumns[] = new int[numFds];
         IVariableTypeEnvironment env = context.getTypeEnvironment(op);
         int j = 0;
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gby.getDecorList()) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
             if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 throw new AlgebricksException("pre-sorted group-by expects variable references.");
             }
@@ -67,12 +68,12 @@
         IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
                 columnList, env, context);
         RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
-        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getOperator(),
+        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(),
                 inputSchemas[0], context);
         MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keys,
                 comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null);
         builder.contributeMicroOperator(gby, runtime, recordDescriptor);
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
index 6787158..5978b10 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
@@ -24,9 +24,9 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
@@ -89,7 +89,7 @@
         AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
 
         if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
-            AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(1).getOperator();
+            AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
             IPhysicalPropertiesVector pv1 = op2.getPhysicalOperator().getDeliveredProperties();
             if (pv1 == null) {
                 pp = null;
@@ -126,10 +126,10 @@
         IOperatorSchema[] conditionInputSchemas = new IOperatorSchema[1];
         conditionInputSchemas[0] = propagatedSchema;
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
-        IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(join.getCondition().getExpression(), context
-                .getTypeEnvironment(op), conditionInputSchemas, context);
-        ITuplePairComparatorFactory comparatorFactory = new TuplePairEvaluatorFactory(cond, context
-                .getBinaryBooleanInspector());
+        IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(join.getCondition().getValue(),
+                context.getTypeEnvironment(op), conditionInputSchemas, context);
+        ITuplePairComparatorFactory comparatorFactory = new TuplePairEvaluatorFactory(cond,
+                context.getBinaryBooleanInspector());
         JobSpecification spec = builder.getJobSpec();
         IOperatorDescriptor opDesc = null;
 
@@ -145,9 +145,9 @@
         }
         contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
 
-        ILogicalOperator src1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getOperator();
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
         builder.contributeGraphEdge(src2, 0, op, 1);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
index 321ecf4..5a4a204 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
@@ -14,10 +14,11 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
@@ -48,9 +49,8 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        LogicalOperatorReference dataSource = ((NestedTupleSourceOperator) op).getDataSourceReference();
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) dataSource.getOperator().getInputs().get(0)
-                .getOperator();
+        Mutable<ILogicalOperator> dataSource = ((NestedTupleSourceOperator) op).getDataSourceReference();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) dataSource.getValue().getInputs().get(0).getValue();
         IPhysicalPropertiesVector inheritedProps = op2.getDeliveredPhysicalProperties();
         deliveredProperties = (StructuralPropertiesVector) inheritedProps.clone();
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
index b7b4668..0ed17d8 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
@@ -14,10 +14,10 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
 
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
@@ -41,7 +41,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
index 5fba08d..c97e57f 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
@@ -24,8 +24,8 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
@@ -66,7 +66,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
         List<ILocalStructuralProperty> propsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
         deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
@@ -117,7 +117,7 @@
 
         contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
 
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
index a539315..780e4b4 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
@@ -16,10 +16,11 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -64,8 +65,8 @@
         int numFds = gby.getDecorList().size();
         int fdColumns[] = new int[numFds];
         int j = 0;
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gby.getDecorList()) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+            ILogicalExpression expr = p.second.getValue();
             if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 throw new AlgebricksException("pre-sorted group-by expects variable references.");
             }
@@ -88,7 +89,7 @@
 
         contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
 
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
index ca7675c..b42dd5b 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
@@ -51,7 +51,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -67,7 +67,7 @@
 
         SplitOperatorDescriptor splitOpDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity);
         contributeOpDesc(builder, (AbstractLogicalOperator) op, splitOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
index 87ccc05..3647deb 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
@@ -16,10 +16,12 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
@@ -53,7 +55,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -62,8 +64,8 @@
             IPhysicalPropertiesVector reqdByParent) {
         IPartitioningProperty pp = null;
         RunningAggregateOperator ragg = (RunningAggregateOperator) op;
-        for (LogicalExpressionReference exprRef : ragg.getExpressions()) {
-            StatefulFunctionCallExpression f = (StatefulFunctionCallExpression) exprRef.getExpression();
+        for (Mutable<ILogicalExpression> exprRef : ragg.getExpressions()) {
+            StatefulFunctionCallExpression f = (StatefulFunctionCallExpression) exprRef.getValue();
             IPartitioningProperty p = f.getRequiredPartitioningProperty();
             if (p != null) {
                 if (pp == null) {
@@ -84,7 +86,7 @@
             throws AlgebricksException {
         RunningAggregateOperator ragg = (RunningAggregateOperator) op;
         List<LogicalVariable> variables = ragg.getVariables();
-        List<LogicalExpressionReference> expressions = ragg.getExpressions();
+        List<Mutable<ILogicalExpression>> expressions = ragg.getExpressions();
         int[] outColumns = new int[variables.size()];
         for (int i = 0; i < outColumns.length; i++) {
             outColumns[i] = opSchema.findVariable(variables.get(i));
@@ -92,9 +94,9 @@
         IRunningAggregateFunctionFactory[] runningAggFuns = new IRunningAggregateFunctionFactory[expressions.size()];
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
         for (int i = 0; i < runningAggFuns.length; i++) {
-            StatefulFunctionCallExpression expr = (StatefulFunctionCallExpression) expressions.get(i).getExpression();
-            runningAggFuns[i] = exprJobGen.createRunningAggregateFunctionFactory(expr, context.getTypeEnvironment(op
-                    .getInputs().get(0).getOperator()), inputSchemas, context);
+            StatefulFunctionCallExpression expr = (StatefulFunctionCallExpression) expressions.get(i).getValue();
+            runningAggFuns[i] = exprJobGen.createRunningAggregateFunctionFactory(expr,
+                    context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
         }
 
         // TODO push projections into the operator
@@ -107,7 +109,7 @@
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
         builder.contributeMicroOperator(ragg, runtime, recDesc);
         // and contribute one edge from its child
-        ILogicalOperator src = ragg.getInputs().get(0).getOperator();
+        ILogicalOperator src = ragg.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, ragg, 0);
 
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
index fe501e2..0375246 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
@@ -43,7 +43,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -60,7 +60,7 @@
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
         SinkRuntimeFactory runtime = new SinkRuntimeFactory();
         builder.contributeMicroOperator(op, runtime, recDesc);
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
index ec664bb..123b0e1 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
@@ -14,11 +14,12 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -56,7 +57,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -77,8 +78,8 @@
         WriteOperator write = (WriteOperator) op;
         int[] columns = new int[write.getExpressions().size()];
         int i = 0;
-        for (LogicalExpressionReference exprRef : write.getExpressions()) {
-            ILogicalExpression expr = exprRef.getExpression();
+        for (Mutable<ILogicalExpression> exprRef : write.getExpressions()) {
+            ILogicalExpression expr = exprRef.getValue();
             if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 throw new NotImplementedException("Only writing variable expressions is supported.");
             }
@@ -87,8 +88,8 @@
             columns[i++] = inputSchemas[0].findVariable(v);
         }
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
-        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getOperator(),
-                inputSchemas[0], context);
+        RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+                context);
 
         IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op),
                 context, columns);
@@ -99,7 +100,7 @@
                 columns, pf, inputDesc);
 
         builder.contributeMicroOperator(write, runtime.first, recDesc, runtime.second);
-        ILogicalOperator src = write.getInputs().get(0).getOperator();
+        ILogicalOperator src = write.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, write, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
index 185a671..98c9be3 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
@@ -17,16 +17,17 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
@@ -34,7 +35,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
@@ -79,7 +79,7 @@
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
             throws AlgebricksException {
-        AbstractLogicalOperator inp1 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator inp1 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         IPhysicalPropertiesVector pv1 = inp1.getDeliveredPhysicalProperties();
         if (pv1 == null) {
             inp1.computeDeliveredPhysicalProperties(context);
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
index 623bd90..73c7dd6 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
@@ -92,7 +92,7 @@
         ExternalSortOperatorDescriptor sortOpDesc = new ExternalSortOperatorDescriptor(spec, maxNumberOfFrames,
                 sortFields, nkcf, comps, recDescriptor);
         contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
index ee2cee3..82ea31c 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
@@ -50,7 +50,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -69,13 +69,13 @@
         DieOperator die = (DieOperator) op;
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
         IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        IEvaluatorFactory afterObjectsFact = exprJobGen.createEvaluatorFactory(die.getAfterObjects().getExpression(),
+        IEvaluatorFactory afterObjectsFact = exprJobGen.createEvaluatorFactory(die.getAfterObjects().getValue(),
                 env, inputSchemas, context);
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
         StreamDieRuntimeFactory runtime = new StreamDieRuntimeFactory(afterObjectsFact, null,
                 context.getBinaryIntegerInspector());
         builder.contributeMicroOperator(die, runtime, recDesc);
-        ILogicalOperator src = die.getInputs().get(0).getOperator();
+        ILogicalOperator src = die.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, die, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
index a70d6c8..a31b2da 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
@@ -55,7 +55,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -78,17 +78,17 @@
         LimitOperator limit = (LimitOperator) op;
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
         IVariableTypeEnvironment env = context.getTypeEnvironment(op);
-        IEvaluatorFactory maxObjectsFact = exprJobGen.createEvaluatorFactory(limit.getMaxObjects().getExpression(),
+        IEvaluatorFactory maxObjectsFact = exprJobGen.createEvaluatorFactory(limit.getMaxObjects().getValue(),
                 env, inputSchemas, context);
-        ILogicalExpression offsetExpr = limit.getOffset().getExpression();
+        ILogicalExpression offsetExpr = limit.getOffset().getValue();
         IEvaluatorFactory offsetFact = (offsetExpr == null) ? null : exprJobGen.createEvaluatorFactory(offsetExpr, env,
                 inputSchemas, context);
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
-        StreamLimitRuntimeFactory runtime = new StreamLimitRuntimeFactory(maxObjectsFact, offsetFact, null, context
-                .getBinaryIntegerInspector());
+        StreamLimitRuntimeFactory runtime = new StreamLimitRuntimeFactory(maxObjectsFact, offsetFact, null,
+                context.getBinaryIntegerInspector());
         builder.contributeMicroOperator(limit, runtime, recDesc);
         // and contribute one edge from its child
-        ILogicalOperator src = limit.getInputs().get(0).getOperator();
+        ILogicalOperator src = limit.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, limit, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
index e28d580..94e2135 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
@@ -64,7 +64,7 @@
         StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList);
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
         builder.contributeMicroOperator(project, runtime, recDesc);
-        ILogicalOperator src = project.getInputs().get(0).getOperator();
+        ILogicalOperator src = project.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, project, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
index 4544e0e..b4112c8 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
@@ -44,7 +44,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        ILogicalOperator op2 = op.getInputs().get(0).getOperator();
+        ILogicalOperator op2 = op.getInputs().get(0).getValue();
         deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -60,7 +60,7 @@
             throws AlgebricksException {
         SelectOperator select = (SelectOperator) op;
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
-        IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(select.getCondition().getExpression(),
+        IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(select.getCondition().getValue(),
                 context.getTypeEnvironment(op), inputSchemas, context);
         StreamSelectRuntimeFactory runtime = new StreamSelectRuntimeFactory(cond, null,
                 context.getBinaryBooleanInspector());
@@ -68,7 +68,7 @@
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
         builder.contributeMicroOperator(select, runtime, recDesc);
         // and contribute one edge from its child
-        ILogicalOperator src = select.getInputs().get(0).getOperator();
+        ILogicalOperator src = select.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, select, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
index 3ca3f40..dfab5ed 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
@@ -26,8 +26,8 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StringStreamingRuntimeFactory;
 import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
-import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.StringStreamingScriptDescription;
 import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription.ScriptKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.StringStreamingScriptDescription;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
 
@@ -64,7 +64,7 @@
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
         builder.contributeMicroOperator(scriptOp, runtime, recDesc);
         // and contribute one edge from its child
-        ILogicalOperator src = scriptOp.getInputs().get(0).getOperator();
+        ILogicalOperator src = scriptOp.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, scriptOp, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
index 7852cdc..4e6ef69 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
@@ -17,11 +17,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
@@ -53,7 +54,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         IPhysicalPropertiesVector childsProperties = op2.getPhysicalOperator().getDeliveredProperties();
         List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
         if (childsProperties.getLocalProperties() != null) {
@@ -62,8 +63,8 @@
         // ... get local properties for newly created variables...
         SubplanOperator subplan = (SubplanOperator) op;
         for (ILogicalPlan plan : subplan.getNestedPlans()) {
-            for (LogicalOperatorReference r : plan.getRoots()) {
-                AbstractLogicalOperator rOp = (AbstractLogicalOperator) r.getOperator();
+            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+                AbstractLogicalOperator rOp = (AbstractLogicalOperator) r.getValue();
                 propsLocal.addAll(rOp.getPhysicalOperator().getDeliveredProperties().getLocalProperties());
             }
         }
@@ -88,7 +89,7 @@
         AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], subplan, opSchema, context);
         assert (subplans.length == 1);
         AlgebricksPipeline np = subplans[0];
-        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getOperator(),
+        RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(),
                 inputSchemas[0], context);
         INullWriterFactory[] nullWriterFactories = new INullWriterFactory[np.getOutputWidth()];
         for (int i = 0; i < nullWriterFactories.length; i++) {
@@ -99,7 +100,7 @@
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
         builder.contributeMicroOperator(subplan, runtime, recDesc);
 
-        ILogicalOperator src = op.getInputs().get(0).getOperator();
+        ILogicalOperator src = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, op, 0);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
index 1091365..da88dd9 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
@@ -52,7 +52,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
         this.deliveredProperties = new StructuralPropertiesVector(pp, new ArrayList<ILocalStructuralProperty>(0));
 
@@ -92,9 +92,9 @@
         // level, there is no restrictions
         UnionAllOperatorDescriptor opDesc = new UnionAllOperatorDescriptor(spec, 2, recordDescriptor);
         contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
-        ILogicalOperator src1 = op.getInputs().get(0).getOperator();
+        ILogicalOperator src1 = op.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src1, 0, op, 0);
-        ILogicalOperator src2 = op.getInputs().get(1).getOperator();
+        ILogicalOperator src2 = op.getInputs().get(1).getValue();
         builder.contributeGraphEdge(src2, 0, op, 1);
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
index 5353316..8f838ba 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
@@ -21,9 +21,9 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
@@ -48,7 +48,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -62,7 +62,7 @@
                     + unnest.getPositionalVariable());
         }
         int outCol = opSchema.findVariable(unnest.getVariable());
-        ILogicalExpression unnestExpr = unnest.getExpressionRef().getExpression();
+        ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue();
         ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
         boolean exit = false;
         if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
@@ -77,13 +77,13 @@
             throw new AlgebricksException("Unnest expression " + unnestExpr + " is not an unnesting function call.");
         }
         UnnestingFunctionCallExpression agg = (UnnestingFunctionCallExpression) unnestExpr;
-        IUnnestingFunctionFactory unnestingFactory = exprJobGen.createUnnestingFunctionFactory(agg, context
-                .getTypeEnvironment(op.getInputs().get(0).getOperator()), inputSchemas, context);
+        IUnnestingFunctionFactory unnestingFactory = exprJobGen.createUnnestingFunctionFactory(agg,
+                context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
         int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
         UnnestRuntimeFactory unnestRuntime = new UnnestRuntimeFactory(outCol, unnestingFactory, projectionList);
         RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
         builder.contributeMicroOperator(unnest, unnestRuntime, recDesc);
-        ILogicalOperator src = unnest.getInputs().get(0).getOperator();
+        ILogicalOperator src = unnest.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, unnest, 0);
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
index 676d6e6..d1f0340 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
@@ -53,7 +53,7 @@
 
     @Override
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
     }
 
@@ -83,7 +83,7 @@
 
         builder.contributeHyracksOperator(writeResultOp, runtimeAndConstraints.first);
         builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
-        ILogicalOperator src = writeResultOp.getInputs().get(0).getOperator();
+        ILogicalOperator src = writeResultOp.getInputs().get(0).getValue();
         builder.contributeGraphEdge(src, 0, writeResultOp, 0);
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
index 34fc909..103f534 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
@@ -17,34 +17,36 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 
 /*
  * Author: Guangqiang Li
  * Created on Jul 9, 2009 
  */
 public class ALogicalPlanImpl implements ILogicalPlan {
-    private List<LogicalOperatorReference> roots;
+    private List<Mutable<ILogicalOperator>> roots;
 
     public ALogicalPlanImpl() {
-        this.roots = new ArrayList<LogicalOperatorReference>();
+        this.roots = new ArrayList<Mutable<ILogicalOperator>>();
     }
 
-    public ALogicalPlanImpl(List<LogicalOperatorReference> roots) {
+    public ALogicalPlanImpl(List<Mutable<ILogicalOperator>> roots) {
         this.roots = roots;
     }
 
-    public ALogicalPlanImpl(LogicalOperatorReference root) {
-        roots = new ArrayList<LogicalOperatorReference>(1);
+    public ALogicalPlanImpl(Mutable<ILogicalOperator> root) {
+        roots = new ArrayList<Mutable<ILogicalOperator>>(1);
         roots.add(root);
     }
 
-    public List<LogicalOperatorReference> getRoots() {
+    public List<Mutable<ILogicalOperator>> getRoots() {
         return roots;
     }
 
-    public void setRoots(List<LogicalOperatorReference> roots) {
+    public void setRoots(List<Mutable<ILogicalOperator>> roots) {
         this.roots = roots;
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
index 9a01651..05ae1ec 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -16,9 +16,10 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
@@ -32,6 +33,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
@@ -49,7 +51,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
 import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
@@ -104,14 +105,14 @@
     @Override
     public String visitInnerJoinOperator(InnerJoinOperator op, Integer indent) {
         StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("join (").append(op.getCondition().getExpression()).append(")");
+        addIndent(buffer, indent).append("join (").append(op.getCondition().getValue()).append(")");
         return buffer.toString();
     }
 
     @Override
     public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Integer indent) {
         StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("left outer join (").append(op.getCondition().getExpression()).append(")");
+        addIndent(buffer, indent).append("left outer join (").append(op.getCondition().getValue()).append(")");
         return buffer.toString();
     }
 
@@ -126,7 +127,7 @@
     public String visitOrderOperator(OrderOperator op, Integer indent) {
         StringBuilder buffer = new StringBuilder();
         addIndent(buffer, indent).append("order ");
-        for (Pair<OrderOperator.IOrder, LogicalExpressionReference> p : op.getOrderExpressions()) {
+        for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
             String fst;
             switch (p.first.getKind()) {
                 case ASC: {
@@ -141,7 +142,7 @@
                     fst = p.first.getExpressionRef().toString();
                 }
             }
-            buffer.append("(" + fst + ", " + p.second.getExpression() + ") ");
+            buffer.append("(" + fst + ", " + p.second.getValue() + ") ");
         }
         return buffer.toString();
     }
@@ -172,7 +173,7 @@
     @Override
     public String visitSelectOperator(SelectOperator op, Integer indent) {
         StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("select " + "(" + op.getCondition().getExpression() + ")");
+        addIndent(buffer, indent).append("select " + "(" + op.getCondition().getValue() + ")");
         return buffer.toString();
     }
 
@@ -215,7 +216,7 @@
         if (op.getPositionalVariable() != null) {
             buffer.append(" at " + op.getPositionalVariable());
         }
-        buffer.append(" <- " + op.getExpressionRef().getExpression());
+        buffer.append(" <- " + op.getExpressionRef().getValue());
         return buffer.toString();
     }
 
@@ -223,7 +224,7 @@
     public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) {
         StringBuilder buffer = new StringBuilder();
         addIndent(buffer, indent).append(
-                "unnest-map " + op.getVariables() + " <- " + op.getExpressionRef().getExpression());
+                "unnest-map " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
         return buffer.toString();
     }
 
@@ -238,8 +239,8 @@
     @Override
     public String visitLimitOperator(LimitOperator op, Integer indent) {
         StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("limit " + op.getMaxObjects().getExpression());
-        ILogicalExpression offset = op.getOffset().getExpression();
+        addIndent(buffer, indent).append("limit " + op.getMaxObjects().getValue());
+        ILogicalExpression offset = op.getOffset().getValue();
         if (offset != null) {
             buffer.append(", " + offset);
         }
@@ -249,7 +250,7 @@
     @Override
     public String visitDieOperator(DieOperator op, Integer indent) {
         StringBuilder buffer = new StringBuilder();
-        addIndent(buffer, indent).append("die after " + op.getAfterObjects().getExpression());
+        addIndent(buffer, indent).append("die after " + op.getAfterObjects().getValue());
         return buffer.toString();
     }
 
@@ -296,16 +297,16 @@
         return buffer.toString();
     }
 
-    private void pprintExprList(List<LogicalExpressionReference> expressions, StringBuilder buffer) {
+    private void pprintExprList(List<Mutable<ILogicalExpression>> expressions, StringBuilder buffer) {
         buffer.append("[");
         boolean first = true;
-        for (LogicalExpressionReference exprRef : expressions) {
+        for (Mutable<ILogicalExpression> exprRef : expressions) {
             if (first) {
                 first = false;
             } else {
                 buffer.append(", ");
             }
-            buffer.append(exprRef.getExpression());
+            buffer.append(exprRef.getValue());
         }
         buffer.append("]");
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
index 91f28f6..174db0b 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
@@ -14,9 +14,11 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -24,14 +26,14 @@
 public class PlanPrettyPrinter {
     public static void printPlan(ILogicalPlan plan, StringBuilder out, LogicalOperatorPrettyPrintVisitor pvisitor,
             int indent) throws AlgebricksException {
-        for (LogicalOperatorReference root : plan.getRoots()) {
-            printOperator((AbstractLogicalOperator) root.getOperator(), out, pvisitor, indent);
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            printOperator((AbstractLogicalOperator) root.getValue(), out, pvisitor, indent);
         }
     }
 
     public static void printPhysicalOps(ILogicalPlan plan, StringBuilder out, int indent) {
-        for (LogicalOperatorReference root : plan.getRoots()) {
-            printPhysicalOperator((AbstractLogicalOperator) root.getOperator(), indent, out);
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            printPhysicalOperator((AbstractLogicalOperator) root.getValue(), indent, out);
         }
     }
 
@@ -48,8 +50,8 @@
             appendln(out, " -- |" + op.getExecutionMode() + "|");
         }
 
-        for (LogicalOperatorReference i : op.getInputs()) {
-            printOperator((AbstractLogicalOperator) i.getOperator(), out, pvisitor, indent + 2);
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            printOperator((AbstractLogicalOperator) i.getValue(), out, pvisitor, indent + 2);
         }
 
     }
@@ -69,8 +71,8 @@
             }
         }
 
-        for (LogicalOperatorReference i : op.getInputs()) {
-            printPhysicalOperator((AbstractLogicalOperator) i.getOperator(), indent + 2, out);
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            printPhysicalOperator((AbstractLogicalOperator) i.getValue(), indent + 2, out);
         }
 
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java
index ead6c37..ecd3e96 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java
@@ -39,7 +39,7 @@
     }
 
     public int getOutputWidth() {
-        return recordDescriptors[recordDescriptors.length - 1].getFields().length;
+        return recordDescriptors[recordDescriptors.length - 1].getFieldCount();
     }
 
     // public int[] getProjectedColumns() {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java
index 4463b43..33102cb 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java
@@ -21,7 +21,6 @@
 
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
 import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
 import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksCountPartitionConstraint;
@@ -159,9 +158,9 @@
         if (opInputs != null) {
             for (IConnectorDescriptor conn : opInputs) {
                 ConnectorDescriptorId cid = conn.getConnectorId();
-                edu.uci.ics.hyracks.api.util.Pair<edu.uci.ics.hyracks.api.util.Pair<IOperatorDescriptor, Integer>, edu.uci.ics.hyracks.api.util.Pair<IOperatorDescriptor, Integer>> p = jobSpec
+                org.apache.commons.lang3.tuple.Pair<org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec
                         .getConnectorOperatorMap().get(cid);
-                IOperatorDescriptor src = p.first.first;
+                IOperatorDescriptor src = p.getLeft().getLeft();
                 // DFS
                 setPartitionConstraintsDFS(src.getOperatorId(), tgtConstraints, opDesc);
 
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java
index afab295..ab170c9 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java
@@ -20,17 +20,18 @@
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.api.job.JobSpecification;
 
 public class PlanCompiler {
     private JobGenContext context;
-    private Map<LogicalOperatorReference, List<LogicalOperatorReference>> operatorVisitedToParents = new HashMap<LogicalOperatorReference, List<LogicalOperatorReference>>();
+    private Map<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> operatorVisitedToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
 
     public PlanCompiler(JobGenContext context) {
         this.context = context;
@@ -44,9 +45,9 @@
         JobSpecification spec = new JobSpecification();
         List<ILogicalOperator> rootOps = new ArrayList<ILogicalOperator>();
         IHyracksJobBuilder builder = new JobBuilder(spec, context.getClusterLocations());
-        for (LogicalOperatorReference opRef : plan.getRoots()) {
+        for (Mutable<ILogicalOperator> opRef : plan.getRoots()) {
             compileOpRef(opRef, spec, builder, outerPlanSchema);
-            rootOps.add(opRef.getOperator());
+            rootOps.add(opRef.getValue());
         }
         reviseEdges(builder);
         operatorVisitedToParents.clear();
@@ -54,24 +55,24 @@
         return spec;
     }
 
-    private void compileOpRef(LogicalOperatorReference opRef, JobSpecification spec, IHyracksJobBuilder builder,
+    private void compileOpRef(Mutable<ILogicalOperator> opRef, JobSpecification spec, IHyracksJobBuilder builder,
             IOperatorSchema outerPlanSchema) throws AlgebricksException {
-        ILogicalOperator op = opRef.getOperator();
+        ILogicalOperator op = opRef.getValue();
         int n = op.getInputs().size();
         IOperatorSchema[] schemas = new IOperatorSchema[n];
         int i = 0;
-        for (LogicalOperatorReference opRef2 : op.getInputs()) {
-            List<LogicalOperatorReference> parents = operatorVisitedToParents.get(opRef2);
+        for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
+            List<Mutable<ILogicalOperator>> parents = operatorVisitedToParents.get(opRef2);
             if (parents == null) {
-                parents = new ArrayList<LogicalOperatorReference>();
+                parents = new ArrayList<Mutable<ILogicalOperator>>();
                 operatorVisitedToParents.put(opRef2, parents);
                 parents.add(opRef);
                 compileOpRef(opRef2, spec, builder, outerPlanSchema);
-                schemas[i++] = context.getSchema(opRef2.getOperator());
+                schemas[i++] = context.getSchema(opRef2.getValue());
             } else {
                 if (!parents.contains(opRef))
                     parents.add(opRef);
-                schemas[i++] = context.getSchema(opRef2.getOperator());
+                schemas[i++] = context.getSchema(opRef2.getValue());
                 continue;
             }
         }
@@ -86,14 +87,14 @@
         /**
          * revise the edges for the case of replicate operator
          */
-        for (Entry<LogicalOperatorReference, List<LogicalOperatorReference>> entry : operatorVisitedToParents
+        for (Entry<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> entry : operatorVisitedToParents
                 .entrySet()) {
-            LogicalOperatorReference child = entry.getKey();
-            List<LogicalOperatorReference> parents = entry.getValue();
+            Mutable<ILogicalOperator> child = entry.getKey();
+            List<Mutable<ILogicalOperator>> parents = entry.getValue();
             if (parents.size() > 1) {
                 int i = 0;
-                for (LogicalOperatorReference parent : parents) {
-                    builder.contributeGraphEdge(child.getOperator(), i, parent.getOperator(), 0);
+                for (Mutable<ILogicalOperator> parent : parents) {
+                    builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0);
                     i++;
                 }
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
index 7c4aaa1..417b900 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
@@ -1,21 +1,23 @@
 package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 
 public class OpRefTypeEnvPointer implements ITypeEnvPointer {
 
-    private final LogicalOperatorReference op;
+    private final Mutable<ILogicalOperator> op;
     private final ITypingContext ctx;
 
-    public OpRefTypeEnvPointer(LogicalOperatorReference op, ITypingContext ctx) {
+    public OpRefTypeEnvPointer(Mutable<ILogicalOperator> op, ITypingContext ctx) {
         this.op = op;
         this.ctx = ctx;
     }
 
     @Override
     public IVariableTypeEnvironment getTypeEnv() {
-        return ctx.getOutputTypeEnvironment(op.getOperator());
+        return ctx.getOutputTypeEnvironment(op.getValue());
     }
 
     @Override
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
index 26109c6..15504cb 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
@@ -3,8 +3,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
@@ -40,8 +41,8 @@
         if (t != null) {
             return t;
         }
-        for (LogicalOperatorReference r : op.getInputs()) {
-            ILogicalOperator c = r.getOperator();
+        for (Mutable<ILogicalOperator> r : op.getInputs()) {
+            ILogicalOperator c = r.getValue();
             IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(c);
             Object t2 = env.getVarType(var, nonNullVariableList);
             if (t2 != null) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
index d1299ff..4604198 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -16,10 +16,12 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -35,15 +37,15 @@
 public class OperatorManipulationUtil {
 
     // Transforms all NestedTupleSource operators to EmptyTupleSource operators
-    public static void ntsToEts(LogicalOperatorReference opRef, IOptimizationContext context)
+    public static void ntsToEts(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
             throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
             EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
             context.computeAndSetTypeEnvironmentForOperator(ets);
-            opRef.setOperator(ets);
+            opRef.setValue(ets);
         } else {
-            for (LogicalOperatorReference i : opRef.getOperator().getInputs()) {
+            for (Mutable<ILogicalOperator> i : opRef.getValue().getInputs()) {
                 ntsToEts(i, context);
             }
         }
@@ -52,20 +54,20 @@
     public static ILogicalOperator eliminateSingleSubplanOverEts(SubplanOperator subplan) {
         if (subplan.getNestedPlans().size() > 1) {
             // not a single subplan
-            List<LogicalOperatorReference> subInpList = subplan.getInputs();
+            List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
             subInpList.clear();
-            subInpList.add(new LogicalOperatorReference(new EmptyTupleSourceOperator()));
+            subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
             return subplan;
         }
         ILogicalPlan plan = subplan.getNestedPlans().get(0);
         if (plan.getRoots().size() > 1) {
             // not a single subplan
-            List<LogicalOperatorReference> subInpList = subplan.getInputs();
+            List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
             subInpList.clear();
-            subInpList.add(new LogicalOperatorReference(new EmptyTupleSourceOperator()));
+            subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
             return subplan;
         }
-        return plan.getRoots().get(0).getOperator();
+        return plan.getRoots().get(0).getValue();
     }
 
     public static boolean setOperatorMode(AbstractLogicalOperator op) {
@@ -75,7 +77,7 @@
                 // ILogicalExpression e = ((UnnestOperator) op).getExpression();
                 // if (AnalysisUtil.isDataSetCall(e)) {
                 op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
-                AbstractLogicalOperator child = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+                AbstractLogicalOperator child = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
                 if (child.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
                     child.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
                 }
@@ -85,8 +87,8 @@
             }
             case NESTEDTUPLESOURCE: {
                 NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
-                AbstractLogicalOperator prevOp = (AbstractLogicalOperator) nts.getDataSourceReference().getOperator()
-                        .getInputs().get(0).getOperator();
+                AbstractLogicalOperator prevOp = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
+                        .getInputs().get(0).getValue();
                 if (prevOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
                     nts.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
                     change = true;
@@ -105,9 +107,9 @@
                     }
                 }
 
-                for (LogicalOperatorReference i : op.getInputs()) {
+                for (Mutable<ILogicalOperator> i : op.getInputs()) {
                     boolean exit = false;
-                    AbstractLogicalOperator inputOp = (AbstractLogicalOperator) i.getOperator();
+                    AbstractLogicalOperator inputOp = (AbstractLogicalOperator) i.getValue();
                     switch (inputOp.getExecutionMode()) {
                         case PARTITIONED: {
                             op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
@@ -134,22 +136,22 @@
     public static void substituteVarRec(AbstractLogicalOperator op, LogicalVariable v1, LogicalVariable v2,
             boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
         VariableUtilities.substituteVariables(op, v1, v2, goThroughNts, ctx);
-        for (LogicalOperatorReference opRef2 : op.getInputs()) {
-            substituteVarRec((AbstractLogicalOperator) opRef2.getOperator(), v1, v2, goThroughNts, ctx);
+        for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
+            substituteVarRec((AbstractLogicalOperator) opRef2.getValue(), v1, v2, goThroughNts, ctx);
         }
         if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE && goThroughNts) {
             NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
             if (nts.getDataSourceReference() != null) {
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) nts.getDataSourceReference().getOperator()
-                        .getInputs().get(0).getOperator();
+                AbstractLogicalOperator op2 = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
+                        .getInputs().get(0).getValue();
                 substituteVarRec(op2, v1, v2, goThroughNts, ctx);
             }
         }
         if (op.hasNestedPlans()) {
             AbstractOperatorWithNestedPlans aonp = (AbstractOperatorWithNestedPlans) op;
             for (ILogicalPlan p : aonp.getNestedPlans()) {
-                for (LogicalOperatorReference ref : p.getRoots()) {
-                    AbstractLogicalOperator aop = (AbstractLogicalOperator) ref.getOperator();
+                for (Mutable<ILogicalOperator> ref : p.getRoots()) {
+                    AbstractLogicalOperator aop = (AbstractLogicalOperator) ref.getValue();
                     substituteVarRec(aop, v1, v2, goThroughNts, ctx);
                 }
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
index 2223991..97bd8ca 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
@@ -18,12 +18,13 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -84,8 +85,8 @@
         if (op.hasNestedPlans()) {
             AbstractOperatorWithNestedPlans s = (AbstractOperatorWithNestedPlans) op;
             for (ILogicalPlan p : s.getNestedPlans()) {
-                for (LogicalOperatorReference r : p.getRoots()) {
-                    getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getOperator(), freeVars);
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
                 }
             }
             s.getUsedVariablesExceptNestedPlans(freeVars);
@@ -93,16 +94,16 @@
             s.getProducedVariablesExceptNestedPlans(produced2);
             freeVars.removeAll(produced);
         }
-        for (LogicalOperatorReference i : op.getInputs()) {
-            getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) i.getOperator(), freeVars);
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) i.getValue(), freeVars);
         }
     }
 
     public static void getFreeVariablesInSubplans(AbstractOperatorWithNestedPlans op, Set<LogicalVariable> freeVars)
             throws AlgebricksException {
         for (ILogicalPlan p : op.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getOperator(), freeVars);
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
             }
         }
     }
@@ -122,14 +123,14 @@
     public static void computeSchemaAndPropertiesRecIfNull(AbstractLogicalOperator op, IOptimizationContext context)
             throws AlgebricksException {
         if (op.getSchema() == null) {
-            for (LogicalOperatorReference i : op.getInputs()) {
-                computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) i.getOperator(), context);
+            for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) i.getValue(), context);
             }
             if (op.hasNestedPlans()) {
                 AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
                 for (ILogicalPlan p : a.getNestedPlans()) {
-                    for (LogicalOperatorReference r : p.getRoots()) {
-                        computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) r.getOperator(), context);
+                    for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                        computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) r.getValue(), context);
                     }
                 }
             }
@@ -140,14 +141,14 @@
 
     public static void computeSchemaRecIfNull(AbstractLogicalOperator op) throws AlgebricksException {
         if (op.getSchema() == null) {
-            for (LogicalOperatorReference i : op.getInputs()) {
-                computeSchemaRecIfNull((AbstractLogicalOperator) i.getOperator());
+            for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                computeSchemaRecIfNull((AbstractLogicalOperator) i.getValue());
             }
             if (op.hasNestedPlans()) {
                 AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
                 for (ILogicalPlan p : a.getNestedPlans()) {
-                    for (LogicalOperatorReference r : p.getRoots()) {
-                        computeSchemaRecIfNull((AbstractLogicalOperator) r.getOperator());
+                    for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                        computeSchemaRecIfNull((AbstractLogicalOperator) r.getValue());
                     }
                 }
             }
@@ -159,11 +160,11 @@
         if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
             return false;
         }
-        AbstractLogicalOperator doubleUnder = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator doubleUnder = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         if (doubleUnder.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
             return false;
         }
-        ILogicalExpression eu = ((SelectOperator) op).getCondition().getExpression();
+        ILogicalExpression eu = ((SelectOperator) op).getCondition().getValue();
         if (eu.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
             return false;
         }
@@ -171,7 +172,7 @@
         if (f1.getFunctionIdentifier() != AlgebricksBuiltinFunctions.NOT) {
             return false;
         }
-        ILogicalExpression a1 = f1.getArguments().get(0).getExpression();
+        ILogicalExpression a1 = f1.getArguments().get(0).getValue();
         if (a1.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
             return false;
         }
@@ -183,14 +184,14 @@
     }
 
     public static void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
-        for (LogicalOperatorReference r : p.getRoots()) {
+        for (Mutable<ILogicalOperator> r : p.getRoots()) {
             typeOpRec(r, context);
         }
     }
 
-    public static void typeOpRec(LogicalOperatorReference r, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getOperator();
-        for (LogicalOperatorReference i : op.getInputs()) {
+    public static void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
             typeOpRec(i, context);
         }
         if (op.hasNestedPlans()) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
index e8f2d06..2ad942a 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
@@ -14,10 +14,12 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
 
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public interface ILogicalExpressionReferenceTransform {
     // returns true if any change is made to the expression
-    public boolean transform(LogicalExpressionReference expression) throws AlgebricksException;
+    public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException;
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
index 25441a9..1293d5b 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
@@ -16,9 +16,11 @@
 
 import java.util.Collection;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
 import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
@@ -45,7 +47,7 @@
      * @param ruleClasses
      * @return true iff one of the rules in the collection fired
      */
-    public abstract boolean rewriteWithRuleCollection(LogicalOperatorReference root,
+    public abstract boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
             Collection<IAlgebraicRewriteRule> rules) throws AlgebricksException;
 
     /**
@@ -54,20 +56,20 @@
      * @return true if any rewrite was fired, either on opRef or any operator
      *         under it.
      */
-    protected boolean rewriteOperatorRef(LogicalOperatorReference opRef, IAlgebraicRewriteRule rule)
+    protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule)
             throws AlgebricksException {
         return rewriteOperatorRef(opRef, rule, true, false);
     }
 
-    private void printRuleApplication(IAlgebraicRewriteRule rule, LogicalOperatorReference opRef)
+    private void printRuleApplication(IAlgebraicRewriteRule rule, Mutable<ILogicalOperator> opRef)
             throws AlgebricksException {
         AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Rule " + rule.getClass() + " fired.\n");
         StringBuilder sb = new StringBuilder();
-        PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getOperator(), sb, pvisitor, 0);
+        PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getValue(), sb, pvisitor, 0);
         AlgebricksConfig.ALGEBRICKS_LOGGER.fine(sb.toString());
     }
 
-    protected boolean rewriteOperatorRef(LogicalOperatorReference opRef, IAlgebraicRewriteRule rule,
+    protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule,
             boolean enterNestedPlans, boolean fullDFS) throws AlgebricksException {
 
         if (rule.rewritePre(opRef, context)) {
@@ -75,9 +77,9 @@
             return true;
         }
         boolean rewritten = false;
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
 
-        for (LogicalOperatorReference inp : op.getInputs()) {
+        for (Mutable<ILogicalOperator> inp : op.getInputs()) {
             if (rewriteOperatorRef(inp, rule, enterNestedPlans, fullDFS)) {
                 rewritten = true;
                 if (!fullDFS) {
@@ -89,7 +91,7 @@
         if (op.hasNestedPlans() && enterNestedPlans) {
             AbstractOperatorWithNestedPlans o2 = (AbstractOperatorWithNestedPlans) op;
             for (ILogicalPlan p : o2.getNestedPlans()) {
-                for (LogicalOperatorReference r : p.getRoots()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
                     if (rewriteOperatorRef(r, rule, enterNestedPlans, fullDFS)) {
                         rewritten = true;
                         if (!fullDFS) {
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
index b8b1066..381f70c 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
@@ -2,9 +2,11 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
@@ -72,7 +74,7 @@
     private void runOptimizationSets(ILogicalPlan plan,
             List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> optimSet) throws AlgebricksException {
         for (Pair<AbstractRuleController, List<IAlgebraicRewriteRule>> ruleList : optimSet) {
-            for (LogicalOperatorReference r : plan.getRoots()) {
+            for (Mutable<ILogicalOperator> r : plan.getRoots()) {
                 ruleList.first.setContext(context);
                 ruleList.first.rewriteWithRuleCollection(r, ruleList.second);
             }
@@ -80,14 +82,14 @@
     }
 
     private static void computeSchemaBottomUpForPlan(ILogicalPlan p) throws AlgebricksException {
-        for (LogicalOperatorReference r : p.getRoots()) {
-            computeSchemaBottomUpForOp((AbstractLogicalOperator) r.getOperator());
+        for (Mutable<ILogicalOperator> r : p.getRoots()) {
+            computeSchemaBottomUpForOp((AbstractLogicalOperator) r.getValue());
         }
     }
 
     private static void computeSchemaBottomUpForOp(AbstractLogicalOperator op) throws AlgebricksException {
-        for (LogicalOperatorReference i : op.getInputs()) {
-            computeSchemaBottomUpForOp((AbstractLogicalOperator) i.getOperator());
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            computeSchemaBottomUpForOp((AbstractLogicalOperator) i.getValue());
         }
         if (op.hasNestedPlans()) {
             AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
index c9aeaaa..bfbd636 100644
--- a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
@@ -14,12 +14,15 @@
  */
 package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 
 public interface IAlgebraicRewriteRule {
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException;
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException;
 
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException;
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException;
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
index cc5f27b..07f6f5d 100644
--- a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
@@ -9,6 +9,9 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder;
 import edu.uci.ics.hyracks.algebricks.compiler.api.ICompiler;
 import edu.uci.ics.hyracks.algebricks.compiler.api.ICompilerFactory;
@@ -17,8 +20,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -158,21 +159,21 @@
 
     private ILogicalPlan translate(List<ASTNode> ast) throws PigletException {
         Map<String, Relation> symMap = new HashMap<String, Relation>();
-        List<LogicalOperatorReference> roots = new ArrayList<LogicalOperatorReference>();
+        List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
         previousOp = null;
         for (ASTNode an : ast) {
             switch (an.getTag()) {
                 case DUMP: {
                     DumpNode dn = (DumpNode) an;
                     Relation input = symMap.get(dn.getAlias());
-                    List<LogicalExpressionReference> expressions = new ArrayList<LogicalExpressionReference>();
+                    List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
                     for (LogicalVariable v : input.schema.values()) {
-                        expressions.add(new LogicalExpressionReference(new VariableReferenceExpression(v)));
+                        expressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v)));
                     }
                     PigletFileDataSink dataSink = new PigletFileDataSink(dn.getFile());
                     ILogicalOperator op = new WriteOperator(expressions, dataSink);
-                    op.getInputs().add(new LogicalOperatorReference(input.op));
-                    roots.add(new LogicalOperatorReference(op));
+                    op.getInputs().add(new MutableObject<ILogicalOperator>(input.op));
+                    roots.add(new MutableObject<ILogicalOperator>(op));
                 }
                     break;
 
@@ -210,7 +211,8 @@
                 PigletFileDataSource ds = new PigletFileDataSource(file, types.toArray());
                 rel.op = new DataSourceScanOperator(variables, ds);
                 rel.op.getInputs().add(
-                        new LogicalOperatorReference(previousOp == null ? new EmptyTupleSourceOperator() : previousOp));
+                        new MutableObject<ILogicalOperator>(previousOp == null ? new EmptyTupleSourceOperator()
+                                : previousOp));
                 return rel;
             }
 
@@ -221,9 +223,9 @@
                 Relation inputRel = findInputRelation(alias, symMap);
                 Pair<Relation, LogicalVariable> tempInput = translateScalarExpression(inputRel, conditionNode);
                 Relation rel = new Relation();
-                rel.op = new SelectOperator(new LogicalExpressionReference(new VariableReferenceExpression(
+                rel.op = new SelectOperator(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
                         tempInput.second)));
-                rel.op.getInputs().add(new LogicalOperatorReference(tempInput.first.op));
+                rel.op.getInputs().add(new MutableObject<ILogicalOperator>(tempInput.first.op));
                 rel.schema.putAll(tempInput.first.schema);
                 return rel;
             }
@@ -251,11 +253,11 @@
                 List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
                 vars.add(var);
 
-                List<LogicalExpressionReference> exprs = new ArrayList<LogicalExpressionReference>();
-                exprs.add(new LogicalExpressionReference(ce));
+                List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
+                exprs.add(new MutableObject<ILogicalExpression>(ce));
 
                 rel.op = new AssignOperator(vars, exprs);
-                rel.op.getInputs().add(new LogicalOperatorReference(inputRel.op));
+                rel.op.getInputs().add(new MutableObject<ILogicalOperator>(inputRel.op));
                 rel.schema.putAll(inputRel.schema);
 
                 return new Pair<Relation, LogicalVariable>(rel, var);
@@ -263,13 +265,13 @@
 
             case SCALAR_FUNCTION: {
                 ScalarFunctionExpressionNode sfen = (ScalarFunctionExpressionNode) expressionNode;
-                List<LogicalExpressionReference> argExprs = new ArrayList<LogicalExpressionReference>();
+                List<Mutable<ILogicalExpression>> argExprs = new ArrayList<Mutable<ILogicalExpression>>();
                 List<ASTNode> arguments = sfen.getArguments();
                 Relation rel = inputRel;
                 for (ASTNode a : arguments) {
                     Pair<Relation, LogicalVariable> argPair = translateScalarExpression(rel, (ExpressionNode) a);
                     rel = argPair.first;
-                    argExprs.add(new LogicalExpressionReference(new VariableReferenceExpression(argPair.second)));
+                    argExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(argPair.second)));
                 }
                 Relation outRel = new Relation();
                 outRel.schema.putAll(rel.schema);
@@ -279,10 +281,10 @@
 
                 IFunctionInfo fInfo = lookupFunction(sfen.getFunctionTag(), sfen.getFunctionName());
 
-                List<LogicalExpressionReference> exprs = new ArrayList<LogicalExpressionReference>();
-                exprs.add(new LogicalExpressionReference(new ScalarFunctionCallExpression(fInfo, argExprs)));
+                List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
+                exprs.add(new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(fInfo, argExprs)));
                 outRel.op = new AssignOperator(vars, exprs);
-                outRel.op.getInputs().add(new LogicalOperatorReference(rel.op));
+                outRel.op.getInputs().add(new MutableObject<ILogicalOperator>(rel.op));
                 return new Pair<Relation, LogicalVariable>(outRel, var);
             }
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
index d06d9a8..63e6f53 100644
--- a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
@@ -4,8 +4,9 @@
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -70,16 +71,16 @@
             case FUNCTION_CALL: {
                 ScalarFunctionCallExpression sfce = (ScalarFunctionCallExpression) expr;
 
-                List<LogicalExpressionReference> argExprs = sfce.getArguments();
+                List<Mutable<ILogicalExpression>> argExprs = sfce.getArguments();
                 IEvaluatorFactory argEvalFactories[] = new IEvaluatorFactory[argExprs.size()];
                 for (int i = 0; i < argEvalFactories.length; ++i) {
-                    LogicalExpressionReference er = argExprs.get(i);
-                    argEvalFactories[i] = createEvaluatorFactory(er.getExpression(), env, inputSchemas, context);
+                    Mutable<ILogicalExpression> er = argExprs.get(i);
+                    argEvalFactories[i] = createEvaluatorFactory(er.getValue(), env, inputSchemas, context);
                 }
                 IEvaluatorFactory funcEvalFactory;
                 try {
-                    funcEvalFactory = PigletFunctionRegistry.createFunctionEvaluatorFactory(sfce
-                            .getFunctionIdentifier(), argEvalFactories);
+                    funcEvalFactory = PigletFunctionRegistry.createFunctionEvaluatorFactory(
+                            sfce.getFunctionIdentifier(), argEvalFactories);
                 } catch (PigletException e) {
                     throw new AlgebricksException(e);
                 }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
index 62c4559..fe33d09 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
@@ -19,11 +19,13 @@
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
@@ -39,7 +41,7 @@
 public abstract class AbstractDecorrelationRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
@@ -52,7 +54,7 @@
         if (op2.getInputs().size() != 1) {
             return false;
         }
-        AbstractLogicalOperator alo = (AbstractLogicalOperator) op2.getInputs().get(0).getOperator();
+        AbstractLogicalOperator alo = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
         if (descOrSelfIsScanOrJoin(alo)) {
             return true;
         }
@@ -76,16 +78,16 @@
     }
 
     protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g,
-            List<Pair<LogicalVariable, LogicalExpressionReference>> outVeList) throws AlgebricksException {
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
         for (LogicalVariable ov : vars) {
             LogicalVariable newVar = context.newVar();
             ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
-            outVeList.add(new Pair<LogicalVariable, LogicalExpressionReference>(ov, new LogicalExpressionReference(
-                    varExpr)));
+            outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
+                    new MutableObject<ILogicalExpression>(varExpr)));
             for (ILogicalPlan p : g.getNestedPlans()) {
-                for (LogicalOperatorReference r : p.getRoots()) {
-                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getOperator(), ov, newVar,
-                            true, context);
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
+                            context);
                 }
             }
             // g.substituteVarInNestedPlans(ov, newVar);
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
index f247492..fbacede 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
@@ -14,11 +14,13 @@
  */
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -26,12 +28,12 @@
 
 public abstract class AbstractExtractExprRule implements IAlgebraicRewriteRule {
 
-    protected LogicalVariable extractExprIntoAssignOpRef(ILogicalExpression gExpr, LogicalOperatorReference opRef2,
+    protected LogicalVariable extractExprIntoAssignOpRef(ILogicalExpression gExpr, Mutable<ILogicalOperator> opRef2,
             IOptimizationContext context) throws AlgebricksException {
         LogicalVariable v = context.newVar();
-        AssignOperator a = new AssignOperator(v, new LogicalExpressionReference(gExpr));
-        a.getInputs().add(new LogicalOperatorReference(opRef2.getOperator()));
-        opRef2.setOperator(a);
+        AssignOperator a = new AssignOperator(v, new MutableObject<ILogicalExpression>(gExpr));
+        a.getInputs().add(new MutableObject<ILogicalOperator>(opRef2.getValue()));
+        opRef2.setValue(a);
         if (gExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
             context.addNotToBeInlinedVar(v);
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
index 7902e2c..55eef21 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
@@ -17,11 +17,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
@@ -30,48 +31,48 @@
 
 public class BreakSelectIntoConjunctsRule implements IAlgebraicRewriteRule {
 
-    private List<LogicalExpressionReference> conjs = new ArrayList<LogicalExpressionReference>();
+    private List<Mutable<ILogicalExpression>> conjs = new ArrayList<Mutable<ILogicalExpression>>();
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
             return false;
         }
         SelectOperator select = (SelectOperator) op;
 
-        ILogicalExpression cond = select.getCondition().getExpression();
+        ILogicalExpression cond = select.getCondition().getValue();
 
         conjs.clear();
         if (!cond.splitIntoConjuncts(conjs)) {
             return false;
         }
 
-        LogicalOperatorReference childOfSelect = select.getInputs().get(0);
+        Mutable<ILogicalOperator> childOfSelect = select.getInputs().get(0);
         boolean fst = true;
         ILogicalOperator botOp = select;
         ILogicalExpression firstExpr = null;
-        for (LogicalExpressionReference eRef : conjs) {
-            ILogicalExpression e = eRef.getExpression();
+        for (Mutable<ILogicalExpression> eRef : conjs) {
+            ILogicalExpression e = eRef.getValue();
             if (fst) {
                 fst = false;
                 firstExpr = e;
             } else {
-                SelectOperator newSelect = new SelectOperator(new LogicalExpressionReference(e));
-                List<LogicalOperatorReference> botInpList = botOp.getInputs();
+                SelectOperator newSelect = new SelectOperator(new MutableObject<ILogicalExpression>(e));
+                List<Mutable<ILogicalOperator>> botInpList = botOp.getInputs();
                 botInpList.clear();
-                botInpList.add(new LogicalOperatorReference(newSelect));
+                botInpList.add(new MutableObject<ILogicalOperator>(newSelect));
                 context.computeAndSetTypeEnvironmentForOperator(botOp);
                 botOp = newSelect;
             }
         }
         botOp.getInputs().add(childOfSelect);
-        select.getCondition().setExpression(firstExpr);
+        select.getCondition().setValue(firstExpr);
         context.computeAndSetTypeEnvironmentForOperator(botOp);
         context.computeAndSetTypeEnvironmentForOperator(select);
 
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
index 5e02124..2ac387d 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
@@ -16,11 +16,13 @@
 
 import java.util.HashSet;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -37,21 +39,22 @@
 public class ComplexJoinInferenceRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator op = opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalOperator op = opRef.getValue();
         if (!(op instanceof AbstractScanOperator)) {
             return false;
         }
 
-        LogicalOperatorReference opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
         if (op2.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
             return false;
         }
         SubplanOperator subplan = (SubplanOperator) op2;
 
-        LogicalOperatorReference opRef3 = subplan.getInputs().get(0);
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getOperator();
+        Mutable<ILogicalOperator> opRef3 = subplan.getInputs().get(0);
+        AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
 
         if (op3.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
                 || op3.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
@@ -73,23 +76,23 @@
         }
 
         ntsToEtsInSubplan(subplan, context);
-        InnerJoinOperator join = new InnerJoinOperator(new LogicalExpressionReference(ConstantExpression.TRUE));
+        InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
         join.getInputs().add(opRef3);
-        opRef2.setOperator(OperatorManipulationUtil.eliminateSingleSubplanOverEts(subplan));
-        join.getInputs().add(new LogicalOperatorReference(op));
-        opRef.setOperator(join);
+        opRef2.setValue(OperatorManipulationUtil.eliminateSingleSubplanOverEts(subplan));
+        join.getInputs().add(new MutableObject<ILogicalOperator>(op));
+        opRef.setValue(join);
         context.computeAndSetTypeEnvironmentForOperator(join);
         return true;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     private static void ntsToEtsInSubplan(SubplanOperator s, IOptimizationContext context) throws AlgebricksException {
         for (ILogicalPlan p : s.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                 OperatorManipulationUtil.ntsToEts(r, context);
             }
         }
@@ -97,8 +100,8 @@
 
     private static boolean subplanHasFreeVariables(SubplanOperator s) throws AlgebricksException {
         for (ILogicalPlan p : s.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
-                if (OperatorPropertiesUtil.hasFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getOperator())) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                if (OperatorPropertiesUtil.hasFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue())) {
                     return true;
                 }
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
index 0812576..67e7cf0 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
@@ -17,8 +17,10 @@
 import java.util.HashSet;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -30,19 +32,19 @@
 public class ConsolidateAssignsRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
             return false;
         }
         AssignOperator assign1 = (AssignOperator) op;
 
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) assign1.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) assign1.getInputs().get(0).getValue();
         if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
             return false;
         }
@@ -60,8 +62,8 @@
         assign1.getVariables().addAll(assign2.getVariables());
         assign1.getExpressions().addAll(assign2.getExpressions());
 
-        LogicalOperatorReference botOpRef = assign2.getInputs().get(0);
-        List<LogicalOperatorReference> asgnInpList = assign1.getInputs();
+        Mutable<ILogicalOperator> botOpRef = assign2.getInputs().get(0);
+        List<Mutable<ILogicalOperator>> asgnInpList = assign1.getInputs();
         asgnInpList.clear();
         asgnInpList.add(botOpRef);
         context.computeAndSetTypeEnvironmentForOperator(assign1);
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
index 6484cff..c2bb13d 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
@@ -16,9 +16,12 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
@@ -31,41 +34,41 @@
 public class ConsolidateSelectsRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
             return false;
         }
         SelectOperator select = (SelectOperator) op;
 
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) select.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) select.getInputs().get(0).getValue();
         if (op2.getOperatorTag() != LogicalOperatorTag.SELECT) {
             return false;
         }
 
-        AbstractFunctionCallExpression conj = new ScalarFunctionCallExpression(AlgebricksBuiltinFunctions
-                .getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
-        conj.getArguments().add(new LogicalExpressionReference(select.getCondition().getExpression()));
+        AbstractFunctionCallExpression conj = new ScalarFunctionCallExpression(
+                AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
+        conj.getArguments().add(new MutableObject<ILogicalExpression>(select.getCondition().getValue()));
         conj.getArguments().add(((SelectOperator) op2).getCondition());
 
-        LogicalOperatorReference botOpRef = select.getInputs().get(0);
+        Mutable<ILogicalOperator> botOpRef = select.getInputs().get(0);
         boolean more = true;
         while (more) {
-            botOpRef = botOpRef.getOperator().getInputs().get(0);
-            AbstractLogicalOperator botOp = (AbstractLogicalOperator) botOpRef.getOperator();
+            botOpRef = botOpRef.getValue().getInputs().get(0);
+            AbstractLogicalOperator botOp = (AbstractLogicalOperator) botOpRef.getValue();
             if (botOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
                 conj.getArguments().add(((SelectOperator) botOp).getCondition());
             } else {
                 more = false;
             }
         }
-        select.getCondition().setExpression(conj);
-        List<LogicalOperatorReference> selInptList = select.getInputs();
+        select.getCondition().setValue(conj);
+        List<Mutable<ILogicalOperator>> selInptList = select.getInputs();
         selInptList.clear();
         selInptList.add(botOpRef);
         context.computeAndSetTypeEnvironmentForOperator(select);
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
index 6a866ae..5c5356c 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
@@ -16,11 +16,13 @@
 
 import java.util.LinkedList;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -34,7 +36,7 @@
 public class EliminateSubplanRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
@@ -46,15 +48,15 @@
      * there are no aggregates?)
      */
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
             return false;
         }
         SubplanOperator subplan = (SubplanOperator) op;
 
-        LogicalOperatorReference outerRef = subplan.getInputs().get(0);
-        AbstractLogicalOperator outerRefOp = (AbstractLogicalOperator) outerRef.getOperator();
+        Mutable<ILogicalOperator> outerRef = subplan.getInputs().get(0);
+        AbstractLogicalOperator outerRefOp = (AbstractLogicalOperator) outerRef.getValue();
         if (outerRefOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
             elimSubplanOverEts(opRef, context);
             return true;
@@ -69,12 +71,12 @@
         return false;
     }
 
-    private boolean elimOneSubplanWithNoFreeVars(LogicalOperatorReference opRef) {
-        SubplanOperator subplan = (SubplanOperator) opRef.getOperator();
+    private boolean elimOneSubplanWithNoFreeVars(Mutable<ILogicalOperator> opRef) {
+        SubplanOperator subplan = (SubplanOperator) opRef.getValue();
         AbstractLogicalOperator rootOp = (AbstractLogicalOperator) subplan.getNestedPlans().get(0).getRoots().get(0)
-                .getOperator();
+                .getValue();
         if (rootOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-            opRef.setOperator(subplan.getInputs().get(0).getOperator());
+            opRef.setValue(subplan.getInputs().get(0).getValue());
             return true;
         } else {
             AbstractLogicalOperator botOp = rootOp;
@@ -82,11 +84,11 @@
                 return false;
             }
             do {
-                LogicalOperatorReference botRef = botOp.getInputs().get(0);
-                botOp = (AbstractLogicalOperator) botRef.getOperator();
+                Mutable<ILogicalOperator> botRef = botOp.getInputs().get(0);
+                botOp = (AbstractLogicalOperator) botRef.getValue();
                 if (botOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-                    botRef.setOperator(subplan.getInputs().get(0).getOperator());
-                    opRef.setOperator(rootOp);
+                    botRef.setValue(subplan.getInputs().get(0).getValue());
+                    opRef.setValue(rootOp);
                     return true;
                 }
             } while (botOp.getInputs().size() == 1);
@@ -94,32 +96,32 @@
         }
     }
 
-    private void elimSubplanOverEts(LogicalOperatorReference opRef, IOptimizationContext ctx)
+    private void elimSubplanOverEts(Mutable<ILogicalOperator> opRef, IOptimizationContext ctx)
             throws AlgebricksException {
-        SubplanOperator subplan = (SubplanOperator) opRef.getOperator();
+        SubplanOperator subplan = (SubplanOperator) opRef.getValue();
         for (ILogicalPlan p : subplan.getNestedPlans()) {
-            for (LogicalOperatorReference r : p.getRoots()) {
+            for (Mutable<ILogicalOperator> r : p.getRoots()) {
                 OperatorManipulationUtil.ntsToEts(r, ctx);
             }
         }
-        LinkedList<LogicalOperatorReference> allRoots = subplan.allRootsInReverseOrder();
+        LinkedList<Mutable<ILogicalOperator>> allRoots = subplan.allRootsInReverseOrder();
         if (allRoots.size() == 1) {
-            opRef.setOperator(allRoots.get(0).getOperator());
+            opRef.setValue(allRoots.get(0).getValue());
         } else {
             ILogicalOperator topOp = null;
-            for (LogicalOperatorReference r : allRoots) {
+            for (Mutable<ILogicalOperator> r : allRoots) {
                 if (topOp == null) {
-                    topOp = r.getOperator();
+                    topOp = r.getValue();
                 } else {
-                    LeftOuterJoinOperator j = new LeftOuterJoinOperator(new LogicalExpressionReference(
+                    LeftOuterJoinOperator j = new LeftOuterJoinOperator(new MutableObject<ILogicalExpression>(
                             ConstantExpression.TRUE));
-                    j.getInputs().add(new LogicalOperatorReference(topOp));
+                    j.getInputs().add(new MutableObject<ILogicalOperator>(topOp));
                     j.getInputs().add(r);
                     ctx.setOutputTypeEnvironment(j, j.computeOutputTypeEnvironment(ctx));
                     topOp = j;
                 }
             }
-            opRef.setOperator(topOp);
+            opRef.setValue(topOp);
         }
     }
 }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
index eaaf799..5ebaad7 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
@@ -6,13 +6,15 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -43,8 +45,10 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.AsterixNodeGroupDomain;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty.PartitioningType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
@@ -56,8 +60,6 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
-import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty.PartitioningType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
@@ -74,13 +76,14 @@
     private PhysicalOptimizationConfig physicalOptimizationConfig;
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         // wait for the physical operators to be set first
         if (op.getPhysicalOperator() == null) {
             return false;
@@ -108,7 +111,7 @@
         AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator() + ": "
                 + op.getDeliveredPhysicalProperties() + "\n");
 
-        context.addToDontApplySet(this, opRef.getOperator());
+        context.addToDontApplySet(this, opRef.getValue());
 
         return changed;
     }
@@ -116,11 +119,11 @@
     private boolean physOptimizePlan(ILogicalPlan plan, IPhysicalPropertiesVector pvector, boolean nestedPlan,
             IOptimizationContext context) throws AlgebricksException {
         boolean changed = false;
-        for (LogicalOperatorReference root : plan.getRoots()) {
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
             if (physOptimizeOp(root, pvector, nestedPlan, context)) {
                 changed = true;
             }
-            AbstractLogicalOperator op = (AbstractLogicalOperator) root.getOperator();
+            AbstractLogicalOperator op = (AbstractLogicalOperator) root.getValue();
             op.computeDeliveredPhysicalProperties(context);
             AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator()
                     + ": " + op.getDeliveredPhysicalProperties() + "\n");
@@ -128,11 +131,11 @@
         return changed;
     }
 
-    private boolean physOptimizeOp(LogicalOperatorReference opRef, IPhysicalPropertiesVector required,
+    private boolean physOptimizeOp(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector required,
             boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
 
         boolean changed = false;
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         optimizeUsingConstraintsAndEquivClasses(op);
         PhysicalRequirements pr = op.getRequiredPhysicalPropertiesForChildren(required);
         IPhysicalPropertiesVector[] reqdProperties = null;
@@ -145,8 +148,8 @@
         INodeDomain childrenDomain = null;
         {
             int j = 0;
-            for (LogicalOperatorReference childRef : op.getInputs()) {
-                AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getOperator();
+            for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+                AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
                 // recursive call
                 if (physOptimizeOp(childRef, reqdProperties[j], nestedPlan, context)) {
                     changed = true;
@@ -177,18 +180,18 @@
 
         IPartitioningProperty firstDeliveredPartitioning = null;
         int i = 0;
-        for (LogicalOperatorReference childRef : op.getInputs()) {
-            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getOperator();
+        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+            AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
             IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
 
             AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Properties delivered by " + child.getPhysicalOperator()
                     + ": " + delivered + "\n");
             IPartitioningRequirementsCoordinator prc = pr.getPartitioningCoordinator();
-            Pair<Boolean, IPartitioningProperty> pbpp = prc.coordinateRequirements(reqdProperties[i]
-                    .getPartitioningProperty(), firstDeliveredPartitioning, op, context);
+            Pair<Boolean, IPartitioningProperty> pbpp = prc.coordinateRequirements(
+                    reqdProperties[i].getPartitioningProperty(), firstDeliveredPartitioning, op, context);
             boolean mayExpandPartitioningProperties = pbpp.first;
-            IPhysicalPropertiesVector rqd = new StructuralPropertiesVector(pbpp.second, reqdProperties[i]
-                    .getLocalProperties());
+            IPhysicalPropertiesVector rqd = new StructuralPropertiesVector(pbpp.second,
+                    reqdProperties[i].getLocalProperties());
 
             AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for " + child.getPhysicalOperator()
                     + ": " + rqd + "\n");
@@ -203,7 +206,7 @@
                 changed = true;
                 addEnforcers(op, i, diff, rqd, delivered, childrenDomain, nestedPlan, context);
 
-                AbstractLogicalOperator newChild = ((AbstractLogicalOperator) op.getInputs().get(i).getOperator());
+                AbstractLogicalOperator newChild = ((AbstractLogicalOperator) op.getInputs().get(i).getValue());
 
                 if (newChild != child) {
                     delivered = newChild.getDeliveredPhysicalProperties();
@@ -245,15 +248,15 @@
                 printOp((AbstractLogicalOperator) op);
             }
             changed = true;
-            AbstractLogicalOperator nextOp = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+            AbstractLogicalOperator nextOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
             if (nextOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
-                nextOp = (AbstractLogicalOperator) nextOp.getInputs().get(0).getOperator();
+                nextOp = (AbstractLogicalOperator) nextOp.getInputs().get(0).getValue();
             }
-            opRef.setOperator(nextOp);
+            opRef.setValue(nextOp);
             // Now, transfer annotations from the original sort op. to this one.
             AbstractLogicalOperator transferTo = nextOp;
             if (transferTo.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
-                transferTo = (AbstractLogicalOperator) transferTo.getInputs().get(0).getOperator();
+                transferTo = (AbstractLogicalOperator) transferTo.getInputs().get(0).getValue();
             }
             transferTo.getAnnotations().putAll(op.getAnnotations());
             physOptimizeOp(opRef, required, nestedPlan, context);
@@ -334,9 +337,9 @@
     /*
      * We assume delivered to be already normalized.
      */
-    private boolean isRedundantSort(LogicalOperatorReference opRef, IPhysicalPropertiesVector delivered,
+    private boolean isRedundantSort(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector delivered,
             IPhysicalPropertiesVector diffOfProperties, IOptimizationContext context) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.ORDER
                 || (op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT && op
                         .getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT)
@@ -346,8 +349,8 @@
         AbstractStableSortPOperator sortOp = (AbstractStableSortPOperator) op.getPhysicalOperator();
         sortOp.computeLocalProperties(op);
         List<ILocalStructuralProperty> orderProps = sortOp.getOrderProperties();
-        return PropertiesUtil.matchLocalProperties(orderProps, delivered.getLocalProperties(), context
-                .getEquivalenceClassMap(op), context.getFDList(op));
+        return PropertiesUtil.matchLocalProperties(orderProps, delivered.getLocalProperties(),
+                context.getEquivalenceClassMap(op), context.getFDList(op));
     }
 
     private void addEnforcers(AbstractLogicalOperator op, int childIndex,
@@ -358,12 +361,12 @@
         IPartitioningProperty pp = diffPropertiesVector.getPartitioningProperty();
         if (pp == null || pp.getPartitioningType() == PartitioningType.UNPARTITIONED) {
             addLocalEnforcers(op, childIndex, diffPropertiesVector.getLocalProperties(), nestedPlan, context);
-            IPhysicalPropertiesVector deliveredByNewChild = ((AbstractLogicalOperator) op.getInputs().get(0)
-                    .getOperator()).getDeliveredPhysicalProperties();
+            IPhysicalPropertiesVector deliveredByNewChild = ((AbstractLogicalOperator) op.getInputs().get(0).getValue())
+                    .getDeliveredPhysicalProperties();
             addPartitioningEnforcers(op, childIndex, pp, required, deliveredByNewChild, domain, context);
         } else {
             addPartitioningEnforcers(op, childIndex, pp, required, deliveredByChild, domain, context);
-            AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getOperator();
+            AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
             IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, required, true, context);
             AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
             if (newDiff != null) {
@@ -383,8 +386,8 @@
             return;
         }
 
-        LogicalOperatorReference topOp = new LogicalOperatorReference();
-        topOp.setOperator(op.getInputs().get(i).getOperator());
+        Mutable<ILogicalOperator> topOp = new MutableObject<ILogicalOperator>();
+        topOp.setValue(op.getInputs().get(i).getValue());
         LinkedList<LocalOrderProperty> oList = new LinkedList<LocalOrderProperty>();
 
         for (ILocalStructuralProperty prop : localProperties) {
@@ -414,17 +417,18 @@
         }
 
         op.getInputs().set(i, topOp);
-        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) topOp.getOperator(), context);
-        printOp((AbstractLogicalOperator) topOp.getOperator());
+        OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) topOp.getValue(), context);
+        printOp((AbstractLogicalOperator) topOp.getValue());
     }
 
-    private LogicalOperatorReference enforceOrderProperties(List<LocalOrderProperty> oList,
-            LogicalOperatorReference topOp, boolean isMicroOp, IOptimizationContext context) throws AlgebricksException {
-        List<Pair<IOrder, LogicalExpressionReference>> oe = new LinkedList<Pair<IOrder, LogicalExpressionReference>>();
+    private Mutable<ILogicalOperator> enforceOrderProperties(List<LocalOrderProperty> oList,
+            Mutable<ILogicalOperator> topOp, boolean isMicroOp, IOptimizationContext context)
+            throws AlgebricksException {
+        List<Pair<IOrder, Mutable<ILogicalExpression>>> oe = new LinkedList<Pair<IOrder, Mutable<ILogicalExpression>>>();
         for (LocalOrderProperty o : oList) {
             IOrder ordType = (o.getOrder() == OrderKind.ASC) ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER;
-            Pair<IOrder, LogicalExpressionReference> pair = new Pair<IOrder, LogicalExpressionReference>(ordType,
-                    new LogicalExpressionReference(new VariableReferenceExpression(o.getColumn())));
+            Pair<IOrder, Mutable<ILogicalExpression>> pair = new Pair<IOrder, Mutable<ILogicalExpression>>(ordType,
+                    new MutableObject<ILogicalExpression>(new VariableReferenceExpression(o.getColumn())));
             oe.add(pair);
         }
         OrderOperator oo = new OrderOperator(oe);
@@ -439,7 +443,7 @@
         if (AlgebricksConfig.DEBUG) {
             AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added sort enforcer " + oo.getPhysicalOperator() + ".\n");
         }
-        return new LogicalOperatorReference(oo);
+        return new MutableObject<ILogicalOperator>(oo);
     }
 
     private void addPartitioningEnforcers(ILogicalOperator op, int i, IPartitioningProperty pp,
@@ -460,14 +464,14 @@
                     break;
                 }
                 case UNORDERED_PARTITIONED: {
-                    List<LogicalVariable> varList = new ArrayList<LogicalVariable>(((UnorderedPartitionedProperty) pp)
-                            .getColumnSet());
+                    List<LogicalVariable> varList = new ArrayList<LogicalVariable>(
+                            ((UnorderedPartitionedProperty) pp).getColumnSet());
                     List<ILocalStructuralProperty> cldLocals = deliveredByChild.getLocalProperties();
                     List<ILocalStructuralProperty> reqdLocals = required.getLocalProperties();
                     boolean propWasSet = false;
                     pop = null;
                     if (reqdLocals != null && cldLocals != null && allAreOrderProps(cldLocals)) {
-                        AbstractLogicalOperator c = (AbstractLogicalOperator) op.getInputs().get(i).getOperator();
+                        AbstractLogicalOperator c = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
                         Map<LogicalVariable, EquivalenceClass> ecs = context.getEquivalenceClassMap(c);
                         List<FunctionalDependency> fds = context.getFDList(c);
                         if (PropertiesUtil.matchLocalProperties(reqdLocals, cldLocals, ecs, fds)) {
@@ -512,7 +516,7 @@
                             + " partitioning type has not been implemented.");
                 }
             }
-            LogicalOperatorReference ci = op.getInputs().get(i);
+            Mutable<ILogicalOperator> ci = op.getInputs().get(i);
             ExchangeOperator exchg = new ExchangeOperator();
             exchg.setPhysicalOperator(pop);
             setNewOp(ci, exchg, context);
@@ -562,11 +566,11 @@
 
     }
 
-    private void setNewOp(LogicalOperatorReference opRef, AbstractLogicalOperator newOp, IOptimizationContext context)
+    private void setNewOp(Mutable<ILogicalOperator> opRef, AbstractLogicalOperator newOp, IOptimizationContext context)
             throws AlgebricksException {
-        ILogicalOperator oldOp = opRef.getOperator();
-        opRef.setOperator(newOp);
-        newOp.getInputs().add(new LogicalOperatorReference(oldOp));
+        ILogicalOperator oldOp = opRef.getValue();
+        opRef.setValue(newOp);
+        newOp.getInputs().add(new MutableObject<ILogicalOperator>(oldOp));
         newOp.recomputeSchema();
         newOp.computeDeliveredPhysicalProperties(context);
         context.computeAndSetTypeEnvironmentForOperator(newOp);
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
index 11f94fe..3501467 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
@@ -20,19 +20,21 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.IsomorphismUtilities;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
@@ -44,25 +46,26 @@
 
 public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule {
 
-    private HashMap<LogicalOperatorReference, List<LogicalOperatorReference>> childrenToParents = new HashMap<LogicalOperatorReference, List<LogicalOperatorReference>>();
-    private List<LogicalOperatorReference> roots = new ArrayList<LogicalOperatorReference>();
-    private List<LogicalOperatorReference> joins = new ArrayList<LogicalOperatorReference>();
-    private List<List<LogicalOperatorReference>> equivalenceClasses = new ArrayList<List<LogicalOperatorReference>>();
+    private HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> childrenToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
+    private List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
+    private List<Mutable<ILogicalOperator>> joins = new ArrayList<Mutable<ILogicalOperator>>();
+    private List<List<Mutable<ILogicalOperator>>> equivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT) {
             return false;
         }
         if (!roots.contains(op))
-            roots.add(new LogicalOperatorReference(op));
+            roots.add(new MutableObject<ILogicalOperator>(op));
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT) {
             return false;
         }
@@ -91,12 +94,12 @@
     }
 
     private void removeTrivialShare() {
-        for (List<LogicalOperatorReference> candidates : equivalenceClasses) {
+        for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
             for (int i = candidates.size() - 1; i >= 0; i--) {
-                LogicalOperatorReference opRef = candidates.get(i);
-                AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getOperator();
+                Mutable<ILogicalOperator> opRef = candidates.get(i);
+                AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue();
                 if (aop.getOperatorTag() == LogicalOperatorTag.EXCHANGE)
-                    aop = (AbstractLogicalOperator) aop.getInputs().get(0).getOperator();
+                    aop = (AbstractLogicalOperator) aop.getInputs().get(0).getValue();
                 if (aop.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE)
                     candidates.remove(i);
             }
@@ -107,11 +110,11 @@
     }
 
     private void removeNonJoinBuildBranchCandidates() {
-        for (List<LogicalOperatorReference> candidates : equivalenceClasses) {
+        for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
             for (int i = candidates.size() - 1; i >= 0; i--) {
-                LogicalOperatorReference opRef = candidates.get(i);
+                Mutable<ILogicalOperator> opRef = candidates.get(i);
                 boolean reserve = false;
-                for (LogicalOperatorReference join : joins)
+                for (Mutable<ILogicalOperator> join : joins)
                     if (isInJoinBuildBranch(join, opRef)) {
                         reserve = true;
                     }
@@ -124,43 +127,43 @@
                 equivalenceClasses.remove(i);
     }
 
-    private boolean isInJoinBuildBranch(LogicalOperatorReference joinRef, LogicalOperatorReference opRef) {
-        LogicalOperatorReference buildBranch = joinRef.getOperator().getInputs().get(1);
+    private boolean isInJoinBuildBranch(Mutable<ILogicalOperator> joinRef, Mutable<ILogicalOperator> opRef) {
+        Mutable<ILogicalOperator> buildBranch = joinRef.getValue().getInputs().get(1);
         do {
             if (buildBranch.equals(opRef)) {
                 return true;
             } else {
-                AbstractLogicalOperator aop = (AbstractLogicalOperator) buildBranch.getOperator();
+                AbstractLogicalOperator aop = (AbstractLogicalOperator) buildBranch.getValue();
                 if (aop.getOperatorTag() == LogicalOperatorTag.INNERJOIN
                         || aop.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN
-                        || buildBranch.getOperator().getInputs().size() == 0)
+                        || buildBranch.getValue().getInputs().size() == 0)
                     return false;
                 else
-                    buildBranch = buildBranch.getOperator().getInputs().get(0);
+                    buildBranch = buildBranch.getValue().getInputs().get(0);
             }
         } while (true);
     }
 
     private boolean rewrite(IOptimizationContext context) throws AlgebricksException {
         boolean changed = false;
-        for (List<LogicalOperatorReference> members : equivalenceClasses) {
+        for (List<Mutable<ILogicalOperator>> members : equivalenceClasses) {
             if (rewriteForOneEquivalentClass(members, context))
                 changed = true;
         }
         return changed;
     }
 
-    private boolean rewriteForOneEquivalentClass(List<LogicalOperatorReference> members, IOptimizationContext context)
+    private boolean rewriteForOneEquivalentClass(List<Mutable<ILogicalOperator>> members, IOptimizationContext context)
             throws AlgebricksException {
-        List<LogicalOperatorReference> group = new ArrayList<LogicalOperatorReference>();
+        List<Mutable<ILogicalOperator>> group = new ArrayList<Mutable<ILogicalOperator>>();
         boolean rewritten = false;
         while (members.size() > 0) {
             group.clear();
-            LogicalOperatorReference candidate = members.remove(members.size() - 1);
+            Mutable<ILogicalOperator> candidate = members.remove(members.size() - 1);
             group.add(candidate);
             for (int i = members.size() - 1; i >= 0; i--) {
-                LogicalOperatorReference peer = members.get(i);
-                if (IsomorphismUtilities.isOperatorIsomorphic(candidate.getOperator(), peer.getOperator())) {
+                Mutable<ILogicalOperator> peer = members.get(i);
+                if (IsomorphismUtilities.isOperatorIsomorphic(candidate.getValue(), peer.getValue())) {
                     group.add(peer);
                     members.remove(i);
                 }
@@ -168,8 +171,8 @@
             AbstractLogicalOperator rop = new ReplicateOperator(group.size());
             rop.setPhysicalOperator(new ReplicatePOperator());
             rop.setExecutionMode(ExecutionMode.PARTITIONED);
-            LogicalOperatorReference ropRef = new LogicalOperatorReference(rop);
-            AbstractLogicalOperator aopCandidate = (AbstractLogicalOperator) candidate.getOperator();
+            Mutable<ILogicalOperator> ropRef = new MutableObject<ILogicalOperator>(rop);
+            AbstractLogicalOperator aopCandidate = (AbstractLogicalOperator) candidate.getValue();
 
             if (aopCandidate.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
                 rop.getInputs().add(candidate);
@@ -178,13 +181,13 @@
                 beforeExchange.setPhysicalOperator(new OneToOneExchangePOperator());
                 beforeExchange.getInputs().add(candidate);
                 context.computeAndSetTypeEnvironmentForOperator(beforeExchange);
-                rop.getInputs().add(new LogicalOperatorReference(beforeExchange));
+                rop.getInputs().add(new MutableObject<ILogicalOperator>(beforeExchange));
             }
             context.computeAndSetTypeEnvironmentForOperator(rop);
 
-            List<LogicalOperatorReference> parents = childrenToParents.get(candidate);
-            for (LogicalOperatorReference parentRef : parents) {
-                AbstractLogicalOperator parent = (AbstractLogicalOperator) parentRef.getOperator();
+            List<Mutable<ILogicalOperator>> parents = childrenToParents.get(candidate);
+            for (Mutable<ILogicalOperator> parentRef : parents) {
+                AbstractLogicalOperator parent = (AbstractLogicalOperator) parentRef.getValue();
                 int index = parent.getInputs().indexOf(candidate);
                 if (parent.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
                     parent.getInputs().set(index, ropRef);
@@ -193,24 +196,23 @@
                     exchange.setPhysicalOperator(new OneToOneExchangePOperator());
                     exchange.getInputs().add(ropRef);
                     context.computeAndSetTypeEnvironmentForOperator(exchange);
-                    // parent.getInputs().get(index).setOperator(exchange);
-                    parent.getInputs().set(index, new LogicalOperatorReference(exchange));
+                    // parent.getInputs().get(index).setValue(exchange);
+                    parent.getInputs().set(index, new MutableObject<ILogicalOperator>(exchange));
                     context.computeAndSetTypeEnvironmentForOperator(parent);
                 }
             }
 
             List<LogicalVariable> liveVarsNew = new ArrayList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(candidate.getOperator(), liveVarsNew);
-            ArrayList<LogicalExpressionReference> assignExprs = new ArrayList<LogicalExpressionReference>();
+            VariableUtilities.getLiveVariables(candidate.getValue(), liveVarsNew);
+            ArrayList<Mutable<ILogicalExpression>> assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
             for (LogicalVariable liveVar : liveVarsNew)
-                assignExprs.add(new LogicalExpressionReference(new VariableReferenceExpression(liveVar)));
-            for (LogicalOperatorReference ref : group) {
+                assignExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(liveVar)));
+            for (Mutable<ILogicalOperator> ref : group) {
                 if (ref.equals(candidate))
                     continue;
                 ArrayList<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
                 Map<LogicalVariable, LogicalVariable> variableMappingBack = new HashMap<LogicalVariable, LogicalVariable>();
-                IsomorphismUtilities.mapVariablesTopDown(ref.getOperator(), candidate.getOperator(),
-                        variableMappingBack);
+                IsomorphismUtilities.mapVariablesTopDown(ref.getValue(), candidate.getValue(), variableMappingBack);
                 for (int i = 0; i < liveVarsNew.size(); i++) {
                     liveVars.add(variableMappingBack.get(liveVarsNew.get(i)));
                 }
@@ -223,20 +225,20 @@
                 exchOp.setPhysicalOperator(new OneToOneExchangePOperator());
                 exchOp.getInputs().add(ropRef);
 
-                assignOperator.getInputs().add(new LogicalOperatorReference(exchOp));
-                projectOperator.getInputs().add(new LogicalOperatorReference(assignOperator));
+                assignOperator.getInputs().add(new MutableObject<ILogicalOperator>(exchOp));
+                projectOperator.getInputs().add(new MutableObject<ILogicalOperator>(assignOperator));
                 // set the types
                 context.computeAndSetTypeEnvironmentForOperator(exchOp);
                 context.computeAndSetTypeEnvironmentForOperator(assignOperator);
                 context.computeAndSetTypeEnvironmentForOperator(projectOperator);
 
-                List<LogicalOperatorReference> parentOpList = childrenToParents.get(ref);
-                for (LogicalOperatorReference parentOpRef : parentOpList) {
-                    AbstractLogicalOperator parentOp = (AbstractLogicalOperator) parentOpRef.getOperator();
+                List<Mutable<ILogicalOperator>> parentOpList = childrenToParents.get(ref);
+                for (Mutable<ILogicalOperator> parentOpRef : parentOpList) {
+                    AbstractLogicalOperator parentOp = (AbstractLogicalOperator) parentOpRef.getValue();
                     int index = parentOp.getInputs().indexOf(ref);
                     if (parentOp.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
                         AbstractLogicalOperator parentOpNext = (AbstractLogicalOperator) childrenToParents
-                                .get(parentOpRef).get(0).getOperator();
+                                .get(parentOpRef).get(0).getValue();
                         if (parentOpNext.isMap()) {
                             index = parentOpNext.getInputs().indexOf(parentOpRef);
                             parentOp = parentOpNext;
@@ -249,10 +251,10 @@
                     ILogicalOperator childOp = parentOp.getOperatorTag() == LogicalOperatorTag.PROJECT ? assignOperator
                             : projectOperator;
                     if (parentOp.isMap()) {
-                        parentOp.getInputs().set(index, new LogicalOperatorReference(childOp));
+                        parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(childOp));
                     } else {
-                        exchg.getInputs().add(new LogicalOperatorReference(childOp));
-                        parentOp.getInputs().set(index, new LogicalOperatorReference(exchg));
+                        exchg.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
+                        parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(exchg));
                     }
                     context.computeAndSetTypeEnvironmentForOperator(exchg);
                 }
@@ -263,19 +265,19 @@
     }
 
     private void genCandidates(IOptimizationContext context) throws AlgebricksException {
-        List<List<LogicalOperatorReference>> previousEquivalenceClasses = new ArrayList<List<LogicalOperatorReference>>();
+        List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
         while (equivalenceClasses.size() > 0) {
             previousEquivalenceClasses.clear();
-            for (List<LogicalOperatorReference> candidates : equivalenceClasses) {
-                List<LogicalOperatorReference> candidatesCopy = new ArrayList<LogicalOperatorReference>();
+            for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+                List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
                 candidatesCopy.addAll(candidates);
                 previousEquivalenceClasses.add(candidatesCopy);
             }
-            List<LogicalOperatorReference> currentLevelOpRefs = new ArrayList<LogicalOperatorReference>();
-            for (List<LogicalOperatorReference> candidates : equivalenceClasses) {
+            List<Mutable<ILogicalOperator>> currentLevelOpRefs = new ArrayList<Mutable<ILogicalOperator>>();
+            for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
                 if (candidates.size() > 0) {
-                    for (LogicalOperatorReference opRef : candidates) {
-                        List<LogicalOperatorReference> refs = childrenToParents.get(opRef);
+                    for (Mutable<ILogicalOperator> opRef : candidates) {
+                        List<Mutable<ILogicalOperator>> refs = childrenToParents.get(opRef);
                         if (refs != null)
                             currentLevelOpRefs.addAll(refs);
                     }
@@ -294,25 +296,25 @@
         }
     }
 
-    private void topDownMaterialization(List<LogicalOperatorReference> tops) {
-        List<LogicalOperatorReference> candidates = new ArrayList<LogicalOperatorReference>();
-        List<LogicalOperatorReference> nextLevel = new ArrayList<LogicalOperatorReference>();
-        for (LogicalOperatorReference op : tops) {
-            AbstractLogicalOperator aop = (AbstractLogicalOperator) op.getOperator();
+    private void topDownMaterialization(List<Mutable<ILogicalOperator>> tops) {
+        List<Mutable<ILogicalOperator>> candidates = new ArrayList<Mutable<ILogicalOperator>>();
+        List<Mutable<ILogicalOperator>> nextLevel = new ArrayList<Mutable<ILogicalOperator>>();
+        for (Mutable<ILogicalOperator> op : tops) {
+            AbstractLogicalOperator aop = (AbstractLogicalOperator) op.getValue();
             if ((aop.getOperatorTag() == LogicalOperatorTag.INNERJOIN || aop.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN)
                     && !joins.contains(op)) {
                 joins.add(op);
             }
-            for (LogicalOperatorReference opRef : op.getOperator().getInputs()) {
-                List<LogicalOperatorReference> opRefList = childrenToParents.get(opRef);
+            for (Mutable<ILogicalOperator> opRef : op.getValue().getInputs()) {
+                List<Mutable<ILogicalOperator>> opRefList = childrenToParents.get(opRef);
                 if (opRefList == null) {
-                    opRefList = new ArrayList<LogicalOperatorReference>();
+                    opRefList = new ArrayList<Mutable<ILogicalOperator>>();
                     childrenToParents.put(opRef, opRefList);
                     nextLevel.add(opRef);
                 }
                 opRefList.add(op);
             }
-            if (op.getOperator().getInputs().size() == 0)
+            if (op.getValue().getInputs().size() == 0)
                 candidates.add(op);
         }
         if (equivalenceClasses.size() > 0) {
@@ -325,17 +327,17 @@
         }
     }
 
-    private void candidatesGrow(List<LogicalOperatorReference> opList, List<LogicalOperatorReference> candidates) {
-        List<LogicalOperatorReference> previousCandidates = new ArrayList<LogicalOperatorReference>();
+    private void candidatesGrow(List<Mutable<ILogicalOperator>> opList, List<Mutable<ILogicalOperator>> candidates) {
+        List<Mutable<ILogicalOperator>> previousCandidates = new ArrayList<Mutable<ILogicalOperator>>();
         previousCandidates.addAll(candidates);
         candidates.clear();
         boolean validCandidate = false;
-        for (LogicalOperatorReference op : opList) {
-            for (LogicalOperatorReference inputRef : op.getOperator().getInputs()) {
+        for (Mutable<ILogicalOperator> op : opList) {
+            for (Mutable<ILogicalOperator> inputRef : op.getValue().getInputs()) {
                 validCandidate = false;
                 // if current input is in candidates
-                for (LogicalOperatorReference candidate : previousCandidates)
-                    if (inputRef.getOperator().equals(candidate.getOperator()))
+                for (Mutable<ILogicalOperator> candidate : previousCandidates)
+                    if (inputRef.getValue().equals(candidate.getValue()))
                         validCandidate = true;
                 // if one input is not in candidates
                 if (!validCandidate)
@@ -348,24 +350,24 @@
     }
 
     private void prune(IOptimizationContext context) throws AlgebricksException {
-        List<List<LogicalOperatorReference>> previousEquivalenceClasses = new ArrayList<List<LogicalOperatorReference>>();
-        for (List<LogicalOperatorReference> candidates : equivalenceClasses) {
-            List<LogicalOperatorReference> candidatesCopy = new ArrayList<LogicalOperatorReference>();
+        List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
+        for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+            List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
             candidatesCopy.addAll(candidates);
             previousEquivalenceClasses.add(candidatesCopy);
         }
         equivalenceClasses.clear();
-        for (List<LogicalOperatorReference> candidates : previousEquivalenceClasses) {
+        for (List<Mutable<ILogicalOperator>> candidates : previousEquivalenceClasses) {
             boolean[] reserved = new boolean[candidates.size()];
             for (int i = 0; i < reserved.length; i++)
                 reserved[i] = false;
             for (int i = candidates.size() - 1; i >= 0; i--) {
                 if (reserved[i] == false) {
-                    List<LogicalOperatorReference> equivalentClass = new ArrayList<LogicalOperatorReference>();
-                    ILogicalOperator candidate = candidates.get(i).getOperator();
+                    List<Mutable<ILogicalOperator>> equivalentClass = new ArrayList<Mutable<ILogicalOperator>>();
+                    ILogicalOperator candidate = candidates.get(i).getValue();
                     equivalentClass.add(candidates.get(i));
                     for (int j = i - 1; j >= 0; j--) {
-                        ILogicalOperator peer = candidates.get(j).getOperator();
+                        ILogicalOperator peer = candidates.get(j).getValue();
                         if (IsomorphismUtilities.isOperatorIsomorphic(candidate, peer)) {
                             reserved[i] = true;
                             reserved[j] = true;
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
index e9b895e..38032dd 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
@@ -14,11 +14,12 @@
  */
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
@@ -34,13 +35,14 @@
 public class ExtractGbyExpressionsRule extends AbstractExtractExprRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
         if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
             return false;
         }
@@ -63,12 +65,12 @@
         if (!gbyHasComplexExpr(g)) {
             return false;
         }
-        LogicalOperatorReference opRef2 = g.getInputs().get(0);
-        for (Pair<LogicalVariable, LogicalExpressionReference> gbyPair : g.getGroupByList()) {
-            ILogicalExpression expr = gbyPair.second.getExpression();
+        Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
+            ILogicalExpression expr = gbyPair.second.getValue();
             if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
-                gbyPair.second.setExpression(new VariableReferenceExpression(v));
+                gbyPair.second.setValue(new VariableReferenceExpression(v));
             }
         }
         return true;
@@ -78,20 +80,20 @@
         if (!decorHasComplexExpr(g)) {
             return false;
         }
-        LogicalOperatorReference opRef2 = g.getInputs().get(0);
-        for (Pair<LogicalVariable, LogicalExpressionReference> decorPair : g.getDecorList()) {
-            ILogicalExpression expr = decorPair.second.getExpression();
+        Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : g.getDecorList()) {
+            ILogicalExpression expr = decorPair.second.getValue();
             if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
-                decorPair.second.setExpression(new VariableReferenceExpression(v));
+                decorPair.second.setValue(new VariableReferenceExpression(v));
             }
         }
         return true;
     }
 
     private boolean gbyHasComplexExpr(GroupByOperator g) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> gbyPair : g.getGroupByList()) {
-            if (gbyPair.second.getExpression().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
+            if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 return true;
             }
         }
@@ -99,8 +101,8 @@
     }
 
     private boolean decorHasComplexExpr(GroupByOperator g) {
-        for (Pair<LogicalVariable, LogicalExpressionReference> gbyPair : g.getDecorList()) {
-            if (gbyPair.second.getExpression().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getDecorList()) {
+            if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 return true;
             }
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
index 91ebde7..99ed003 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
@@ -19,11 +19,13 @@
 import java.util.ListIterator;
 import java.util.Map;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
@@ -37,13 +39,14 @@
 public class FactorRedundantGroupAndDecorVarsRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
             return false;
         }
@@ -55,26 +58,26 @@
         return gvChanged || dvChanged;
     }
 
-    private boolean factorRedundantRhsVars(List<Pair<LogicalVariable, LogicalExpressionReference>> veList,
-            LogicalOperatorReference opRef, Map<LogicalVariable, LogicalVariable> varRhsToLhs,
+    private boolean factorRedundantRhsVars(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> veList,
+            Mutable<ILogicalOperator> opRef, Map<LogicalVariable, LogicalVariable> varRhsToLhs,
             IOptimizationContext context) throws AlgebricksException {
         varRhsToLhs.clear();
-        ListIterator<Pair<LogicalVariable, LogicalExpressionReference>> iter = veList.listIterator();
+        ListIterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = veList.listIterator();
         boolean changed = false;
         while (iter.hasNext()) {
-            Pair<LogicalVariable, LogicalExpressionReference> p = iter.next();
-            if (p.second.getExpression().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+            Pair<LogicalVariable, Mutable<ILogicalExpression>> p = iter.next();
+            if (p.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 continue;
             }
             LogicalVariable v = GroupByOperator.getDecorVariable(p);
             LogicalVariable lhs = varRhsToLhs.get(v);
             if (lhs != null) {
                 if (p.first != null) {
-                    AssignOperator assign = new AssignOperator(p.first, new LogicalExpressionReference(
+                    AssignOperator assign = new AssignOperator(p.first, new MutableObject<ILogicalExpression>(
                             new VariableReferenceExpression(lhs)));
-                    ILogicalOperator op = opRef.getOperator();
-                    assign.getInputs().add(new LogicalOperatorReference(op));
-                    opRef.setOperator(assign);
+                    ILogicalOperator op = opRef.getValue();
+                    assign.getInputs().add(new MutableObject<ILogicalOperator>(op));
+                    opRef.setValue(assign);
                     context.computeAndSetTypeEnvironmentForOperator(assign);
                 }
                 iter.remove();
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
index a69db2a..e1d5549 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
@@ -1,21 +1,23 @@
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 
 public class InferTypesRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator op = opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        ILogicalOperator op = opRef.getValue();
         if (context.getOutputTypeEnvironment(op) != null) {
             return false;
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
index 3a093d7..0cc2b53 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
@@ -19,13 +19,14 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -46,7 +47,7 @@
 public class InlineVariablesRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
@@ -56,8 +57,8 @@
      * Does one big DFS sweep over the plan.
      * 
      */
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getOperator())) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
             return false;
         }
         VariableSubstitutionVisitor substVisitor = new VariableSubstitutionVisitor(false);
@@ -69,12 +70,12 @@
         return bb.first;
     }
 
-    private Pair<Boolean, Boolean> collectEqClassesAndRemoveRedundantOps(LogicalOperatorReference opRef,
+    private Pair<Boolean, Boolean> collectEqClassesAndRemoveRedundantOps(Mutable<ILogicalOperator> opRef,
             IOptimizationContext context, boolean first, List<EquivalenceClass> equivClasses,
             VariableSubstitutionVisitor substVisitor, VariableSubstitutionVisitor substVisitorForWrites)
             throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
-        // if (context.checkIfInDontApplySet(this, opRef.getOperator())) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        // if (context.checkIfInDontApplySet(this, opRef.getValue())) {
         // return false;
         // }
         if (op.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
@@ -83,7 +84,7 @@
         boolean modified = false;
         boolean ecChange = false;
         int cnt = 0;
-        for (LogicalOperatorReference i : op.getInputs()) {
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
             boolean isOuterInputBranch = op.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN && cnt == 1;
             List<EquivalenceClass> eqc = isOuterInputBranch ? new LinkedList<EquivalenceClass>() : equivClasses;
 
@@ -120,7 +121,7 @@
                 eqc = equivClasses;
             }
             for (ILogicalPlan p : n.getNestedPlans()) {
-                for (LogicalOperatorReference r : p.getRoots()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
                     Pair<Boolean, Boolean> bb = collectEqClassesAndRemoveRedundantOps(r, context, false, eqc,
                             substVisitor, substVisitorForWrites);
                     if (bb.first) {
@@ -135,7 +136,7 @@
         // we assume a variable is assigned a value only once
         if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
             AssignOperator a = (AssignOperator) op;
-            ILogicalExpression rhs = a.getExpressions().get(0).getExpression();
+            ILogicalExpression rhs = a.getExpressions().get(0).getValue();
             if (rhs.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 LogicalVariable varLeft = a.getVariables().get(0);
                 VariableReferenceExpression varRef = (VariableReferenceExpression) rhs;
@@ -185,18 +186,18 @@
                     modified = true;
                     if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
                         GroupByOperator group = (GroupByOperator) op;
-                        for (Pair<LogicalVariable, LogicalExpressionReference> gp : group.getGroupByList()) {
+                        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gp : group.getGroupByList()) {
                             if (gp.first != null
-                                    && gp.second.getExpression().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                                LogicalVariable gv = ((VariableReferenceExpression) gp.second.getExpression())
+                                    && gp.second.getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                                LogicalVariable gv = ((VariableReferenceExpression) gp.second.getValue())
                                         .getVariableReference();
-                                Iterator<Pair<LogicalVariable, LogicalExpressionReference>> iter = group.getDecorList()
-                                        .iterator();
+                                Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = group
+                                        .getDecorList().iterator();
                                 while (iter.hasNext()) {
-                                    Pair<LogicalVariable, LogicalExpressionReference> dp = iter.next();
+                                    Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
                                     if (dp.first == null
-                                            && dp.second.getExpression().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                                        LogicalVariable dv = ((VariableReferenceExpression) dp.second.getExpression())
+                                            && dp.second.getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                                        LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue())
                                                 .getVariableReference();
                                         if (dv == gv) {
                                             // The decor variable is redundant,
@@ -227,12 +228,12 @@
         return new Pair<Boolean, Boolean>(modified, ecChange);
     }
 
-    private Pair<Boolean, Boolean> processVarExprPairs(List<Pair<LogicalVariable, LogicalExpressionReference>> vePairs,
-            List<EquivalenceClass> equivClasses) {
+    private Pair<Boolean, Boolean> processVarExprPairs(
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairs, List<EquivalenceClass> equivClasses) {
         boolean ecFromGroup = false;
         boolean modified = false;
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : vePairs) {
-            ILogicalExpression expr = p.second.getExpression();
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : vePairs) {
+            ILogicalExpression expr = p.second.getValue();
             if (p.first != null && expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                 VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                 LogicalVariable rhsVar = varRef.getVariableReference();
@@ -294,8 +295,8 @@
         }
 
         @Override
-        public boolean transform(LogicalExpressionReference exprRef) {
-            ILogicalExpression e = exprRef.getExpression();
+        public boolean transform(Mutable<ILogicalExpression> exprRef) {
+            ILogicalExpression e = exprRef.getValue();
             switch (((AbstractLogicalExpression) e).getExpressionTag()) {
                 case VARIABLE: {
                     // look for a required substitution
@@ -311,12 +312,12 @@
                         if (doNotSubstWithConst) {
                             return false;
                         }
-                        exprRef.setExpression(ec.getConstRepresentative());
+                        exprRef.setValue(ec.getConstRepresentative());
                         return true;
                     } else {
                         LogicalVariable r = ec.getVariableRepresentative();
                         if (!r.equals(var)) {
-                            exprRef.setExpression(new VariableReferenceExpression(r));
+                            exprRef.setValue(new VariableReferenceExpression(r));
                             return true;
                         } else {
                             return false;
@@ -326,7 +327,7 @@
                 case FUNCTION_CALL: {
                     AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) e;
                     boolean m = false;
-                    for (LogicalExpressionReference arg : fce.getArguments()) {
+                    for (Mutable<ILogicalExpression> arg : fce.getArguments()) {
                         if (transform(arg)) {
                             m = true;
                         }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
index d15af57..25df119 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
@@ -16,9 +16,11 @@
 
 import java.util.Iterator;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -31,13 +33,13 @@
 public class InsertOuterJoinRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
         if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
             return false;
         }
@@ -54,47 +56,47 @@
         if (p.getRoots().size() != 1) {
             return false;
         }
-        LogicalOperatorReference subplanRoot = p.getRoots().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getOperator();
-        LogicalOperatorReference opUnder = subplan.getInputs().get(0);
+        Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
+        Mutable<ILogicalOperator> opUnder = subplan.getInputs().get(0);
 
-        if (OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) opUnder.getOperator())) {
+        if (OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) opUnder.getValue())) {
             return false;
         }
 
         switch (op1.getOperatorTag()) {
             case INNERJOIN: {
                 InnerJoinOperator join = (InnerJoinOperator) op1;
-                LogicalOperatorReference leftRef = join.getInputs().get(0);
-                LogicalOperatorReference rightRef = join.getInputs().get(1);
-                LogicalOperatorReference ntsRef = getNtsAtEndOfPipeline(leftRef);
+                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
+                Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
+                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
                 if (ntsRef == null) {
                     ntsRef = getNtsAtEndOfPipeline(rightRef);
                     if (ntsRef == null) {
                         return false;
                     } else {
-                        LogicalOperatorReference t = leftRef;
+                        Mutable<ILogicalOperator> t = leftRef;
                         leftRef = rightRef;
                         rightRef = t;
                     }
                 }
-                ntsRef.setOperator(opUnder.getOperator());
+                ntsRef.setValue(opUnder.getValue());
                 LeftOuterJoinOperator loj = new LeftOuterJoinOperator(join.getCondition());
                 loj.getInputs().add(leftRef);
                 loj.getInputs().add(rightRef);
-                opRef.setOperator(loj);
+                opRef.setValue(loj);
                 context.computeAndSetTypeEnvironmentForOperator(loj);
                 return true;
             }
             case LEFTOUTERJOIN: {
                 LeftOuterJoinOperator join = (LeftOuterJoinOperator) op1;
-                LogicalOperatorReference leftRef = join.getInputs().get(0);
-                LogicalOperatorReference ntsRef = getNtsAtEndOfPipeline(leftRef);
+                Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
+                Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
                 if (ntsRef == null) {
                     return false;
                 }
-                ntsRef.setOperator(opUnder.getOperator());
-                opRef.setOperator(join);
+                ntsRef.setValue(opUnder.getValue());
+                opRef.setValue(join);
                 context.computeAndSetTypeEnvironmentForOperator(join);
                 return true;
             }
@@ -104,8 +106,8 @@
         }
     }
 
-    private LogicalOperatorReference getNtsAtEndOfPipeline(LogicalOperatorReference opRef) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    private Mutable<ILogicalOperator> getNtsAtEndOfPipeline(Mutable<ILogicalOperator> opRef) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
             return opRef;
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
index d87f00b..337117e 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
@@ -17,9 +17,11 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -34,7 +36,8 @@
 public class InsertProjectBeforeUnionRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         return false;
     }
 
@@ -44,8 +47,8 @@
      * write order
      */
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
             return false;
         }
@@ -60,10 +63,10 @@
         }
 
         ArrayList<LogicalVariable> inputSchemaOne = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(opUnion.getInputs().get(0).getOperator(), inputSchemaOne);
+        VariableUtilities.getLiveVariables(opUnion.getInputs().get(0).getValue(), inputSchemaOne);
 
         ArrayList<LogicalVariable> inputSchemaTwo = new ArrayList<LogicalVariable>();
-        VariableUtilities.getLiveVariables(opUnion.getInputs().get(1).getOperator(), inputSchemaTwo);
+        VariableUtilities.getLiveVariables(opUnion.getInputs().get(1).getValue(), inputSchemaTwo);
 
         boolean rewritten = false;
         if (!isIdentical(usedVariablesFromOne, inputSchemaOne)) {
@@ -80,9 +83,9 @@
     private void insertProjectOperator(UnionAllOperator opUnion, int branch, ArrayList<LogicalVariable> usedVariables,
             IOptimizationContext context) throws AlgebricksException {
         ProjectOperator projectOp = new ProjectOperator(usedVariables);
-        ILogicalOperator parentOp = opUnion.getInputs().get(branch).getOperator();
-        projectOp.getInputs().add(new LogicalOperatorReference(parentOp));
-        opUnion.getInputs().get(branch).setOperator(projectOp);
+        ILogicalOperator parentOp = opUnion.getInputs().get(branch).getValue();
+        projectOp.getInputs().add(new MutableObject<ILogicalOperator>(parentOp));
+        opUnion.getInputs().get(branch).setValue(projectOp);
         projectOp.setPhysicalOperator(new StreamProjectPOperator());
         context.computeAndSetTypeEnvironmentForOperator(parentOp);
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
index 494a7ba..0e12bcc 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
@@ -3,8 +3,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
@@ -17,18 +19,19 @@
 public class IntroHashPartitionMergeExchange implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
         if (op1.getPhysicalOperator() == null
                 || op1.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_EXCHANGE) {
             return false;
         }
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
         if (op2.getPhysicalOperator() == null
                 || op2.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.SORT_MERGE_EXCHANGE) {
             return false;
@@ -42,7 +45,7 @@
         HashPartitionMergeExchangePOperator hpme = new HashPartitionMergeExchangePOperator(ocList, hpe.getHashFields(),
                 hpe.getDomain());
         op1.setPhysicalOperator(hpme);
-        op1.getInputs().get(0).setOperator(op2.getInputs().get(0).getOperator());
+        op1.getInputs().get(0).setValue(op2.getInputs().get(0).getValue());
         return true;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
index 31bb3b7..acff2de 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
@@ -18,10 +18,13 @@
 import java.util.ListIterator;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -36,15 +39,16 @@
 public class IntroJoinInsideSubplanRule extends AbstractDecorrelationRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
         if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
             return false;
         }
         SubplanOperator subplan = (SubplanOperator) op0;
 
-        LogicalOperatorReference leftRef = subplan.getInputs().get(0);
-        if (((AbstractLogicalOperator) leftRef.getOperator()).getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+        Mutable<ILogicalOperator> leftRef = subplan.getInputs().get(0);
+        if (((AbstractLogicalOperator) leftRef.getValue()).getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
             return false;
         }
 
@@ -59,16 +63,16 @@
         if (p.getRoots().size() != 1) {
             return false;
         }
-        LogicalOperatorReference opRef1 = p.getRoots().get(0);
+        Mutable<ILogicalOperator> opRef1 = p.getRoots().get(0);
 
         while (true) {
-            AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef1.getOperator();
+            AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef1.getValue();
             if (op1.getInputs().size() != 1) {
                 return false;
             }
             if (op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
-                LogicalOperatorReference op2Ref = op1.getInputs().get(0);
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getOperator();
+                Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
                 if (op2.getOperatorTag() != LogicalOperatorTag.SELECT && descOrSelfIsScanOrJoin(op2)) {
                     Set<LogicalVariable> free2 = new HashSet<LogicalVariable>();
                     OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, free2);
@@ -77,13 +81,13 @@
                         OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op1, free1);
                         if (!free1.isEmpty()) {
                             OperatorManipulationUtil.ntsToEts(op2Ref, context);
-                            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new LogicalOperatorReference(
-                                    subplan));
-                            LogicalOperatorReference ntsRef = new LogicalOperatorReference(nts);
-                            LogicalOperatorReference innerRef = new LogicalOperatorReference(op2);
-                            InnerJoinOperator join = new InnerJoinOperator(new LogicalExpressionReference(
+                            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(
+                                    new MutableObject<ILogicalOperator>(subplan));
+                            Mutable<ILogicalOperator> ntsRef = new MutableObject<ILogicalOperator>(nts);
+                            Mutable<ILogicalOperator> innerRef = new MutableObject<ILogicalOperator>(op2);
+                            InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(
                                     ConstantExpression.TRUE), ntsRef, innerRef);
-                            op2Ref.setOperator(join);
+                            op2Ref.setValue(join);
                             context.computeAndSetTypeEnvironmentForOperator(nts);
                             context.computeAndSetTypeEnvironmentForOperator(join);
                             return true;
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java
index d90504c..c05572b 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java
@@ -9,12 +9,13 @@
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
@@ -23,11 +24,11 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
 import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
 import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
@@ -38,13 +39,14 @@
 public class IntroduceCombinerRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (context.checkIfInDontApplySet(this, op)) {
             return false;
         }
@@ -70,17 +72,17 @@
         for (Map.Entry<AggregateFunctionCallExpression, SimilarAggregatesInfo> entry : toReplaceMap.entrySet()) {
             SimilarAggregatesInfo sai = entry.getValue();
             for (AggregateExprInfo aei : sai.simAggs) {
-                AbstractFunctionCallExpression afce = (AbstractFunctionCallExpression) aei.aggExprRef.getExpression();
+                AbstractFunctionCallExpression afce = (AbstractFunctionCallExpression) aei.aggExprRef.getValue();
                 afce.setFunctionInfo(aei.newFunInfo);
                 afce.getArguments().clear();
-                afce.getArguments().add(new LogicalExpressionReference(sai.stepOneResult));
+                afce.getArguments().add(new MutableObject<ILogicalExpression>(sai.stepOneResult));
             }
         }
 
-        for (Pair<LogicalVariable, LogicalExpressionReference> p : gbyOp.getDecorList()) {
+        for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) {
             LogicalVariable newDecorVar = context.newVar();
-            newGbyOp.addDecorExpression(newDecorVar, p.second.getExpression());
-            p.second.setExpression(new VariableReferenceExpression(newDecorVar));
+            newGbyOp.addDecorExpression(newDecorVar, p.second.getValue());
+            p.second.setValue(new VariableReferenceExpression(newDecorVar));
         }
         newGbyOp.setExecutionMode(ExecutionMode.LOCAL);
         Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY);
@@ -99,13 +101,13 @@
             if (!propagatedVars.contains(var)) {
                 LogicalVariable newDecorVar = context.newVar();
                 newGbyOp.addDecorExpression(newDecorVar, new VariableReferenceExpression(var));
-                VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getOperator(),
-                        var, newDecorVar, context);
+                VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var,
+                        newDecorVar, context);
             }
         }
 
-        LogicalOperatorReference opRef3 = gbyOp.getInputs().get(0);
-        opRef3.setOperator(newGbyOp);
+        Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+        opRef3.setValue(newGbyOp);
         typeGby(newGbyOp, context);
         typeGby(gbyOp, context);
         return true;
@@ -121,10 +123,10 @@
     private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context)
             throws AlgebricksException {
 
-        LogicalOperatorReference opRef3 = gbyOp.getInputs().get(0);
-        ILogicalOperator op3 = opRef3.getOperator();
+        Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+        ILogicalOperator op3 = opRef3.getValue();
         GroupByOperator newGbyOp = new GroupByOperator();
-        newGbyOp.getInputs().add(new LogicalOperatorReference(op3));
+        newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3));
         // copy annotations
         Map<String, Object> annotations = newGbyOp.getAnnotations();
         for (Entry<String, Object> a : gbyOp.getAnnotations().entrySet())
@@ -183,15 +185,15 @@
 
     private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan p, GroupByOperator oldGbyOp,
             GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context) {
-        List<LogicalOperatorReference> pushedRoots = new ArrayList<LogicalOperatorReference>();
-        List<LogicalOperatorReference> toPushR = new ArrayList<LogicalOperatorReference>();
-        for (LogicalOperatorReference r : p.getRoots()) {
+        List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
+        List<Mutable<ILogicalOperator>> toPushR = new ArrayList<Mutable<ILogicalOperator>>();
+        for (Mutable<ILogicalOperator> r : p.getRoots()) {
             if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, toPushR)) {
                 // for now, if we cannot push everything, give up
                 return new Pair<Boolean, ILogicalPlan>(false, null);
             }
         }
-        for (LogicalOperatorReference root : toPushR) {
+        for (Mutable<ILogicalOperator> root : toPushR) {
             pushedRoots.add(root);
         }
         if (pushedRoots.isEmpty()) {
@@ -201,21 +203,21 @@
         }
     }
 
-    private boolean tryToPushRoot(LogicalOperatorReference r, GroupByOperator oldGbyOp, GroupByOperator newGbyOp,
+    private boolean tryToPushRoot(Mutable<ILogicalOperator> r, GroupByOperator oldGbyOp, GroupByOperator newGbyOp,
             BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context,
-            List<LogicalOperatorReference> toPushAccumulate) {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getOperator();
+            List<Mutable<ILogicalOperator>> toPushAccumulate) {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
         if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
             return false;
         }
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
         if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
             AggregateOperator initAgg = (AggregateOperator) op1;
-            Pair<Boolean, LogicalOperatorReference> pOpRef = tryToPushAgg(initAgg, newGbyOp, bi.toReplaceMap, context);
+            Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, bi.toReplaceMap, context);
             if (!pOpRef.first) {
                 return false;
             }
-            LogicalOperatorReference opRef = pOpRef.second;
+            Mutable<ILogicalOperator> opRef = pOpRef.second;
             if (opRef != null) {
                 toPushAccumulate.add(opRef);
             }
@@ -223,7 +225,7 @@
             return true;
         } else {
             while (op2.getOperatorTag() != LogicalOperatorTag.GROUP && op2.getInputs().size() == 1) {
-                op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getOperator();
+                op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
             }
             if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
                 return false;
@@ -233,7 +235,7 @@
             List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
             concatGbyVars.addAll(gbyVars2);
             for (ILogicalPlan p : nestedGby.getNestedPlans()) {
-                for (LogicalOperatorReference r2 : p.getRoots()) {
+                for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
                     if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate)) {
                         return false;
                     }
@@ -243,34 +245,34 @@
         }
     }
 
-    private Pair<Boolean, LogicalOperatorReference> tryToPushAgg(AggregateOperator initAgg, GroupByOperator newGbyOp,
+    private Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg, GroupByOperator newGbyOp,
             Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap, IOptimizationContext context) {
 
         ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
-        ArrayList<LogicalExpressionReference> pushedExprs = new ArrayList<LogicalExpressionReference>();
+        ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
 
         List<LogicalVariable> initVars = initAgg.getVariables();
-        List<LogicalExpressionReference> initExprs = initAgg.getExpressions();
+        List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
         int sz = initVars.size();
         for (int i = 0; i < sz; i++) {
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getExpression();
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
             if (!aggFun.isTwoStep()) {
-                return new Pair<Boolean, LogicalOperatorReference>(false, null);
+                return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
             }
         }
 
         boolean haveAggToReplace = false;
         for (int i = 0; i < sz; i++) {
-            LogicalExpressionReference expRef = initExprs.get(i);
-            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getExpression();
+            Mutable<ILogicalExpression> expRef = initExprs.get(i);
+            AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
             IFunctionInfo fi1 = aggFun.getStepOneAggregate();
-            List<LogicalExpressionReference> newArgs = new ArrayList<LogicalExpressionReference>(aggFun.getArguments()
-                    .size());
-            for (LogicalExpressionReference er : aggFun.getArguments()) {
-                newArgs.add(new LogicalExpressionReference(er.getExpression().cloneExpression()));
+            List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun
+                    .getArguments().size());
+            for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
+                newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
             }
-//            AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
-//            pushedExprs.add(new LogicalExpressionReference(aggLocal));
+            //            AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
+            //            pushedExprs.add(new Mutable<ILogicalExpression>(aggLocal));
 
             IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
 
@@ -283,7 +285,7 @@
                 inf.simAggs = new ArrayList<AggregateExprInfo>();
                 toReplaceMap.put(aggFun, inf);
                 AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
-                pushedExprs.add(new LogicalExpressionReference(aggLocal));
+                pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
             }
             AggregateExprInfo aei = new AggregateExprInfo();
             aei.aggExprRef = expRef;
@@ -295,12 +297,12 @@
         if (!pushedVars.isEmpty()) {
             AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
             pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
-            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new LogicalOperatorReference(newGbyOp));
+            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(newGbyOp));
             nts.setExecutionMode(ExecutionMode.LOCAL);
-            pushedAgg.getInputs().add(new LogicalOperatorReference(nts));
-            return new Pair<Boolean, LogicalOperatorReference>(true, new LogicalOperatorReference(pushedAgg));
+            pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(nts));
+            return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
         } else {
-            return new Pair<Boolean, LogicalOperatorReference>(haveAggToReplace, null);
+            return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
         }
     }
 
@@ -310,7 +312,7 @@
     }
 
     private class AggregateExprInfo {
-        LogicalExpressionReference aggExprRef;
+        Mutable<ILogicalExpression> aggExprRef;
         IFunctionInfo newFunInfo;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java
index eeaac23..34060c2 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java
@@ -18,10 +18,13 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -46,18 +49,19 @@
 public class IntroduceGroupByForStandaloneAggregRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
             return false;
         }
-        LogicalOperatorReference opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
         if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
             return false;
         }
@@ -71,26 +75,26 @@
         List<LogicalVariable> used = new LinkedList<LogicalVariable>();
         VariableUtilities.getUsedVariables(assign, used);
         if (used.contains(aggVar)) {
-            LogicalOperatorReference opRef3 = op2.getInputs().get(0);
-            List<Pair<LogicalVariable, LogicalExpressionReference>> groupByList = new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>();
+            Mutable<ILogicalOperator> opRef3 = op2.getInputs().get(0);
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
             LogicalVariable gbyVar = context.newVar();
             // ILogicalExpression constOne = new ConstantExpression(new
             // IntegerLiteral(new Integer(1)));
-            groupByList.add(new Pair<LogicalVariable, LogicalExpressionReference>(gbyVar,
-                    new LogicalExpressionReference(ConstantExpression.TRUE)));
-            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new LogicalOperatorReference());
-            List<LogicalOperatorReference> aggInpList = agg.getInputs();
+            groupByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(gbyVar,
+                    new MutableObject<ILogicalExpression>(ConstantExpression.TRUE)));
+            NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>());
+            List<Mutable<ILogicalOperator>> aggInpList = agg.getInputs();
             aggInpList.clear();
-            aggInpList.add(new LogicalOperatorReference(nts));
+            aggInpList.add(new MutableObject<ILogicalOperator>(nts));
             ILogicalPlan np1 = new ALogicalPlanImpl(opRef2);
             ArrayList<ILogicalPlan> nestedPlans = new ArrayList<ILogicalPlan>();
             nestedPlans.add(np1);
             GroupByOperator gbyOp = new GroupByOperator(groupByList,
-                    new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>(), nestedPlans);
-            LogicalOperatorReference opRefGby = new LogicalOperatorReference(gbyOp);
-            nts.getDataSourceReference().setOperator(gbyOp);
+                    new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>(), nestedPlans);
+            Mutable<ILogicalOperator> opRefGby = new MutableObject<ILogicalOperator>(gbyOp);
+            nts.getDataSourceReference().setValue(gbyOp);
             gbyOp.getInputs().add(opRef3);
-            List<LogicalOperatorReference> asgnInpList = assign.getInputs();
+            List<Mutable<ILogicalOperator>> asgnInpList = assign.getInputs();
             context.computeAndSetTypeEnvironmentForOperator(nts);
             context.computeAndSetTypeEnvironmentForOperator(agg);
             context.computeAndSetTypeEnvironmentForOperator(gbyOp);
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
index 4f9b2ea..87297e7 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
@@ -23,11 +23,13 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -60,13 +62,14 @@
 public class IntroduceGroupByForSubplanRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
         if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
             return false;
         }
@@ -83,10 +86,10 @@
         if (p.getRoots().size() != 1) {
             return false;
         }
-        LogicalOperatorReference subplanRoot = p.getRoots().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getOperator();
+        Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
 
-        LogicalOperatorReference botRef = subplanRoot;
+        Mutable<ILogicalOperator> botRef = subplanRoot;
         AbstractLogicalOperator op2;
         // Project is optional
         if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
@@ -94,7 +97,7 @@
         } else {
             ProjectOperator project = (ProjectOperator) op1;
             botRef = project.getInputs().get(0);
-            op2 = (AbstractLogicalOperator) botRef.getOperator();
+            op2 = (AbstractLogicalOperator) botRef.getValue();
         }
         if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
             return false;
@@ -104,8 +107,8 @@
         Set<LogicalVariable> free = new HashSet<LogicalVariable>();
         VariableUtilities.getUsedVariables(aggregate, free);
 
-        LogicalOperatorReference op3Ref = aggregate.getInputs().get(0);
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getOperator();
+        Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
+        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
 
         while (op3.getInputs().size() == 1) {
             Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
@@ -114,7 +117,7 @@
             VariableUtilities.getUsedVariables(op3, free);
             botRef = op3Ref;
             op3Ref = op3.getInputs().get(0);
-            op3 = (AbstractLogicalOperator) op3Ref.getOperator();
+            op3 = (AbstractLogicalOperator) op3Ref.getValue();
         }
 
         if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN
@@ -122,16 +125,16 @@
             return false;
         }
         AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
-        if (join.getCondition().getExpression() == ConstantExpression.TRUE) {
+        if (join.getCondition().getValue() == ConstantExpression.TRUE) {
             return false;
         }
         VariableUtilities.getUsedVariables(join, free);
 
-        AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getOperator();
+        AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
         // see if there's an NTS at the end of the pipeline
         NestedTupleSourceOperator outerNts = getNts(b0);
         if (outerNts == null) {
-            AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getOperator();
+            AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
             outerNts = getNts(b1);
             if (outerNts == null) {
                 return false;
@@ -145,9 +148,9 @@
         }
         AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
 
-        LogicalOperatorReference rightRef = join.getInputs().get(1);
+        Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
         LogicalVariable testForNull = null;
-        AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getOperator();
+        AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
         switch (right.getOperatorTag()) {
             case UNNEST: {
                 UnnestOperator innerUnnest = (UnnestOperator) right;
@@ -166,43 +169,43 @@
         }
         if (testForNull == null) {
             testForNull = context.newVar();
-            AssignOperator tmpAsgn = new AssignOperator(testForNull, new LogicalExpressionReference(
+            AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(
                     ConstantExpression.TRUE));
-            tmpAsgn.getInputs().add(new LogicalOperatorReference(rightRef.getOperator()));
-            rightRef.setOperator(tmpAsgn);
+            tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
+            rightRef.setValue(tmpAsgn);
             context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
         }
 
         IFunctionInfo finfoEq = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.IS_NULL);
-        ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq, new LogicalExpressionReference(
-                new VariableReferenceExpression(testForNull)));
+        ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq,
+                new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
         IFunctionInfo finfoNot = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NOT);
         ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot,
-                new LogicalExpressionReference(isNullTest));
-        SelectOperator selectNonNull = new SelectOperator(new LogicalExpressionReference(nonNullTest));
+                new MutableObject<ILogicalExpression>(isNullTest));
+        SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest));
         GroupByOperator g = new GroupByOperator();
-        LogicalOperatorReference newSubplanRef = new LogicalOperatorReference(subplan);
-        NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new LogicalOperatorReference(g));
-        opRef.setOperator(g);
-        selectNonNull.getInputs().add(new LogicalOperatorReference(nts));
+        Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
+        NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
+        opRef.setValue(g);
+        selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
 
-        List<LogicalOperatorReference> prodInpList = botRef.getOperator().getInputs();
+        List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
         prodInpList.clear();
-        prodInpList.add(new LogicalOperatorReference(selectNonNull));
+        prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
 
-        ILogicalPlan gPlan = new ALogicalPlanImpl(new LogicalOperatorReference(subplanRoot.getOperator()));
+        ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
         g.getNestedPlans().add(gPlan);
-        subplanRoot.setOperator(op3Ref.getOperator());
+        subplanRoot.setValue(op3Ref.getValue());
         g.getInputs().add(newSubplanRef);
 
         HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
-        VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getOperator(), underVars);
+        VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
         underVars.removeAll(pkVars);
         Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
         context.updatePrimaryKeys(mappedVars);
         for (LogicalVariable uv : underVars) {
             g.getDecorList().add(
-                    new Pair<LogicalVariable, LogicalExpressionReference>(null, new LogicalExpressionReference(
+                    new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(
                             new VariableReferenceExpression(uv))));
         }
         OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
@@ -220,7 +223,7 @@
             if (alo.getInputs().size() != 1) {
                 return null;
             }
-            alo = (AbstractLogicalOperator) alo.getInputs().get(0).getOperator();
+            alo = (AbstractLogicalOperator) alo.getInputs().get(0).getValue();
         } while (true);
     }
 
@@ -245,20 +248,20 @@
 
     private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars,
             IOptimizationContext context, GroupByOperator g,
-            List<Pair<LogicalVariable, LogicalExpressionReference>> outVeList) throws AlgebricksException {
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
         Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>();
         for (LogicalVariable ov : vars) {
             LogicalVariable newVar = context.newVar();
             ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
-            outVeList.add(new Pair<LogicalVariable, LogicalExpressionReference>(ov, new LogicalExpressionReference(
-                    varExpr)));
+            outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
+                    new MutableObject<ILogicalExpression>(varExpr)));
             for (ILogicalPlan p : g.getNestedPlans()) {
-                for (LogicalOperatorReference r : p.getRoots()) {
-                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getOperator(), ov, newVar,
-                            true, context);
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
+                            context);
                 }
             }
-            AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getOperator();
+            AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue();
             OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context);
             m.put(ov, newVar);
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
index 40b5c85..194b818 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
@@ -14,15 +14,17 @@
  */
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -37,8 +39,9 @@
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         IPhysicalOperator pt = op.getPhysicalOperator();
 
         if (pt == null || op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
@@ -52,17 +55,17 @@
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
-    private boolean testIfExchangeBelow(LogicalOperatorReference opRef, IOptimizationContext context)
+    private boolean testIfExchangeBelow(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
             throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         boolean exchInserted = false;
 
-        for (LogicalOperatorReference i : op.getInputs()) {
-            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getOperator();
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
             if (c.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
                 if (c.getPhysicalOperator() == null) {
                     return false;
@@ -73,8 +76,8 @@
         }
         IPhysicalOperator pt = op.getPhysicalOperator();
         if (pt.isJobGenDisabledBelowMe() || arrayContains(operatorsBelowWhichJobGenIsDisabled, pt.getOperatorTag())) {
-            for (LogicalOperatorReference i : op.getInputs()) {
-                disableJobGenRec(i.getOperator());
+            for (Mutable<ILogicalOperator> i : op.getInputs()) {
+                disableJobGenRec(i.getValue());
             }
         }
         return exchInserted;
@@ -83,20 +86,20 @@
     private void disableJobGenRec(ILogicalOperator operator) {
         AbstractLogicalOperator op = (AbstractLogicalOperator) operator;
         op.disableJobGen();
-        for (LogicalOperatorReference i : op.getInputs()) {
-            disableJobGenRec(i.getOperator());
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            disableJobGenRec(i.getValue());
         }
     }
 
-    private boolean testIfExchangeAbove(LogicalOperatorReference opRef, IOptimizationContext context)
+    private boolean testIfExchangeAbove(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
             throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
             return false;
         }
         boolean exchInserted = false;
-        for (LogicalOperatorReference i : op.getInputs()) {
-            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getOperator();
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
             IPhysicalOperator cpop = c.getPhysicalOperator();
             if (c.getOperatorTag() == LogicalOperatorTag.EXCHANGE || cpop == null) {
                 continue;
@@ -118,14 +121,14 @@
         return false;
     }
 
-    private final static void insertOneToOneExchange(LogicalOperatorReference i, IOptimizationContext context)
+    private final static void insertOneToOneExchange(Mutable<ILogicalOperator> i, IOptimizationContext context)
             throws AlgebricksException {
         ExchangeOperator e = new ExchangeOperator();
         e.setPhysicalOperator(new OneToOneExchangePOperator());
-        ILogicalOperator inOp = i.getOperator();
+        ILogicalOperator inOp = i.getValue();
 
-        e.getInputs().add(new LogicalOperatorReference(inOp));
-        i.setOperator(e);
+        e.getInputs().add(new MutableObject<ILogicalOperator>(inOp));
+        i.setValue(e);
         // e.recomputeSchema();
         OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(e, context);
         ExecutionMode em = ((AbstractLogicalOperator) inOp).getExecutionMode();
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
index 6755e59..7ee1f49 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
@@ -17,11 +17,13 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
@@ -36,17 +38,18 @@
 
 public class PullSelectOutOfEqJoin implements IAlgebraicRewriteRule {
 
-    private List<LogicalExpressionReference> eqVarVarComps = new ArrayList<LogicalExpressionReference>();
-    private List<LogicalExpressionReference> otherPredicates = new ArrayList<LogicalExpressionReference>();
+    private List<Mutable<ILogicalExpression>> eqVarVarComps = new ArrayList<Mutable<ILogicalExpression>>();
+    private List<Mutable<ILogicalExpression>> otherPredicates = new ArrayList<Mutable<ILogicalExpression>>();
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
 
         if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN
                 && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
@@ -54,7 +57,7 @@
         }
         AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
 
-        ILogicalExpression expr = join.getCondition().getExpression();
+        ILogicalExpression expr = join.getCondition().getValue();
         if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
             return false;
         }
@@ -65,8 +68,8 @@
         }
         eqVarVarComps.clear();
         otherPredicates.clear();
-        for (LogicalExpressionReference arg : fexp.getArguments()) {
-            if (isEqVarVar(arg.getExpression())) {
+        for (Mutable<ILogicalExpression> arg : fexp.getArguments()) {
+            if (isEqVarVar(arg.getValue())) {
                 eqVarVarComps.add(arg);
             } else {
                 otherPredicates.add(arg);
@@ -77,21 +80,21 @@
         }
         // pull up
         ILogicalExpression pulledCond = makeCondition(otherPredicates);
-        SelectOperator select = new SelectOperator(new LogicalExpressionReference(pulledCond));
+        SelectOperator select = new SelectOperator(new MutableObject<ILogicalExpression>(pulledCond));
         ILogicalExpression newJoinCond = makeCondition(eqVarVarComps);
-        join.getCondition().setExpression(newJoinCond);
-        select.getInputs().add(new LogicalOperatorReference(join));
-        opRef.setOperator(select);
+        join.getCondition().setValue(newJoinCond);
+        select.getInputs().add(new MutableObject<ILogicalOperator>(join));
+        opRef.setValue(select);
         context.computeAndSetTypeEnvironmentForOperator(select);
         return true;
     }
 
-    private ILogicalExpression makeCondition(List<LogicalExpressionReference> predList) {
+    private ILogicalExpression makeCondition(List<Mutable<ILogicalExpression>> predList) {
         if (predList.size() > 1) {
             IFunctionInfo finfo = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND);
             return new ScalarFunctionCallExpression(finfo, predList);
         } else {
-            return predList.get(0).getExpression();
+            return predList.get(0).getValue();
         }
     }
 
@@ -103,11 +106,11 @@
         if (f.getFunctionIdentifier() != AlgebricksBuiltinFunctions.EQ) {
             return false;
         }
-        ILogicalExpression e1 = f.getArguments().get(0).getExpression();
+        ILogicalExpression e1 = f.getArguments().get(0).getValue();
         if (e1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
             return false;
         } else {
-            ILogicalExpression e2 = f.getArguments().get(1).getExpression();
+            ILogicalExpression e2 = f.getArguments().get(1).getValue();
             return e2.getExpressionTag() == LogicalExpressionTag.VARIABLE;
         }
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
index b7e650c..5eb7160 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
@@ -17,9 +17,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
@@ -32,49 +33,50 @@
 public class PushAssignDownThroughProductRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
         if (op1.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
             return false;
         }
-        LogicalOperatorReference op2Ref = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getOperator();
+        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
         if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
             return false;
         }
         AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
-        if (join.getCondition().getExpression() != ConstantExpression.TRUE) {
+        if (join.getCondition().getValue() != ConstantExpression.TRUE) {
             return false;
         }
 
         List<LogicalVariable> used = new ArrayList<LogicalVariable>();
         VariableUtilities.getUsedVariables(op1, used);
 
-        LogicalOperatorReference b0Ref = op2.getInputs().get(0);
-        ILogicalOperator b0 = b0Ref.getOperator();
+        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+        ILogicalOperator b0 = b0Ref.getValue();
         List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
         VariableUtilities.getLiveVariables(b0, b0Scm);
         if (b0Scm.containsAll(used)) {
             // push assign on left branch
-            op2Ref.setOperator(b0);
-            b0Ref.setOperator(op1);
-            opRef.setOperator(op2);
+            op2Ref.setValue(b0);
+            b0Ref.setValue(op1);
+            opRef.setValue(op2);
             return true;
         } else {
-            LogicalOperatorReference b1Ref = op2.getInputs().get(1);
-            ILogicalOperator b1 = b1Ref.getOperator();
+            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+            ILogicalOperator b1 = b1Ref.getValue();
             List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
             VariableUtilities.getLiveVariables(b1, b1Scm);
             if (b1Scm.containsAll(used)) {
                 // push assign on right branch
-                op2Ref.setOperator(b1);
-                b1Ref.setOperator(op1);
-                opRef.setOperator(op2);
+                op2Ref.setValue(b1);
+                b1Ref.setValue(op1);
+                opRef.setValue(op2);
                 return true;
             } else {
                 return false;
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
index 2a71c1b..ef7822f 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
@@ -1,7 +1,10 @@
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -10,16 +13,17 @@
 public class PushDieUpRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
         if (op0.getInputs().size() == 0)
             return false;
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getValue();
 
         if (op1.getInputs().size() == 0)
             return false;
@@ -28,15 +32,15 @@
                 || tag == LogicalOperatorTag.INSERT_DELETE || tag == LogicalOperatorTag.WRITE_RESULT)
             return false;
 
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
         if (op2.getOperatorTag() == LogicalOperatorTag.DIE) {
-            op0.getInputs().get(0).setOperator(op2);
+            op0.getInputs().get(0).setValue(op2);
             op1.getInputs().clear();
-            for (LogicalOperatorReference ref : op2.getInputs())
+            for (Mutable<ILogicalOperator> ref : op2.getInputs())
                 op1.getInputs().add(ref);
             op2.getInputs().clear();
-            op2.getInputs().add(new LogicalOperatorReference(op1));
-            
+            op2.getInputs().add(new MutableObject<ILogicalOperator>(op1));
+
             context.computeAndSetTypeEnvironmentForOperator(op0);
             context.computeAndSetTypeEnvironmentForOperator(op1);
             context.computeAndSetTypeEnvironmentForOperator(op2);
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
index 467e8cb..1b096ea 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
@@ -16,8 +16,11 @@
 
 import java.util.LinkedList;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -35,7 +38,7 @@
 public class PushLimitDownRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
@@ -46,8 +49,8 @@
      */
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.LIMIT) {
             return false;
         }
@@ -56,8 +59,8 @@
             return false;
         }
 
-        LogicalOperatorReference opRef2 = opLim.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        Mutable<ILogicalOperator> opRef2 = opLim.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
 
         if (context.checkAndAddToAlreadyCompared(op, op2)) {
             return false;
@@ -90,25 +93,25 @@
             }
             // we assume pipelineable ops. have only one input
             opRef2 = op2.getInputs().get(0);
-            op2 = (AbstractLogicalOperator) opRef2.getOperator();
+            op2 = (AbstractLogicalOperator) opRef2.getValue();
         } while (true);
 
         LimitOperator clone2 = null;
-        if (opLim.getOffset().getExpression() == null) {
-            clone2 = new LimitOperator(opLim.getMaxObjects().getExpression(), false);
+        if (opLim.getOffset().getValue() == null) {
+            clone2 = new LimitOperator(opLim.getMaxObjects().getValue(), false);
         } else {
             // push limit (max+offset)
             IFunctionInfo finfoAdd = AlgebricksBuiltinFunctions
                     .getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NUMERIC_ADD);
-            ScalarFunctionCallExpression maxPlusOffset = new ScalarFunctionCallExpression(finfoAdd, opLim
-                    .getMaxObjects(), opLim.getOffset());
+            ScalarFunctionCallExpression maxPlusOffset = new ScalarFunctionCallExpression(finfoAdd,
+                    opLim.getMaxObjects(), opLim.getOffset());
             clone2 = new LimitOperator(maxPlusOffset, false);
         }
         clone2.setPhysicalOperator(new StreamLimitPOperator(false));
-        clone2.getInputs().add(new LogicalOperatorReference(op2));
+        clone2.getInputs().add(new MutableObject<ILogicalOperator>(op2));
         clone2.setExecutionMode(op2.getExecutionMode());
         clone2.recomputeSchema();
-        opRef2.setOperator(clone2);
+        opRef2.setValue(clone2);
         context.computeAndSetTypeEnvironmentForOperator(clone2);
         return true;
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
index 72bef8d..b925cfe 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
@@ -17,11 +17,12 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
@@ -40,13 +41,14 @@
 public class PushNestedOrderByUnderPreSortedGroupByRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
             return false;
         }
@@ -59,12 +61,12 @@
         }
         GroupByOperator gby = (GroupByOperator) op;
         ILogicalPlan plan = gby.getNestedPlans().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) plan.getRoots().get(0).getOperator();
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) plan.getRoots().get(0).getValue();
         if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
             return false;
         }
-        LogicalOperatorReference opRef2 = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
         if (op2.getOperatorTag() != LogicalOperatorTag.ORDER) {
             return false;
         }
@@ -78,7 +80,7 @@
             return false;
         }
         // StableSortPOperator sort1 = (StableSortPOperator) pOrder1;
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         if (op3.getOperatorTag() != LogicalOperatorTag.ORDER) {
             return false;
         }
@@ -93,7 +95,7 @@
         // OrderColumn[] sortColumns = new OrderColumn[n2 + n1];
         // System.arraycopy(sort2.getSortColumns(), 0, sortColumns, 0, n2);
         // int k = 0;
-        for (Pair<IOrder, LogicalExpressionReference> oe : order1.getOrderExpressions()) {
+        for (Pair<IOrder, Mutable<ILogicalExpression>> oe : order1.getOrderExpressions()) {
             order2.getOrderExpressions().add(oe);
             // sortColumns[n2 + k] = sort1.getSortColumns()[k];
             // ++k;
@@ -101,8 +103,8 @@
         // sort2.setSortColumns(sortColumns);
         sort2.computeDeliveredProperties(order2, null);
         // remove order1
-        ILogicalOperator underOrder1 = order1.getInputs().get(0).getOperator();
-        opRef2.setOperator(underOrder1);
+        ILogicalOperator underOrder1 = order1.getInputs().get(0).getValue();
+        opRef2.setValue(underOrder1);
         return true;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
index f393739..b70d88d 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
@@ -20,11 +20,13 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -47,18 +49,18 @@
 public class PushProjectDownRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.PROJECT) {
             return false;
         }
         ProjectOperator pi = (ProjectOperator) op;
-        LogicalOperatorReference opRef2 = pi.getInputs().get(0);
+        Mutable<ILogicalOperator> opRef2 = pi.getInputs().get(0);
 
         HashSet<LogicalVariable> toPush = new HashSet<LogicalVariable>();
         toPush.addAll(pi.getVariables());
@@ -66,7 +68,7 @@
         Pair<Boolean, Boolean> p = pushThroughOp(toPush, opRef2, op, context);
         boolean smthWasPushed = p.first;
         if (p.second) { // the original projection is redundant
-            opRef.setOperator(op.getInputs().get(0).getOperator());
+            opRef.setValue(op.getInputs().get(0).getValue());
             smthWasPushed = true;
         }
 
@@ -74,10 +76,10 @@
     }
 
     private static Pair<Boolean, Boolean> pushThroughOp(HashSet<LogicalVariable> toPush,
-            LogicalOperatorReference opRef2, ILogicalOperator initialOp, IOptimizationContext context)
+            Mutable<ILogicalOperator> opRef2, ILogicalOperator initialOp, IOptimizationContext context)
             throws AlgebricksException {
         List<LogicalVariable> initProjectList = new ArrayList<LogicalVariable>(toPush);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
         do {
             if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
                     || op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE
@@ -96,7 +98,7 @@
             toPush.removeAll(producedVars);
             // we assume pipelineable ops. have only one input
             opRef2 = op2.getInputs().get(0);
-            op2 = (AbstractLogicalOperator) opRef2.getOperator();
+            op2 = (AbstractLogicalOperator) opRef2.getValue();
         } while (true);
 
         LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
@@ -112,8 +114,8 @@
         if (!canCommuteProjection && op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
             boolean gbyChanged = false;
             GroupByOperator gby = (GroupByOperator) op2;
-            List<Pair<LogicalVariable, LogicalExpressionReference>> newDecorList = new ArrayList<Pair<LogicalVariable, LogicalExpressionReference>>();
-            for (Pair<LogicalVariable, LogicalExpressionReference> p : gby.getDecorList()) {
+            List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+            for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
                 LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
                 if (!toPush.contains(decorVar)) {
                     used2.remove(decorVar);
@@ -139,7 +141,7 @@
         }
 
         boolean smthWasPushed = false;
-        for (LogicalOperatorReference c : op2.getInputs()) {
+        for (Mutable<ILogicalOperator> c : op2.getInputs()) {
             if (pushNeededProjections(toPush, c, context, initialOp)) {
                 smthWasPushed = true;
             }
@@ -147,7 +149,7 @@
         if (op2.hasNestedPlans()) {
             AbstractOperatorWithNestedPlans n = (AbstractOperatorWithNestedPlans) op2;
             for (ILogicalPlan p : n.getNestedPlans()) {
-                for (LogicalOperatorReference r : p.getRoots()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
                     if (pushNeededProjections(toPush, r, context, initialOp)) {
                         smthWasPushed = true;
                     }
@@ -158,10 +160,10 @@
     }
 
     // It does not try to push above another Projection.
-    private static boolean pushNeededProjections(HashSet<LogicalVariable> toPush, LogicalOperatorReference opRef,
+    private static boolean pushNeededProjections(HashSet<LogicalVariable> toPush, Mutable<ILogicalOperator> opRef,
             IOptimizationContext context, ILogicalOperator initialOp) throws AlgebricksException {
         HashSet<LogicalVariable> allP = new HashSet<LogicalVariable>();
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         VariableUtilities.getLiveVariables(op, allP);
 
         HashSet<LogicalVariable> toProject = new HashSet<LogicalVariable>();
@@ -185,12 +187,12 @@
 
     // It does not try to push above another Projection.
     private static boolean pushAllProjectionsOnTopOf(Collection<LogicalVariable> toPush,
-            LogicalOperatorReference opRef, IOptimizationContext context, ILogicalOperator initialOp)
+            Mutable<ILogicalOperator> opRef, IOptimizationContext context, ILogicalOperator initialOp)
             throws AlgebricksException {
         if (toPush.isEmpty()) {
             return false;
         }
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
 
         if (context.checkAndAddToAlreadyCompared(initialOp, op)) {
             return false;
@@ -198,8 +200,8 @@
 
         switch (op.getOperatorTag()) {
             case EXCHANGE: {
-                opRef = opRef.getOperator().getInputs().get(0);
-                op = (AbstractLogicalOperator) opRef.getOperator();
+                opRef = opRef.getValue().getInputs().get(0);
+                op = (AbstractLogicalOperator) opRef.getValue();
                 break;
             }
             case PROJECT: {
@@ -208,8 +210,8 @@
         }
 
         ProjectOperator pi2 = new ProjectOperator(new ArrayList<LogicalVariable>(toPush));
-        pi2.getInputs().add(new LogicalOperatorReference(op));
-        opRef.setOperator(pi2);
+        pi2.getInputs().add(new MutableObject<ILogicalOperator>(op));
+        opRef.setValue(pi2);
         pi2.setExecutionMode(op.getExecutionMode());
         context.computeAndSetTypeEnvironmentForOperator(pi2);
         return true;
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
index 6358a7d..4e14f9a 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
@@ -1,7 +1,9 @@
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
@@ -12,22 +14,23 @@
 public class PushProjectIntoDataSourceScanRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getInputs().size() <= 0)
             return false;
-        AbstractLogicalOperator project = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
+        AbstractLogicalOperator project = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         if (project.getOperatorTag() != LogicalOperatorTag.PROJECT)
             return false;
-        AbstractLogicalOperator exchange = (AbstractLogicalOperator) project.getInputs().get(0).getOperator();
+        AbstractLogicalOperator exchange = (AbstractLogicalOperator) project.getInputs().get(0).getValue();
         if (exchange.getOperatorTag() != LogicalOperatorTag.EXCHANGE)
             return false;
-        AbstractLogicalOperator inputOp = (AbstractLogicalOperator) exchange.getInputs().get(0).getOperator();
+        AbstractLogicalOperator inputOp = (AbstractLogicalOperator) exchange.getInputs().get(0).getValue();
         if (inputOp.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN)
             return false;
         DataSourceScanOperator scanOp = (DataSourceScanOperator) inputOp;
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
index 972d257..fedcad9 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
@@ -17,8 +17,10 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -31,19 +33,19 @@
 public class PushSelectDownRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
             return false;
         }
 
-        LogicalOperatorReference opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
 
         if (context.checkAndAddToAlreadyCompared(op, op2)) {
             return false;
@@ -63,27 +65,27 @@
         }
     }
 
-    private static boolean propagateSelectionRec(LogicalOperatorReference sigmaRef, LogicalOperatorReference opRef2)
+    private static boolean propagateSelectionRec(Mutable<ILogicalOperator> sigmaRef, Mutable<ILogicalOperator> opRef2)
             throws AlgebricksException {
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
         if (op2.getInputs().size() != 1 || op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
             return false;
         }
 
-        SelectOperator sigma = (SelectOperator) sigmaRef.getOperator();
+        SelectOperator sigma = (SelectOperator) sigmaRef.getValue();
         LinkedList<LogicalVariable> usedInSigma = new LinkedList<LogicalVariable>();
-        sigma.getCondition().getExpression().getUsedVariables(usedInSigma);
+        sigma.getCondition().getValue().getUsedVariables(usedInSigma);
 
         LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
         VariableUtilities.getProducedVariables(op2, produced2);
         if (OperatorPropertiesUtil.disjoint(produced2, usedInSigma)) {
             // just swap
-            opRef2.setOperator(sigma);
-            sigmaRef.setOperator(op2);
-            List<LogicalOperatorReference> sigmaInpList = sigma.getInputs();
+            opRef2.setValue(sigma);
+            sigmaRef.setValue(op2);
+            List<Mutable<ILogicalOperator>> sigmaInpList = sigma.getInputs();
             sigmaInpList.clear();
             sigmaInpList.addAll(op2.getInputs());
-            List<LogicalOperatorReference> op2InpList = op2.getInputs();
+            List<Mutable<ILogicalOperator>> op2InpList = op2.getInputs();
             op2InpList.clear();
             op2InpList.add(opRef2);
             propagateSelectionRec(opRef2, sigma.getInputs().get(0));
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
index eea4978..315abdb 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
@@ -22,12 +22,13 @@
 import java.util.List;
 import java.util.ListIterator;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -44,12 +45,13 @@
 public class PushSelectIntoJoinRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         Collection<LogicalVariable> joinLiveVarsLeft = new HashSet<LogicalVariable>();
         Collection<LogicalVariable> joinLiveVarsRight = new HashSet<LogicalVariable>();
         Collection<LogicalVariable> liveInOpsToPushLeft = new HashSet<LogicalVariable>();
@@ -61,19 +63,19 @@
         Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
         Collection<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
 
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
             return false;
         }
         SelectOperator select = (SelectOperator) op;
-        LogicalOperatorReference opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator son = (AbstractLogicalOperator) opRef2.getOperator();
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator son = (AbstractLogicalOperator) opRef2.getValue();
         AbstractLogicalOperator op2 = son;
         boolean needToPushOps = false;
         while (son.isMap()) {
             needToPushOps = true;
-            LogicalOperatorReference opRefLink = son.getInputs().get(0);
-            son = (AbstractLogicalOperator) opRefLink.getOperator();
+            Mutable<ILogicalOperator> opRefLink = son.getInputs().get(0);
+            son = (AbstractLogicalOperator) opRefLink.getValue();
         }
 
         if (son.getOperatorTag() != LogicalOperatorTag.INNERJOIN
@@ -83,15 +85,15 @@
         boolean isLoj = son.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN;
         AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) son;
 
-        LogicalOperatorReference joinBranchLeftRef = join.getInputs().get(0);
-        LogicalOperatorReference joinBranchRightRef = join.getInputs().get(1);
+        Mutable<ILogicalOperator> joinBranchLeftRef = join.getInputs().get(0);
+        Mutable<ILogicalOperator> joinBranchRightRef = join.getInputs().get(1);
 
         if (needToPushOps) {
-            ILogicalOperator joinBranchLeft = joinBranchLeftRef.getOperator();
-            ILogicalOperator joinBranchRight = joinBranchRightRef.getOperator();
+            ILogicalOperator joinBranchLeft = joinBranchLeftRef.getValue();
+            ILogicalOperator joinBranchRight = joinBranchRightRef.getValue();
             VariableUtilities.getLiveVariables(joinBranchLeft, joinLiveVarsLeft);
             VariableUtilities.getLiveVariables(joinBranchRight, joinLiveVarsRight);
-            LogicalOperatorReference opIterRef = opRef2;
+            Mutable<ILogicalOperator> opIterRef = opRef2;
             ILogicalOperator opIter = op2;
             while (opIter != join) {
                 LogicalOperatorTag tag = ((AbstractLogicalOperator) opIter).getOperatorTag();
@@ -111,7 +113,7 @@
                     }
                 }
                 opIterRef = opIter.getInputs().get(0);
-                opIter = opIterRef.getOperator();
+                opIter = opIterRef.getValue();
             }
             if (isLoj && pushedOnLeft.isEmpty()) {
                 return false;
@@ -121,11 +123,11 @@
         boolean intersectsAllBranches = true;
         boolean[] intersectsBranch = new boolean[join.getInputs().size()];
         LinkedList<LogicalVariable> selectVars = new LinkedList<LogicalVariable>();
-        select.getCondition().getExpression().getUsedVariables(selectVars);
+        select.getCondition().getValue().getUsedVariables(selectVars);
         int i = 0;
-        for (LogicalOperatorReference branch : join.getInputs()) {
+        for (Mutable<ILogicalOperator> branch : join.getInputs()) {
             LinkedList<LogicalVariable> branchVars = new LinkedList<LogicalVariable>();
-            VariableUtilities.getLiveVariables(branch.getOperator(), branchVars);
+            VariableUtilities.getLiveVariables(branch.getValue(), branchVars);
             if (i == 0) {
                 branchVars.addAll(liveInOpsToPushLeft);
             } else {
@@ -148,10 +150,10 @@
             }
             addCondToJoin(select, join);
         } else { // push down
-            Iterator<LogicalOperatorReference> branchIter = join.getInputs().iterator();
+            Iterator<Mutable<ILogicalOperator>> branchIter = join.getInputs().iterator();
 
             for (int j = 0; j < intersectsBranch.length; j++) {
-                LogicalOperatorReference branch = branchIter.next();
+                Mutable<ILogicalOperator> branch = branchIter.next();
                 boolean inter = intersectsBranch[j];
                 if (inter) {
                     if (needToPushOps) {
@@ -174,36 +176,36 @@
         }
         ILogicalOperator top = join;
         for (ILogicalOperator npOp : notPushedStack) {
-            List<LogicalOperatorReference> npInpList = npOp.getInputs();
+            List<Mutable<ILogicalOperator>> npInpList = npOp.getInputs();
             npInpList.clear();
-            npInpList.add(new LogicalOperatorReference(top));
+            npInpList.add(new MutableObject<ILogicalOperator>(top));
             context.computeAndSetTypeEnvironmentForOperator(npOp);
             top = npOp;
         }
-        opRef.setOperator(top);
+        opRef.setValue(top);
         return true;
 
     }
 
-    private void pushOps(List<ILogicalOperator> opList, LogicalOperatorReference joinBranch,
+    private void pushOps(List<ILogicalOperator> opList, Mutable<ILogicalOperator> joinBranch,
             IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator topOp = joinBranch.getOperator();
+        ILogicalOperator topOp = joinBranch.getValue();
         ListIterator<ILogicalOperator> iter = opList.listIterator(opList.size());
         while (iter.hasPrevious()) {
             ILogicalOperator op = iter.previous();
-            List<LogicalOperatorReference> opInpList = op.getInputs();
+            List<Mutable<ILogicalOperator>> opInpList = op.getInputs();
             opInpList.clear();
-            opInpList.add(new LogicalOperatorReference(topOp));
+            opInpList.add(new MutableObject<ILogicalOperator>(topOp));
             topOp = op;
             context.computeAndSetTypeEnvironmentForOperator(op);
         }
-        joinBranch.setOperator(topOp);
+        joinBranch.setValue(topOp);
     }
 
     private static void addCondToJoin(SelectOperator select, AbstractBinaryJoinOperator join) {
-        ILogicalExpression cond = join.getCondition().getExpression();
+        ILogicalExpression cond = join.getCondition().getValue();
         if (OperatorPropertiesUtil.isAlwaysTrueCond(cond)) { // the join was a product
-            join.getCondition().setExpression(select.getCondition().getExpression());
+            join.getCondition().setValue(select.getCondition().getValue());
         } else {
             boolean bAddedToConj = false;
             if (cond.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
@@ -213,25 +215,25 @@
                             AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
                     newCond.getArguments().add(select.getCondition());
                     newCond.getArguments().addAll(fcond.getArguments());
-                    join.getCondition().setExpression(newCond);
+                    join.getCondition().setValue(newCond);
                     bAddedToConj = true;
                 }
             }
             if (!bAddedToConj) {
-                AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(AlgebricksBuiltinFunctions
-                        .getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND), select.getCondition(),
-                        new LogicalExpressionReference(join.getCondition().getExpression()));
-                join.getCondition().setExpression(newCond);
+                AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(
+                        AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND),
+                        select.getCondition(), new MutableObject<ILogicalExpression>(join.getCondition().getValue()));
+                join.getCondition().setValue(newCond);
             }
         }
     }
 
-    private static void copySelectToBranch(SelectOperator select, LogicalOperatorReference branch,
+    private static void copySelectToBranch(SelectOperator select, Mutable<ILogicalOperator> branch,
             IOptimizationContext context) throws AlgebricksException {
         ILogicalOperator newSelect = new SelectOperator(select.getCondition());
-        LogicalOperatorReference newRef = new LogicalOperatorReference(branch.getOperator());
+        Mutable<ILogicalOperator> newRef = new MutableObject<ILogicalOperator>(branch.getValue());
         newSelect.getInputs().add(newRef);
-        branch.setOperator(newSelect);
+        branch.setValue(newSelect);
         context.computeAndSetTypeEnvironmentForOperator(newSelect);
     }
 }
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
index 32ff5c5..25855e5 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
@@ -19,10 +19,11 @@
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
@@ -36,13 +37,14 @@
 public class PushSubplanWithAggregateDownThroughProductRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
         if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
             return false;
         }
@@ -54,44 +56,44 @@
         if (p.getRoots().size() != 1) {
             return false;
         }
-        LogicalOperatorReference r = p.getRoots().get(0);
-        if (((AbstractLogicalOperator) r.getOperator()).getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+        Mutable<ILogicalOperator> r = p.getRoots().get(0);
+        if (((AbstractLogicalOperator) r.getValue()).getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
             return false;
         }
 
         Set<LogicalVariable> free = new HashSet<LogicalVariable>();
         OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
 
-        LogicalOperatorReference op2Ref = op1.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getOperator();
+        Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
         if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
             return false;
         }
         AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
-        if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getExpression())) {
+        if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
             return false;
         }
 
-        LogicalOperatorReference b0Ref = op2.getInputs().get(0);
-        ILogicalOperator b0 = b0Ref.getOperator();
+        Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+        ILogicalOperator b0 = b0Ref.getValue();
         List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
         VariableUtilities.getLiveVariables(b0, b0Scm);
         if (b0Scm.containsAll(free)) {
             // push subplan on left branch
-            op2Ref.setOperator(b0);
-            b0Ref.setOperator(op1);
-            opRef.setOperator(op2);
+            op2Ref.setValue(b0);
+            b0Ref.setValue(op1);
+            opRef.setValue(op2);
             return true;
         } else {
-            LogicalOperatorReference b1Ref = op2.getInputs().get(1);
-            ILogicalOperator b1 = b1Ref.getOperator();
+            Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+            ILogicalOperator b1 = b1Ref.getValue();
             List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
             VariableUtilities.getLiveVariables(b1, b1Scm);
             if (b1Scm.containsAll(free)) {
                 // push subplan on right branch
-                op2Ref.setOperator(b1);
-                b1Ref.setOperator(op1);
-                opRef.setOperator(op2);
+                op2Ref.setValue(b1);
+                b1Ref.setValue(op1);
+                opRef.setValue(op2);
                 return true;
             } else {
                 return false;
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
index 55e83ae..f06a2ac 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
@@ -1,8 +1,10 @@
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
@@ -11,13 +13,14 @@
 public class ReinferAllTypesRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getOperator())) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
             return false;
         }
         typeOpRec(opRef, context);
@@ -25,14 +28,14 @@
     }
 
     private void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
-        for (LogicalOperatorReference r : p.getRoots()) {
+        for (Mutable<ILogicalOperator> r : p.getRoots()) {
             typeOpRec(r, context);
         }
     }
 
-    private void typeOpRec(LogicalOperatorReference r, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getOperator();
-        for (LogicalOperatorReference i : op.getInputs()) {
+    private void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
             typeOpRec(i, context);
         }
         if (op.hasNestedPlans()) {
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
index e11def8..cdc6086 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
@@ -17,8 +17,10 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -42,33 +44,33 @@
 public class RemoveRedundantProjectionRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
         if (op1.getOperatorTag() == LogicalOperatorTag.PROJECT) {
-            LogicalOperatorReference opRef2 = op1.getInputs().get(0);
-            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+            Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
             if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
                 return false;
             }
             ProjectOperator pi2 = (ProjectOperator) op2;
-            opRef2.setOperator(pi2.getInputs().get(0).getOperator());
+            opRef2.setValue(pi2.getInputs().get(0).getValue());
         } else {
             if (op1.getInputs().size() <= 0)
                 return false;
-            LogicalOperatorReference opRef2 = op1.getInputs().get(0);
-            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+            Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+            AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
             if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
                 return false;
             }
             if (op2.getInputs().size() <= 0)
                 return false;
-            LogicalOperatorReference opRef3 = op2.getInputs().get(0);
-            AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getOperator();
+            Mutable<ILogicalOperator> opRef3 = op2.getInputs().get(0);
+            AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
 
             List<LogicalVariable> liveVars2 = new ArrayList<LogicalVariable>();
             List<LogicalVariable> liveVars3 = new ArrayList<LogicalVariable>();
@@ -78,7 +80,7 @@
 
             if (!VariableUtilities.varListEqualUnordered(liveVars2, liveVars3))
                 return false;
-            opRef2.setOperator(op3);
+            opRef2.setValue(op3);
         }
 
         return true;
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
index 6d6bc70..ff1a102 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -20,10 +20,12 @@
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -37,17 +39,17 @@
 public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getOperator())) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        if (context.checkIfInDontApplySet(this, opRef.getValue())) {
             return false;
         }
         Set<LogicalVariable> toRemove = new HashSet<LogicalVariable>();
-        collectUnusedAssignedVars((AbstractLogicalOperator) opRef.getOperator(), toRemove, true, context);
+        collectUnusedAssignedVars((AbstractLogicalOperator) opRef.getValue(), toRemove, true, context);
         boolean smthToRemove = !toRemove.isEmpty();
         if (smthToRemove) {
             removeUnusedAssigns(opRef, toRemove, context);
@@ -55,19 +57,19 @@
         return smthToRemove;
     }
 
-    private void removeUnusedAssigns(LogicalOperatorReference opRef, Set<LogicalVariable> toRemove,
+    private void removeUnusedAssigns(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> toRemove,
             IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         while (removeFromAssigns(op, toRemove, context) == 0) {
             if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
                 break;
             }
-            op = (AbstractLogicalOperator) op.getInputs().get(0).getOperator();
-            opRef.setOperator(op);
+            op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+            opRef.setValue(op);
         }
-        Iterator<LogicalOperatorReference> childIter = op.getInputs().iterator();
+        Iterator<Mutable<ILogicalOperator>> childIter = op.getInputs().iterator();
         while (childIter.hasNext()) {
-            LogicalOperatorReference cRef = childIter.next();
+            Mutable<ILogicalOperator> cRef = childIter.next();
             removeUnusedAssigns(cRef, toRemove, context);
         }
         if (op.hasNestedPlans()) {
@@ -75,7 +77,7 @@
             Iterator<ILogicalPlan> planIter = opWithNest.getNestedPlans().iterator();
             while (planIter.hasNext()) {
                 ILogicalPlan p = planIter.next();
-                for (LogicalOperatorReference r : p.getRoots()) {
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
                     removeUnusedAssigns(r, toRemove, context);
                 }
             }
@@ -101,10 +103,10 @@
     }
 
     private boolean removeUnusedVarsAndExprs(Set<LogicalVariable> toRemove, List<LogicalVariable> varList,
-            List<LogicalExpressionReference> exprList) {
+            List<Mutable<ILogicalExpression>> exprList) {
         boolean changed = false;
         Iterator<LogicalVariable> varIter = varList.iterator();
-        Iterator<LogicalExpressionReference> exprIter = exprList.iterator();
+        Iterator<Mutable<ILogicalExpression>> exprIter = exprList.iterator();
         while (varIter.hasNext()) {
             LogicalVariable v = varIter.next();
             exprIter.next();
@@ -122,14 +124,14 @@
         if (!first) {
             context.addToDontApplySet(this, op);
         }
-        for (LogicalOperatorReference c : op.getInputs()) {
-            collectUnusedAssignedVars((AbstractLogicalOperator) c.getOperator(), toRemove, false, context);
+        for (Mutable<ILogicalOperator> c : op.getInputs()) {
+            collectUnusedAssignedVars((AbstractLogicalOperator) c.getValue(), toRemove, false, context);
         }
         if (op.hasNestedPlans()) {
             AbstractOperatorWithNestedPlans opWithNested = (AbstractOperatorWithNestedPlans) op;
             for (ILogicalPlan plan : opWithNested.getNestedPlans()) {
-                for (LogicalOperatorReference r : plan.getRoots()) {
-                    collectUnusedAssignedVars((AbstractLogicalOperator) r.getOperator(), toRemove, false, context);
+                for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+                    collectUnusedAssignedVars((AbstractLogicalOperator) r.getValue(), toRemove, false, context);
                 }
             }
         }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
index 23d607f..f1f3533 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -3,12 +3,14 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
@@ -16,6 +18,7 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
 import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
@@ -27,9 +30,8 @@
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AggregatePOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.DataSourceScanPOperator;
@@ -66,13 +68,14 @@
 public class SetAlgebricksPhysicalOperatorsRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         // if (context.checkIfInDontApplySet(this, op)) {
         // return false;
         // }
@@ -87,8 +90,8 @@
 
     private static void setPhysicalOperators(ILogicalPlan plan, boolean topLevelOp, IOptimizationContext context)
             throws AlgebricksException {
-        for (LogicalOperatorReference root : plan.getRoots()) {
-            computeDefaultPhysicalOp((AbstractLogicalOperator) root.getOperator(), topLevelOp, context);
+        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+            computeDefaultPhysicalOp((AbstractLogicalOperator) root.getValue(), topLevelOp, context);
         }
     }
 
@@ -144,10 +147,10 @@
                         }
                     }
 
-                    List<Pair<LogicalVariable, LogicalExpressionReference>> gbyList = gby.getGroupByList();
+                    List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
                     List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
-                    for (Pair<LogicalVariable, LogicalExpressionReference> p : gbyList) {
-                        ILogicalExpression expr = p.second.getExpression();
+                    for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
+                        ILogicalExpression expr = p.second.getValue();
                         if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
                             VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
                             columnList.add(varRef.getVariableReference());
@@ -180,8 +183,8 @@
                 }
                 case ORDER: {
                     OrderOperator oo = (OrderOperator) op;
-                    for (Pair<IOrder, LogicalExpressionReference> p : oo.getOrderExpressions()) {
-                        ILogicalExpression e = p.second.getExpression();
+                    for (Pair<IOrder, Mutable<ILogicalExpression>> p : oo.getOrderExpressions()) {
+                        ILogicalExpression e = p.second.getValue();
                         if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                             throw new AlgebricksException("Order expression " + e + " has not been normalized.");
                         }
@@ -284,14 +287,14 @@
                 setPhysicalOperators(p, false, context);
             }
         }
-        for (LogicalOperatorReference opRef : op.getInputs()) {
-            computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getOperator(), topLevelOp, context);
+        for (Mutable<ILogicalOperator> opRef : op.getInputs()) {
+            computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getValue(), topLevelOp, context);
         }
     }
 
-    private static void getKeys(List<LogicalExpressionReference> keyExpressions, List<LogicalVariable> keys) {
-        for (LogicalExpressionReference kExpr : keyExpressions) {
-            ILogicalExpression e = kExpr.getExpression();
+    private static void getKeys(List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
+        for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
+            ILogicalExpression e = kExpr.getValue();
             if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 throw new NotImplementedException();
             }
@@ -299,16 +302,16 @@
         }
     }
 
-    private static LogicalVariable getKeysAndLoad(LogicalExpressionReference payloadExpr,
-            List<LogicalExpressionReference> keyExpressions, List<LogicalVariable> keys) {
+    private static LogicalVariable getKeysAndLoad(Mutable<ILogicalExpression> payloadExpr,
+            List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
         LogicalVariable payload;
-        if (payloadExpr.getExpression().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+        if (payloadExpr.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
             throw new NotImplementedException();
         }
-        payload = ((VariableReferenceExpression) payloadExpr.getExpression()).getVariableReference();
+        payload = ((VariableReferenceExpression) payloadExpr.getValue()).getVariableReference();
 
-        for (LogicalExpressionReference kExpr : keyExpressions) {
-            ILogicalExpression e = kExpr.getExpression();
+        for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
+            ILogicalExpression e = kExpr.getValue();
             if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                 throw new NotImplementedException();
             }
@@ -332,15 +335,15 @@
         }
         IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
                 .getMergeAggregationExpressionFactory();
-        LogicalOperatorReference r0 = p0.getRoots().get(0);
-        AggregateOperator aggOp = (AggregateOperator) r0.getOperator();
-        List<LogicalExpressionReference> aggFuncRefs = aggOp.getExpressions();
+        Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+        AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+        List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
         int n = aggOp.getExpressions().size();
-        List<LogicalExpressionReference> mergeExpressionRefs = new ArrayList<LogicalExpressionReference>();
+        List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
         for (int i = 0; i < n; i++) {
             ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(aggFuncRefs.get(i)
-                    .getExpression(), context);
-            mergeExpressionRefs.add(new LogicalExpressionReference(mergeExpr));
+                    .getValue(), context);
+            mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
         }
         aggOp.setMergeExpressions(mergeExpressionRefs);
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
index 5ecd14e..bdd56b6 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
@@ -14,8 +14,10 @@
  */
 package edu.uci.ics.hyracks.algebricks.rewriter.rules;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
@@ -35,23 +37,23 @@
 public class SetExecutionModeRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         boolean changed = OperatorManipulationUtil.setOperatorMode(op);
         if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED
                 || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) {
             return changed;
         }
         switch (op.getOperatorTag()) {
-            // case DISTINCT:
-            // case AGGREGATE:
-            // case GROUP:
-            // case ORDER:
-            // case INNERJOIN:
-            // case LEFTOUTERJOIN: {
-            // op.setExecutionMode(ExecutionMode.GLOBAL);
-            // return true;
-            // }
+        // case DISTINCT:
+        // case AGGREGATE:
+        // case GROUP:
+        // case ORDER:
+        // case INNERJOIN:
+        // case LEFTOUTERJOIN: {
+        // op.setExecutionMode(ExecutionMode.GLOBAL);
+        // return true;
+        // }
 
             case PARTITIONINGSPLIT: {
                 throw new NotImplementedException();
@@ -64,7 +66,7 @@
     }
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
index 25f21a9..fb7a49f 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
@@ -16,9 +16,12 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -31,34 +34,36 @@
 public class SimpleUnnestToProductRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
         if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
             return false;
         }
 
-        LogicalOperatorReference opRef2 = op.getInputs().get(0);
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getOperator();
+        Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
 
         if (!(op2 instanceof AbstractScanOperator) && !descOrSelfIsSourceScan(op2)) {
             return false;
         }
-        InnerJoinOperator product = new InnerJoinOperator(new LogicalExpressionReference(ConstantExpression.TRUE));
+        InnerJoinOperator product = new InnerJoinOperator(
+                new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
 
         EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
         context.computeAndSetTypeEnvironmentForOperator(ets);
-        LogicalOperatorReference emptySrc = new LogicalOperatorReference(ets);
-        List<LogicalOperatorReference> opInpList = op.getInputs();
+        Mutable<ILogicalOperator> emptySrc = new MutableObject<ILogicalOperator>(ets);
+        List<Mutable<ILogicalOperator>> opInpList = op.getInputs();
         opInpList.clear();
         opInpList.add(emptySrc);
         product.getInputs().add(opRef2); // outer branch
-        product.getInputs().add(new LogicalOperatorReference(op));
-        opRef.setOperator(product); // plug the product in the plan
+        product.getInputs().add(new MutableObject<ILogicalOperator>(op));
+        opRef.setValue(product); // plug the product in the plan
         context.computeAndSetTypeEnvironmentForOperator(product);
         return true;
     }
@@ -67,8 +72,8 @@
         if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
             return true;
         }
-        for (LogicalOperatorReference cRef : op2.getInputs()) {
-            AbstractLogicalOperator alo = (AbstractLogicalOperator) cRef.getOperator();
+        for (Mutable<ILogicalOperator> cRef : op2.getInputs()) {
+            AbstractLogicalOperator alo = (AbstractLogicalOperator) cRef.getValue();
             if (descOrSelfIsSourceScan(alo)) {
                 return true;
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
index bc86d53..7c31fb9 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
@@ -18,10 +18,12 @@
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
@@ -54,13 +56,14 @@
 public class SubplanOutOfGroupRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
         return false;
     }
 
     @Override
-    public boolean rewritePost(LogicalOperatorReference opRef, IOptimizationContext context) throws AlgebricksException {
-        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getOperator();
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
         if (op0.getOperatorTag() != LogicalOperatorTag.GROUP) {
             return false;
         }
@@ -77,18 +80,18 @@
         if (p.getRoots().size() != 1) {
             return false;
         }
-        LogicalOperatorReference op1Ref = p.getRoots().get(0);
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op1Ref.getOperator();
+        Mutable<ILogicalOperator> op1Ref = p.getRoots().get(0);
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op1Ref.getValue();
         boolean found = false;
         while (op1.getInputs().size() == 1) {
             if (op1.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
                 SubplanOperator subplan = (SubplanOperator) op1;
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) subplan.getInputs().get(0).getOperator();
+                AbstractLogicalOperator op2 = (AbstractLogicalOperator) subplan.getInputs().get(0).getValue();
                 if (OperatorPropertiesUtil.isNullTest(op2)) {
                     if (subplan.getNestedPlans().size() == 1) {
                         ILogicalPlan p1 = subplan.getNestedPlans().get(0);
                         if (p1.getRoots().size() == 1) {
-                            AbstractLogicalOperator r1 = (AbstractLogicalOperator) p1.getRoots().get(0).getOperator();
+                            AbstractLogicalOperator r1 = (AbstractLogicalOperator) p1.getRoots().get(0).getValue();
                             if (r1.getOperatorTag() == LogicalOperatorTag.INNERJOIN
                                     || r1.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
                                 // now, check that it propagates all variables,
@@ -107,20 +110,20 @@
                 }
             }
             op1Ref = op1.getInputs().get(0);
-            op1 = (AbstractLogicalOperator) op1Ref.getOperator();
+            op1 = (AbstractLogicalOperator) op1Ref.getValue();
         }
         if (!found) {
             return false;
         }
 
         ILogicalOperator subplan = op1;
-        ILogicalOperator op2 = op1.getInputs().get(0).getOperator();
-        op1Ref.setOperator(op2);
-        LogicalOperatorReference opUnderRef = gby.getInputs().get(0);
-        ILogicalOperator opUnder = opUnderRef.getOperator();
+        ILogicalOperator op2 = op1.getInputs().get(0).getValue();
+        op1Ref.setValue(op2);
+        Mutable<ILogicalOperator> opUnderRef = gby.getInputs().get(0);
+        ILogicalOperator opUnder = opUnderRef.getValue();
         subplan.getInputs().clear();
-        subplan.getInputs().add(new LogicalOperatorReference(opUnder));
-        opUnderRef.setOperator(subplan);
+        subplan.getInputs().add(new MutableObject<ILogicalOperator>(opUnder));
+        opUnderRef.setValue(subplan);
 
         return true;
     }
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
index 711e859..1714a24 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
@@ -19,27 +19,27 @@
 import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalPropertiesVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractJoinPOperator.JoinPartitioningType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HybridHashJoinPOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryHashJoinPOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NLJoinPOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractJoinPOperator.JoinPartitioningType;
 import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
 import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
 import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
@@ -58,10 +58,10 @@
             throws AlgebricksException {
         List<LogicalVariable> sideLeft = new LinkedList<LogicalVariable>();
         List<LogicalVariable> sideRight = new LinkedList<LogicalVariable>();
-        List<LogicalVariable> varsLeft = op.getInputs().get(0).getOperator().getSchema();
-        List<LogicalVariable> varsRight = op.getInputs().get(1).getOperator().getSchema();
-        if (isHashJoinCondition(op.getCondition().getExpression(), varsLeft, varsRight, sideLeft, sideRight)) {
-            BroadcastSide side = getBroadcastJoinSide(op.getCondition().getExpression(), varsLeft, varsRight);
+        List<LogicalVariable> varsLeft = op.getInputs().get(0).getValue().getSchema();
+        List<LogicalVariable> varsRight = op.getInputs().get(1).getValue().getSchema();
+        if (isHashJoinCondition(op.getCondition().getValue(), varsLeft, varsRight, sideLeft, sideRight)) {
+            BroadcastSide side = getBroadcastJoinSide(op.getCondition().getValue(), varsLeft, varsRight);
             if (side == null) {
                 setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
             } else {
@@ -70,11 +70,11 @@
                         setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideLeft, sideRight, context);
                         break;
                     case LEFT:
-                        LogicalOperatorReference opRef0 = op.getInputs().get(0);
-                        LogicalOperatorReference opRef1 = op.getInputs().get(1);
-                        ILogicalOperator tmp = opRef0.getOperator();
-                        opRef0.setOperator(opRef1.getOperator());
-                        opRef1.setOperator(tmp);
+                        Mutable<ILogicalOperator> opRef0 = op.getInputs().get(0);
+                        Mutable<ILogicalOperator> opRef1 = op.getInputs().get(1);
+                        ILogicalOperator tmp = opRef0.getValue();
+                        opRef0.setValue(opRef1.getValue());
+                        opRef1.setValue(tmp);
                         setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideRight, sideLeft, context);
                         break;
                     default:
@@ -108,7 +108,7 @@
 
     private static void hybridToInMemHashJoin(AbstractBinaryJoinOperator op, IOptimizationContext context)
             throws AlgebricksException {
-        ILogicalOperator opBuild = op.getInputs().get(1).getOperator();
+        ILogicalOperator opBuild = op.getInputs().get(1).getValue();
         LogicalPropertiesVisitor.computeLogicalPropertiesDFS(opBuild, context);
         ILogicalPropertiesVector v = context.getLogicalPropertiesVector(opBuild);
         AlgebricksConfig.ALGEBRICKS_LOGGER.fine("// HybridHashJoin inner branch -- Logical properties for " + opBuild
@@ -135,8 +135,8 @@
                 AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
                 FunctionIdentifier fi = fexp.getFunctionIdentifier();
                 if (fi == AlgebricksBuiltinFunctions.AND) {
-                    for (LogicalExpressionReference a : fexp.getArguments()) {
-                        if (!isHashJoinCondition(a.getExpression(), inLeftAll, inRightAll, outLeftFields,
+                    for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
+                        if (!isHashJoinCondition(a.getValue(), inLeftAll, inRightAll, outLeftFields,
                                 outRightFields)) {
                             return false;
                         }
@@ -147,8 +147,8 @@
                     if (ck != ComparisonKind.EQ) {
                         return false;
                     }
-                    ILogicalExpression opLeft = fexp.getArguments().get(0).getExpression();
-                    ILogicalExpression opRight = fexp.getArguments().get(1).getExpression();
+                    ILogicalExpression opLeft = fexp.getArguments().get(0).getValue();
+                    ILogicalExpression opRight = fexp.getArguments().get(1).getValue();
                     if (opLeft.getExpressionTag() != LogicalExpressionTag.VARIABLE
                             || opRight.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
                         return false;
@@ -204,7 +204,7 @@
                 return null;
         }
         ArrayList<LogicalVariable> vars = new ArrayList<LogicalVariable>();
-        fexp.getArguments().get(i).getExpression().getUsedVariables(vars);
+        fexp.getArguments().get(i).getValue().getUsedVariables(vars);
         if (varsLeft.containsAll(vars)) {
             return BroadcastSide.LEFT;
         } else if (varsRight.containsAll(vars)) {
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
index 2af9fe4..795c767 100644
--- a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
@@ -3,10 +3,11 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import org.apache.commons.lang3.mutable.Mutable;
+
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorReference;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
@@ -27,20 +28,20 @@
     private static void computeFDsAndEqClassesWithVisitorRec(AbstractLogicalOperator op, IOptimizationContext ctx,
             FDsAndEquivClassesVisitor visitor, Set<ILogicalOperator> visitSet) throws AlgebricksException {
         visitSet.add(op);
-        for (LogicalOperatorReference i : op.getInputs()) {
-            computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) i.getOperator(), ctx, visitor, visitSet);
+        for (Mutable<ILogicalOperator> i : op.getInputs()) {
+            computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) i.getValue(), ctx, visitor, visitSet);
         }
         if (op.hasNestedPlans()) {
             for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
-                for (LogicalOperatorReference r : p.getRoots()) {
-                    AbstractLogicalOperator rootOp = (AbstractLogicalOperator) r.getOperator();
+                for (Mutable<ILogicalOperator> r : p.getRoots()) {
+                    AbstractLogicalOperator rootOp = (AbstractLogicalOperator) r.getValue();
                     computeFDsAndEqClassesWithVisitorRec(rootOp, ctx, visitor, visitSet);
                 }
             }
         }
         if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
             NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
-            ILogicalOperator source = nts.getDataSourceReference().getOperator().getInputs().get(0).getOperator();
+            ILogicalOperator source = nts.getDataSourceReference().getValue().getInputs().get(0).getValue();
             if (!visitSet.contains(source)) {
                 computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) source, ctx, visitor, visitSet);
             }
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
index d6c2165..922b0fd 100644
--- a/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
@@ -17,7 +17,7 @@
 import java.util.EnumSet;
 
 import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
-import edu.uci.ics.hyracks.api.client.HyracksLocalConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.job.JobFlag;
 import edu.uci.ics.hyracks.api.job.JobId;
@@ -32,9 +32,8 @@
     public static final String NC1_ID = "nc1";
     public static final String NC2_ID = "nc2";
 
-    public static final int DEFAULT_HYRACKS_CC_PORT = 1099;
-
-    public static final int TEST_HYRACKS_CC_PORT = 4322;
+    public static final int TEST_HYRACKS_CC_CLUSTER_NET_PORT = 4322;
+    public static final int TEST_HYRACKS_CC_CLIENT_NET_PORT = 4321;
 
     private static ClusterControllerService cc;
     private static NodeControllerService nc1;
@@ -43,14 +42,18 @@
 
     public static void init() throws Exception {
         CCConfig ccConfig = new CCConfig();
-        ccConfig.port = TEST_HYRACKS_CC_PORT;
+        ccConfig.clientNetIpAddress = "127.0.0.1";
+        ccConfig.clientNetPort = TEST_HYRACKS_CC_CLIENT_NET_PORT;
+        ccConfig.clusterNetIpAddress = "127.0.0.1";
+        ccConfig.clusterNetPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT;
         // ccConfig.useJOL = true;
         cc = new ClusterControllerService(ccConfig);
         cc.start();
 
         NCConfig ncConfig1 = new NCConfig();
         ncConfig1.ccHost = "localhost";
-        ncConfig1.ccPort = TEST_HYRACKS_CC_PORT;
+        ncConfig1.ccPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT;
+        ncConfig1.clusterNetIPAddress = "127.0.0.1";
         ncConfig1.dataIPAddress = "127.0.0.1";
         ncConfig1.nodeId = NC1_ID;
         nc1 = new NodeControllerService(ncConfig1);
@@ -58,13 +61,14 @@
 
         NCConfig ncConfig2 = new NCConfig();
         ncConfig2.ccHost = "localhost";
-        ncConfig2.ccPort = TEST_HYRACKS_CC_PORT;
+        ncConfig2.ccPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT;
+        ncConfig2.clusterNetIPAddress = "127.0.0.1";
         ncConfig2.dataIPAddress = "127.0.0.1";
         ncConfig2.nodeId = NC2_ID;
         nc2 = new NodeControllerService(ncConfig2);
         nc2.start();
 
-        hcc = new HyracksLocalConnection(cc);
+        hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
         hcc.createApplication(AlgebricksConfig.HYRACKS_APP_NAME, null);
     }
 
@@ -77,9 +81,9 @@
     public static void runJob(JobSpecification spec) throws Exception {
         JobId jobId = hcc.createJob(AlgebricksConfig.HYRACKS_APP_NAME, spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
         AlgebricksConfig.ALGEBRICKS_LOGGER.info(spec.toJSON().toString());
-        cc.start(jobId);
+        hcc.start(jobId);
         AlgebricksConfig.ALGEBRICKS_LOGGER.info(jobId.toString());
-        cc.waitForCompletion(jobId);
+        hcc.waitForCompletion(jobId);
     }
 
-}
+}
\ No newline at end of file
diff --git a/hyracks-api/pom.xml b/hyracks-api/pom.xml
index f73c27a..c18995d 100644
--- a/hyracks-api/pom.xml
+++ b/hyracks-api/pom.xml
@@ -1,9 +1,6 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
-  <groupId>edu.uci.ics.hyracks</groupId>
   <artifactId>hyracks-api</artifactId>
-  <version>0.2.0-SNAPSHOT</version>
-
   <parent>
     <groupId>edu.uci.ics.hyracks</groupId>
     <artifactId>hyracks</artifactId>
@@ -45,5 +42,15 @@
   		<type>jar</type>
   		<scope>compile</scope>
   	</dependency>
+  	<dependency>
+  		<groupId>edu.uci.ics.hyracks</groupId>
+  		<artifactId>hyracks-ipc</artifactId>
+  		<version>0.2.0-SNAPSHOT</version>
+  	</dependency>
+  	<dependency>
+  		<groupId>org.apache.commons</groupId>
+  		<artifactId>commons-lang3</artifactId>
+  		<version>3.1</version>
+  	</dependency>
   </dependencies>
 </project>
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java
new file mode 100644
index 0000000..a7f324d
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client;
+
+import java.io.Serializable;
+import java.util.EnumSet;
+
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+
+public class HyracksClientInterfaceFunctions {
+    public enum FunctionId {
+        GET_CLUSTER_CONTROLLER_INFO,
+        CREATE_APPLICATION,
+        START_APPLICATION,
+        DESTROY_APPLICATION,
+        CREATE_JOB,
+        GET_JOB_STATUS,
+        START_JOB,
+        WAIT_FOR_COMPLETION,
+        GET_NODE_CONTROLLERS_INFO
+    }
+
+    public abstract static class Function implements Serializable {
+        private static final long serialVersionUID = 1L;
+
+        public abstract FunctionId getFunctionId();
+    }
+
+    public static class GetClusterControllerInfoFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.GET_CLUSTER_CONTROLLER_INFO;
+        }
+    }
+
+    public static class CreateApplicationFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String appName;
+
+        public CreateApplicationFunction(String appName) {
+            this.appName = appName;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.CREATE_APPLICATION;
+        }
+
+        public String getAppName() {
+            return appName;
+        }
+    }
+
+    public static class StartApplicationFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String appName;
+
+        public StartApplicationFunction(String appName) {
+            this.appName = appName;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.START_APPLICATION;
+        }
+
+        public String getAppName() {
+            return appName;
+        }
+    }
+
+    public static class DestroyApplicationFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String appName;
+
+        public DestroyApplicationFunction(String appName) {
+            this.appName = appName;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.DESTROY_APPLICATION;
+        }
+
+        public String getAppName() {
+            return appName;
+        }
+    }
+
+    public static class CreateJobFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String appName;
+        private final byte[] jobSpec;
+        private final EnumSet<JobFlag> jobFlags;
+
+        public CreateJobFunction(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) {
+            this.appName = appName;
+            this.jobSpec = jobSpec;
+            this.jobFlags = jobFlags;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.CREATE_JOB;
+        }
+
+        public String getAppName() {
+            return appName;
+        }
+
+        public byte[] getJobSpec() {
+            return jobSpec;
+        }
+
+        public EnumSet<JobFlag> getJobFlags() {
+            return jobFlags;
+        }
+    }
+
+    public static class GetJobStatusFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+
+        public GetJobStatusFunction(JobId jobId) {
+            this.jobId = jobId;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.GET_JOB_STATUS;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+    }
+
+    public static class StartJobFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+
+        public StartJobFunction(JobId jobId) {
+            this.jobId = jobId;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.START_JOB;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+    }
+
+    public static class WaitForCompletionFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+
+        public WaitForCompletionFunction(JobId jobId) {
+            this.jobId = jobId;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.WAIT_FOR_COMPLETION;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+    }
+
+    public static class GetNodeControllersInfoFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.GET_NODE_CONTROLLERS_INFO;
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
new file mode 100644
index 0000000..602288c
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client;
+
+import java.util.EnumSet;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.RPCInterface;
+
+public class HyracksClientInterfaceRemoteProxy implements IHyracksClientInterface {
+    private final IIPCHandle ipcHandle;
+
+    private final RPCInterface rpci;
+
+    public HyracksClientInterfaceRemoteProxy(IIPCHandle ipcHandle, RPCInterface rpci) {
+        this.ipcHandle = ipcHandle;
+        this.rpci = rpci;
+    }
+
+    @Override
+    public ClusterControllerInfo getClusterControllerInfo() throws Exception {
+        HyracksClientInterfaceFunctions.GetClusterControllerInfoFunction gccif = new HyracksClientInterfaceFunctions.GetClusterControllerInfoFunction();
+        return (ClusterControllerInfo) rpci.call(ipcHandle, gccif);
+    }
+
+    @Override
+    public void createApplication(String appName) throws Exception {
+        HyracksClientInterfaceFunctions.CreateApplicationFunction caf = new HyracksClientInterfaceFunctions.CreateApplicationFunction(
+                appName);
+        rpci.call(ipcHandle, caf);
+    }
+
+    @Override
+    public void startApplication(String appName) throws Exception {
+        HyracksClientInterfaceFunctions.StartApplicationFunction saf = new HyracksClientInterfaceFunctions.StartApplicationFunction(
+                appName);
+        rpci.call(ipcHandle, saf);
+    }
+
+    @Override
+    public void destroyApplication(String appName) throws Exception {
+        HyracksClientInterfaceFunctions.DestroyApplicationFunction daf = new HyracksClientInterfaceFunctions.DestroyApplicationFunction(
+                appName);
+        rpci.call(ipcHandle, daf);
+    }
+
+    @Override
+    public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
+        HyracksClientInterfaceFunctions.CreateJobFunction cjf = new HyracksClientInterfaceFunctions.CreateJobFunction(
+                appName, jobSpec, jobFlags);
+        return (JobId) rpci.call(ipcHandle, cjf);
+    }
+
+    @Override
+    public JobStatus getJobStatus(JobId jobId) throws Exception {
+        HyracksClientInterfaceFunctions.GetJobStatusFunction gjsf = new HyracksClientInterfaceFunctions.GetJobStatusFunction(
+                jobId);
+        return (JobStatus) rpci.call(ipcHandle, gjsf);
+    }
+
+    @Override
+    public void startJob(JobId jobId) throws Exception {
+        HyracksClientInterfaceFunctions.StartJobFunction sjf = new HyracksClientInterfaceFunctions.StartJobFunction(
+                jobId);
+        rpci.call(ipcHandle, sjf);
+    }
+
+    @Override
+    public void waitForCompletion(JobId jobId) throws Exception {
+        HyracksClientInterfaceFunctions.WaitForCompletionFunction wfcf = new HyracksClientInterfaceFunctions.WaitForCompletionFunction(
+                jobId);
+        rpci.call(ipcHandle, wfcf);
+    }
+
+    @Override
+    public Map<String, NodeControllerInfo> getNodeControllersInfo() throws Exception {
+        HyracksClientInterfaceFunctions.GetNodeControllersInfoFunction gncif = new HyracksClientInterfaceFunctions.GetNodeControllersInfoFunction();
+        return (Map<String, NodeControllerInfo>) rpci.call(ipcHandle, gncif);
+    }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/AbstractHyracksConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
similarity index 73%
rename from hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/AbstractHyracksConnection.java
rename to hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
index 90bbfb1..b474ee1 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/AbstractHyracksConnection.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
@@ -15,6 +15,7 @@
 package edu.uci.ics.hyracks.api.client;
 
 import java.io.File;
+import java.net.InetSocketAddress;
 import java.util.EnumSet;
 import java.util.Map;
 
@@ -30,17 +31,44 @@
 import edu.uci.ics.hyracks.api.job.JobSpecification;
 import edu.uci.ics.hyracks.api.job.JobStatus;
 import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.RPCInterface;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
 
-abstract class AbstractHyracksConnection implements IHyracksClientConnection {
+/**
+ * Connection Class used by a Hyracks Client to interact with a Hyracks Cluster
+ * Controller.
+ * 
+ * @author vinayakb
+ * 
+ */
+public final class HyracksConnection implements IHyracksClientConnection {
     private final String ccHost;
 
+    private final IPCSystem ipc;
+
     private final IHyracksClientInterface hci;
 
     private final ClusterControllerInfo ccInfo;
 
-    public AbstractHyracksConnection(String ccHost, IHyracksClientInterface hci) throws Exception {
+    /**
+     * Constructor to create a connection to the Hyracks Cluster Controller.
+     * 
+     * @param ccHost
+     *            Host name (or IP Address) where the Cluster Controller can be
+     *            reached.
+     * @param ccPort
+     *            Port to reach the Hyracks Cluster Controller at the specified
+     *            host name.
+     * @throws Exception
+     */
+    public HyracksConnection(String ccHost, int ccPort) throws Exception {
         this.ccHost = ccHost;
-        this.hci = hci;
+        RPCInterface rpci = new RPCInterface();
+        ipc = new IPCSystem(new InetSocketAddress(0), rpci);
+        ipc.start();
+        IIPCHandle ccIpchandle = ipc.getHandle(new InetSocketAddress(ccHost, ccPort));
+        this.hci = new HyracksClientInterfaceRemoteProxy(ccIpchandle, rpci);
         ccInfo = hci.getClusterControllerInfo();
     }
 
@@ -82,7 +110,7 @@
 
     @Override
     public void start(JobId jobId) throws Exception {
-        hci.start(jobId);
+        hci.startJob(jobId);
     }
 
     @Override
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksLocalConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksLocalConnection.java
deleted file mode 100644
index ee0b276..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksLocalConnection.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.client;
-
-/**
- * Connection Class used by a Hyracks Client that is colocated in the same VM
- * with the Cluster Controller. Usually, clients must not use this class. This
- * is used internally for testing purposes.
- * 
- * @author vinayakb
- * 
- */
-public final class HyracksLocalConnection extends AbstractHyracksConnection {
-    public HyracksLocalConnection(IHyracksClientInterface hci) throws Exception {
-        super("localhost", hci);
-    }
-}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksRMIConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksRMIConnection.java
deleted file mode 100644
index 65d82a8..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksRMIConnection.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.client;
-
-import java.rmi.NotBoundException;
-import java.rmi.RemoteException;
-import java.rmi.registry.LocateRegistry;
-import java.rmi.registry.Registry;
-
-/**
- * Connection Class used by a Hyracks Client to interact with a Hyracks Cluster
- * Controller using RMI. Usually, such a connection would be used when the CC
- * runs in a separate JVM from the client (The most common case).
- * 
- * @author vinayakb
- * 
- */
-public final class HyracksRMIConnection extends AbstractHyracksConnection {
-    /**
-     * Constructor to create a connection to the Hyracks Cluster Controller.
-     * 
-     * @param host
-     *            Host name (or IP Address) where the Cluster Controller can be
-     *            reached.
-     * @param port
-     *            Port to reach the Hyracks Cluster Controller at the specified
-     *            host name.
-     * @throws Exception
-     */
-    public HyracksRMIConnection(String host, int port) throws Exception {
-        super(host, lookupHCI(host, port));
-    }
-
-    private static IHyracksClientInterface lookupHCI(String host, int port) throws RemoteException, NotBoundException {
-        Registry registry = LocateRegistry.getRegistry(host, port);
-        return (IHyracksClientInterface) registry.lookup(IHyracksClientInterface.class.getName());
-    }
-}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
index 6e1eedc..866d307 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
@@ -14,7 +14,6 @@
  */
 package edu.uci.ics.hyracks.api.client;
 
-import java.rmi.Remote;
 import java.util.EnumSet;
 import java.util.Map;
 
@@ -22,7 +21,7 @@
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.api.job.JobStatus;
 
-public interface IHyracksClientInterface extends Remote {
+public interface IHyracksClientInterface {
     public ClusterControllerInfo getClusterControllerInfo() throws Exception;
 
     public void createApplication(String appName) throws Exception;
@@ -35,7 +34,7 @@
 
     public JobStatus getJobStatus(JobId jobId) throws Exception;
 
-    public void start(JobId jobId) throws Exception;
+    public void startJob(JobId jobId) throws Exception;
 
     public void waitForCompletion(JobId jobId) throws Exception;
 
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
index 266ebe9..abd9c20 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
@@ -18,5 +18,5 @@
 import java.util.Set;
 
 public interface ICCContext {
-    public Map<String, Set<String>> getIPAddressNodeMap();
+    public void getIPAddressNodeMap(Map<String, Set<String>> map) throws Exception;
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java
index c6ca51c..75cc245 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java
@@ -52,6 +52,16 @@
     }
 
     public String toString() {
-        return "ANID:[" + odId + "]:" + id;
+        return "ANID:" + odId + ":" + id;
+    }
+
+    public static ActivityId parse(String str) {
+        if (str.startsWith("ANID:")) {
+            str = str.substring(5);
+            int idIdx = str.lastIndexOf(':');
+            return new ActivityId(OperatorDescriptorId.parse(str.substring(0, idIdx)), Integer.parseInt(str
+                    .substring(idIdx + 1)));
+        }
+        throw new IllegalArgumentException("Unable to parse: " + str);
     }
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
index b858736..8794e09 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
@@ -49,4 +49,12 @@
     public String toString() {
         return "ODID:" + id;
     }
+
+    public static OperatorDescriptorId parse(String str) {
+        if (str.startsWith("ODID:")) {
+            str = str.substring(5);
+            return new OperatorDescriptorId(Integer.parseInt(str));
+        }
+        throw new IllegalArgumentException("Unable to parse: " + str);
+    }
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java
index d4e6972..0fb44c1 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java
@@ -52,6 +52,15 @@
 
     @Override
     public String toString() {
-        return "TAID:[" + taskId + "]:" + attempt;
+        return "TAID:" + taskId + ":" + attempt;
+    }
+
+    public static TaskAttemptId parse(String str) {
+        if (str.startsWith("TAID:")) {
+            str = str.substring(5);
+            int idIdx = str.lastIndexOf(':');
+            return new TaskAttemptId(TaskId.parse(str.substring(0, idIdx)), Integer.parseInt(str.substring(idIdx + 1)));
+        }
+        throw new IllegalArgumentException("Unable to parse: " + str);
     }
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java
index ee63355..7e0b22d 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java
@@ -52,6 +52,15 @@
 
     @Override
     public String toString() {
-        return "TID:[" + activityId + "]:" + partition;
+        return "TID:" + activityId + ":" + partition;
+    }
+
+    public static TaskId parse(String str) {
+        if (str.startsWith("TID:")) {
+            str = str.substring(4);
+            int idIdx = str.lastIndexOf(':');
+            return new TaskId(ActivityId.parse(str.substring(0, idIdx)), Integer.parseInt(str.substring(idIdx + 1)));
+        }
+        throw new IllegalArgumentException("Unable to parse: " + str);
     }
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
index 3d03c32..b04fe86 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
@@ -22,24 +22,28 @@
 
     private final ISerializerDeserializer[] fields;
     private final ITypeTraits[] typeTraits;
-    
+
     // leaving this constructor for backwards-compatibility
     public RecordDescriptor(ISerializerDeserializer[] fields) {
         this.fields = fields;
         this.typeTraits = null;
     }
-    
+
     // temporarily adding constructor to include type traits
     public RecordDescriptor(ISerializerDeserializer[] fields, ITypeTraits[] typeTraits) {
-    	this.fields = fields;
+        this.fields = fields;
         this.typeTraits = typeTraits;
     }
 
+    public int getFieldCount() {
+        return fields.length;
+    }
+
     public ISerializerDeserializer[] getFields() {
         return fields;
     }
-    
+
     public ITypeTraits[] getTypeTraits() {
-    	return typeTraits;
+        return typeTraits;
     }
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
index 930d299..2b6d361 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -33,7 +34,6 @@
 import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
 import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
 import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.util.Pair;
 
 public class JobActivityGraph implements Serializable {
     private static final long serialVersionUID = 1L;
@@ -148,8 +148,8 @@
         Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connEdge = jobSpec
                 .getConnectorOperatorMap().get(cdId);
 
-        OperatorDescriptorId consumerOpId = connEdge.second.first.getOperatorId();
-        int consumerInputIdx = connEdge.second.second;
+        OperatorDescriptorId consumerOpId = connEdge.getRight().getLeft().getOperatorId();
+        int consumerInputIdx = connEdge.getRight().getRight();
 
         for (ActivityId anId : operatorActivityMap.get(consumerOpId)) {
             List<Integer> anInputs = activityInputMap.get(anId);
@@ -168,8 +168,8 @@
         Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connEdge = jobSpec
                 .getConnectorOperatorMap().get(cdId);
 
-        OperatorDescriptorId producerOpId = connEdge.first.first.getOperatorId();
-        int producerInputIdx = connEdge.first.second;
+        OperatorDescriptorId producerOpId = connEdge.getLeft().getLeft().getOperatorId();
+        int producerInputIdx = connEdge.getLeft().getRight();
 
         for (ActivityId anId : operatorActivityMap.get(producerOpId)) {
             List<Integer> anOutputs = activityOutputMap.get(anId);
@@ -209,13 +209,11 @@
     public JSONObject toJSON() throws JSONException {
         JSONObject jplan = new JSONObject();
 
-        jplan.put("type", "plan");
         jplan.put("flags", jobFlags.toString());
 
         JSONArray jans = new JSONArray();
         for (IActivity an : activityNodes.values()) {
             JSONObject jan = new JSONObject();
-            jan.put("type", "activity");
             jan.put("id", an.getActivityId().toString());
             jan.put("java-class", an.getClass().getName());
             jan.put("operator-id", an.getActivityId().getOperatorDescriptorId().toString());
@@ -225,7 +223,6 @@
                 JSONArray jInputs = new JSONArray();
                 for (int i = 0; i < inputs.size(); ++i) {
                     JSONObject jInput = new JSONObject();
-                    jInput.put("type", "activity-input");
                     jInput.put("input-port", i);
                     jInput.put("connector-id", inputs.get(i).getConnectorId().toString());
                     jInputs.put(jInput);
@@ -238,7 +235,6 @@
                 JSONArray jOutputs = new JSONArray();
                 for (int i = 0; i < outputs.size(); ++i) {
                     JSONObject jOutput = new JSONObject();
-                    jOutput.put("type", "activity-output");
                     jOutput.put("output-port", i);
                     jOutput.put("connector-id", outputs.get(i).getConnectorId().toString());
                     jOutputs.put(jOutput);
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
index 23b33f0..9c6b88c 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.tuple.Pair;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -33,7 +34,6 @@
 import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
 import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
 import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.util.Pair;
 
 public class JobSpecification implements Serializable {
     private static final long serialVersionUID = 1L;
@@ -94,10 +94,11 @@
             IOperatorDescriptor consumerOp, int consumerPort) {
         insertIntoIndexedMap(opInputMap, consumerOp.getOperatorId(), consumerPort, conn);
         insertIntoIndexedMap(opOutputMap, producerOp.getOperatorId(), producerPort, conn);
-        connectorOpMap.put(conn.getConnectorId(),
-                new Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>(
-                        new Pair<IOperatorDescriptor, Integer>(producerOp, producerPort),
-                        new Pair<IOperatorDescriptor, Integer>(consumerOp, consumerPort)));
+        connectorOpMap.put(
+                conn.getConnectorId(),
+                Pair.<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> of(
+                        Pair.<IOperatorDescriptor, Integer> of(producerOp, producerPort),
+                        Pair.<IOperatorDescriptor, Integer> of(consumerOp, consumerPort)));
     }
 
     public void setProperty(String name, Serializable value) {
@@ -126,19 +127,19 @@
     public RecordDescriptor getConnectorRecordDescriptor(IConnectorDescriptor conn) {
         Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
                 .getConnectorId());
-        return connInfo.first.first.getOutputRecordDescriptors()[connInfo.first.second];
+        return connInfo.getLeft().getLeft().getOutputRecordDescriptors()[connInfo.getLeft().getRight()];
     }
 
     public IOperatorDescriptor getConsumer(IConnectorDescriptor conn) {
         Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
                 .getConnectorId());
-        return connInfo.second.first;
+        return connInfo.getRight().getLeft();
     }
 
     public int getConsumerInputIndex(IConnectorDescriptor conn) {
         Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
                 .getConnectorId());
-        return connInfo.second.second;
+        return connInfo.getRight().getRight();
     }
 
     public IConnectorDescriptor getInputConnectorDescriptor(IOperatorDescriptor op, int inputIndex) {
@@ -180,13 +181,13 @@
     public IOperatorDescriptor getProducer(IConnectorDescriptor conn) {
         Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
                 .getConnectorId());
-        return connInfo.first.first;
+        return connInfo.getLeft().getLeft();
     }
 
     public int getProducerOutputIndex(IConnectorDescriptor conn) {
         Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
                 .getConnectorId());
-        return connInfo.first.second;
+        return connInfo.getLeft().getRight();
     }
 
     public List<OperatorDescriptorId> getRoots() {
@@ -264,8 +265,6 @@
     public JSONObject toJSON() throws JSONException {
         JSONObject jjob = new JSONObject();
 
-        jjob.put("type", "job");
-
         JSONArray jopArray = new JSONArray();
         for (Map.Entry<OperatorDescriptorId, IOperatorDescriptor> e : opMap.entrySet()) {
             jopArray.put(e.getValue().toJSON());
@@ -277,12 +276,11 @@
             JSONObject conn = new JSONObject();
             Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connection = connectorOpMap
                     .get(e.getKey());
-            conn.put("type", "connector-info");
             if (connection != null) {
-                conn.put("in-operator-id", connection.first.first.getOperatorId().toString());
-                conn.put("in-operator-port", connection.first.second.intValue());
-                conn.put("out-operator-id", connection.second.first.getOperatorId().toString());
-                conn.put("out-operator-port", connection.second.second.intValue());
+                conn.put("in-operator-id", connection.getLeft().getLeft().getOperatorId().toString());
+                conn.put("in-operator-port", connection.getLeft().getRight().intValue());
+                conn.put("out-operator-id", connection.getRight().getLeft().getOperatorId().toString());
+                conn.put("out-operator-port", connection.getRight().getRight().intValue());
             }
             conn.put("connector", e.getValue().toJSON());
             jcArray.put(conn);
diff --git a/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java b/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java
index e1a1fd2..168280b 100644
--- a/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java
+++ b/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java
@@ -1,6 +1,6 @@
 package edu.uci.ics.hyracks.cli.commands;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.cli.Session;
 
@@ -12,7 +12,7 @@
     public ConnectCommand(String hostPortStr) {
         int idx = hostPortStr.indexOf(':');
         host = hostPortStr;
-        port = 1099;
+        port = 1098;
         if (idx != -1) {
             host = hostPortStr.substring(0, idx);
             port = Integer.valueOf(hostPortStr.substring(idx + 1));
@@ -22,7 +22,7 @@
     @Override
     public void run(Session session) throws Exception {
         System.err.println("Connecting to host: " + host + ", port: " + port);
-        IHyracksClientConnection conn = new HyracksRMIConnection(host, port);
+        IHyracksClientConnection conn = new HyracksConnection(host, port);
         session.setConnection(conn);
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/CCClientInterface.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/CCClientInterface.java
deleted file mode 100644
index 70e85a1..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/CCClientInterface.java
+++ /dev/null
@@ -1,68 +0,0 @@
-package edu.uci.ics.hyracks.control.cc;
-
-import java.rmi.RemoteException;
-import java.rmi.server.UnicastRemoteObject;
-import java.util.EnumSet;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.api.client.ClusterControllerInfo;
-import edu.uci.ics.hyracks.api.client.IHyracksClientInterface;
-import edu.uci.ics.hyracks.api.client.NodeControllerInfo;
-import edu.uci.ics.hyracks.api.job.JobFlag;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-
-public class CCClientInterface extends UnicastRemoteObject implements IHyracksClientInterface {
-    private static final long serialVersionUID = 1L;
-
-    private final ClusterControllerService ccs;
-
-    public CCClientInterface(ClusterControllerService ccs) throws RemoteException {
-        this.ccs = ccs;
-    }
-
-    @Override
-    public ClusterControllerInfo getClusterControllerInfo() throws Exception {
-        return ccs.getClusterControllerInfo();
-    }
-
-    @Override
-    public void createApplication(String appName) throws Exception {
-        ccs.createApplication(appName);
-    }
-
-    @Override
-    public void startApplication(String appName) throws Exception {
-        ccs.startApplication(appName);
-    }
-
-    @Override
-    public void destroyApplication(String appName) throws Exception {
-        ccs.destroyApplication(appName);
-    }
-
-    @Override
-    public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
-        return ccs.createJob(appName, jobSpec, jobFlags);
-    }
-
-    @Override
-    public JobStatus getJobStatus(JobId jobId) throws Exception {
-        return ccs.getJobStatus(jobId);
-    }
-
-    @Override
-    public void start(JobId jobId) throws Exception {
-        ccs.start(jobId);
-    }
-
-    @Override
-    public void waitForCompletion(JobId jobId) throws Exception {
-        ccs.waitForCompletion(jobId);
-    }
-
-    @Override
-    public Map<String, NodeControllerInfo> getNodeControllersInfo() throws Exception {
-        return ccs.getNodeControllersInfo();
-    }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
index bcc3c30..2578b88 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
@@ -15,13 +15,10 @@
 package edu.uci.ics.hyracks.control.cc;
 
 import java.io.File;
-import java.rmi.registry.LocateRegistry;
-import java.rmi.registry.Registry;
-import java.util.EnumSet;
+import java.net.InetSocketAddress;
 import java.util.HashMap;
 import java.util.Hashtable;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.Timer;
@@ -32,21 +29,19 @@
 import java.util.logging.Logger;
 
 import edu.uci.ics.hyracks.api.client.ClusterControllerInfo;
-import edu.uci.ics.hyracks.api.client.IHyracksClientInterface;
+import edu.uci.ics.hyracks.api.client.HyracksClientInterfaceFunctions;
 import edu.uci.ics.hyracks.api.client.NodeControllerInfo;
 import edu.uci.ics.hyracks.api.context.ICCContext;
-import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobFlag;
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.api.job.JobStatus;
 import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
-import edu.uci.ics.hyracks.control.cc.job.IJobStatusConditionVariable;
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
 import edu.uci.ics.hyracks.control.cc.web.WebServer;
+import edu.uci.ics.hyracks.control.cc.work.ApplicationCreateWork;
 import edu.uci.ics.hyracks.control.cc.work.ApplicationDestroyWork;
 import edu.uci.ics.hyracks.control.cc.work.ApplicationStartWork;
-import edu.uci.ics.hyracks.control.cc.work.GetJobStatusConditionVariableWork;
+import edu.uci.ics.hyracks.control.cc.work.ApplicationStateChangeWork;
+import edu.uci.ics.hyracks.control.cc.work.GetIpAddressNodeNameMapWork;
 import edu.uci.ics.hyracks.control.cc.work.GetJobStatusWork;
 import edu.uci.ics.hyracks.control.cc.work.GetNodeControllersInfoWork;
 import edu.uci.ics.hyracks.control.cc.work.JobCreateWork;
@@ -61,29 +56,28 @@
 import edu.uci.ics.hyracks.control.cc.work.TaskCompleteWork;
 import edu.uci.ics.hyracks.control.cc.work.TaskFailureWork;
 import edu.uci.ics.hyracks.control.cc.work.UnregisterNodeWork;
+import edu.uci.ics.hyracks.control.cc.work.WaitForJobCompletionWork;
 import edu.uci.ics.hyracks.control.common.AbstractRemoteService;
-import edu.uci.ics.hyracks.control.common.base.IClusterController;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
 import edu.uci.ics.hyracks.control.common.context.ServerContext;
 import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
-import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
-import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
-import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
-import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
-import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
-import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
-import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+import edu.uci.ics.hyracks.control.common.ipc.ClusterControllerFunctions;
+import edu.uci.ics.hyracks.control.common.ipc.ClusterControllerFunctions.Function;
 import edu.uci.ics.hyracks.control.common.logs.LogFile;
-import edu.uci.ics.hyracks.control.common.work.FutureValue;
+import edu.uci.ics.hyracks.control.common.work.IPCResponder;
 import edu.uci.ics.hyracks.control.common.work.WorkQueue;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
 
-public class ClusterControllerService extends AbstractRemoteService implements IClusterController,
-        IHyracksClientInterface {
-    private static final long serialVersionUID = 1L;
+public class ClusterControllerService extends AbstractRemoteService {
+    private static Logger LOGGER = Logger.getLogger(ClusterControllerService.class.getName());
 
     private final CCConfig ccConfig;
 
-    private static Logger LOGGER = Logger.getLogger(ClusterControllerService.class.getName());
+    private IPCSystem clusterIPC;
+
+    private IPCSystem clientIPC;
 
     private final LogFile jobLog;
 
@@ -105,12 +99,10 @@
 
     private final WorkQueue workQueue;
 
-    private final Executor taskExecutor;
+    private final Executor executor;
 
     private final Timer timer;
 
-    private final CCClientInterface ccci;
-
     private final ICCContext ccContext;
 
     private final DeadNodeSweeper sweeper;
@@ -125,7 +117,11 @@
         ipAddressNodeNameMap = new HashMap<String, Set<String>>();
         applications = new Hashtable<String, CCApplicationContext>();
         serverCtx = new ServerContext(ServerContext.ServerType.CLUSTER_CONTROLLER, new File(ccConfig.ccRoot));
-        taskExecutor = Executors.newCachedThreadPool();
+        executor = Executors.newCachedThreadPool();
+        IIPCI ccIPCI = new ClusterControllerIPCI();
+        clusterIPC = new IPCSystem(new InetSocketAddress(ccConfig.clusterNetPort), ccIPCI);
+        IIPCI ciIPCI = new HyracksClientInterfaceIPCI();
+        clientIPC = new IPCSystem(new InetSocketAddress(ccConfig.clientNetIpAddress, ccConfig.clientNetPort), ciIPCI);
         webServer = new WebServer(this);
         activeRunMap = new HashMap<JobId, JobRun>();
         runMapArchive = new LinkedHashMap<JobId, JobRun>() {
@@ -137,11 +133,11 @@
         };
         workQueue = new WorkQueue();
         this.timer = new Timer(true);
-        ccci = new CCClientInterface(this);
         ccContext = new ICCContext() {
             @Override
-            public Map<String, Set<String>> getIPAddressNodeMap() {
-                return ipAddressNodeNameMap;
+            public void getIPAddressNodeMap(Map<String, Set<String>> map) throws Exception {
+                GetIpAddressNodeNameMapWork ginmw = new GetIpAddressNodeNameMapWork(ClusterControllerService.this, map);
+                workQueue.scheduleAndSync(ginmw);
             }
         };
         sweeper = new DeadNodeSweeper();
@@ -151,9 +147,8 @@
     @Override
     public void start() throws Exception {
         LOGGER.log(Level.INFO, "Starting ClusterControllerService: " + this);
-        Registry registry = LocateRegistry.createRegistry(ccConfig.port);
-        registry.rebind(IHyracksClientInterface.class.getName(), ccci);
-        registry.rebind(IClusterController.class.getName(), this);
+        clusterIPC.start();
+        clientIPC.start();
         webServer.setPort(ccConfig.httpPort);
         webServer.start();
         workQueue.start();
@@ -174,6 +169,14 @@
         LOGGER.log(Level.INFO, "Stopped ClusterControllerService");
     }
 
+    public ServerContext getServerContext() {
+        return serverCtx;
+    }
+
+    public ICCContext getCCContext() {
+        return ccContext;
+    }
+
     public Map<String, CCApplicationContext> getApplicationMap() {
         return applications;
     }
@@ -186,6 +189,10 @@
         return runMapArchive;
     }
 
+    public Map<String, Set<String>> getIpAddressNodeNameMap() {
+        return ipAddressNodeNameMap;
+    }
+
     public LogFile getJobLogFile() {
         return jobLog;
     }
@@ -195,7 +202,7 @@
     }
 
     public Executor getExecutor() {
-        return taskExecutor;
+        return executor;
     }
 
     public Map<String, NodeControllerState> getNodeMap() {
@@ -210,134 +217,16 @@
         return new JobId(jobCounter++);
     }
 
-    @Override
-    public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
-        JobId jobId = createJobId();
-        JobCreateWork jce = new JobCreateWork(this, jobId, appName, jobSpec, jobFlags);
-        workQueue.schedule(jce);
-        jce.sync();
-        return jobId;
-    }
-
-    @Override
-    public NodeParameters registerNode(NodeRegistration reg) throws Exception {
-        INodeController nodeController = reg.getNodeController();
-        String id = reg.getNodeId();
-        NodeControllerState state = new NodeControllerState(nodeController, reg);
-        workQueue.scheduleAndSync(new RegisterNodeWork(this, id, state));
-        nodeController.notifyRegistration(this);
-        LOGGER.log(Level.INFO, "Registered INodeController: id = " + id);
-        NodeParameters params = new NodeParameters();
-        params.setClusterControllerInfo(info);
-        params.setHeartbeatPeriod(ccConfig.heartbeatPeriod);
-        params.setProfileDumpPeriod(ccConfig.profileDumpPeriod);
-        return params;
-    }
-
-    @Override
-    public void unregisterNode(INodeController nodeController) throws Exception {
-        String id = nodeController.getId();
-        workQueue.scheduleAndSync(new UnregisterNodeWork(this, id));
-        LOGGER.log(Level.INFO, "Unregistered INodeController");
-    }
-
-    @Override
-    public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics)
-            throws Exception {
-        TaskCompleteWork sce = new TaskCompleteWork(this, jobId, taskId, nodeId, statistics);
-        workQueue.schedule(sce);
-    }
-
-    @Override
-    public void notifyTaskFailure(JobId jobId, TaskAttemptId taskId, String nodeId, String details) throws Exception {
-        TaskFailureWork tfe = new TaskFailureWork(this, jobId, taskId, nodeId, details);
-        workQueue.schedule(tfe);
-    }
-
-    @Override
-    public void notifyJobletCleanup(JobId jobId, String nodeId) throws Exception {
-        JobletCleanupNotificationWork jcnw = new JobletCleanupNotificationWork(this, jobId, nodeId);
-        workQueue.schedule(jcnw);
-    }
-
-    @Override
-    public JobStatus getJobStatus(JobId jobId) throws Exception {
-        GetJobStatusWork gse = new GetJobStatusWork(this, jobId);
-        workQueue.scheduleAndSync(gse);
-        return gse.getStatus();
-    }
-
-    @Override
-    public void start(JobId jobId) throws Exception {
-        JobStartWork jse = new JobStartWork(this, jobId);
-        workQueue.schedule(jse);
-    }
-
-    @Override
-    public void waitForCompletion(JobId jobId) throws Exception {
-        GetJobStatusConditionVariableWork e = new GetJobStatusConditionVariableWork(this, jobId);
-        workQueue.scheduleAndSync(e);
-        IJobStatusConditionVariable var = e.getConditionVariable();
-        if (var != null) {
-            var.waitForCompletion();
-        }
-    }
-
-    @Override
-    public void reportProfile(String id, List<JobProfile> profiles) throws Exception {
-        workQueue.schedule(new ReportProfilesWork(this, profiles));
-    }
-
-    @Override
-    public synchronized void nodeHeartbeat(String id, HeartbeatData hbData) throws Exception {
-        workQueue.schedule(new NodeHeartbeatWork(this, id, hbData));
-    }
-
-    @Override
-    public void createApplication(String appName) throws Exception {
-        synchronized (applications) {
-            if (applications.containsKey(appName)) {
-                throw new HyracksException("Duplicate application with name: " + appName + " being created.");
-            }
-            CCApplicationContext appCtx = new CCApplicationContext(serverCtx, ccContext, appName);
-            applications.put(appName, appCtx);
-        }
-    }
-
-    @Override
-    public void destroyApplication(String appName) throws Exception {
-        FutureValue<Object> fv = new FutureValue<Object>();
-        workQueue.schedule(new ApplicationDestroyWork(this, appName, fv));
-        fv.get();
-    }
-
-    @Override
-    public void startApplication(final String appName) throws Exception {
-        FutureValue<Object> fv = new FutureValue<Object>();
-        workQueue.schedule(new ApplicationStartWork(this, appName, fv));
-        fv.get();
-    }
-
-    @Override
-    public ClusterControllerInfo getClusterControllerInfo() throws Exception {
+    public ClusterControllerInfo getClusterControllerInfo() {
         return info;
     }
 
-    @Override
-    public Map<String, NodeControllerInfo> getNodeControllersInfo() throws Exception {
-        FutureValue<Map<String, NodeControllerInfo>> fv = new FutureValue<Map<String, NodeControllerInfo>>();
-        workQueue.schedule(new GetNodeControllersInfoWork(this, fv));
-        return fv.get();
+    public CCConfig getCCConfig() {
+        return ccConfig;
     }
 
-    @Override
-    public void registerPartitionProvider(PartitionDescriptor partitionDescriptor) {
-        workQueue.schedule(new RegisterPartitionAvailibilityWork(this, partitionDescriptor));
-    }
-
-    @Override
-    public void registerPartitionRequest(PartitionRequest partitionRequest) {
-        workQueue.schedule(new RegisterPartitionRequestWork(this, partitionRequest));
+    public IPCSystem getClusterIPC() {
+        return clusterIPC;
     }
 
     private class DeadNodeSweeper extends TimerTask {
@@ -346,4 +235,156 @@
             workQueue.schedule(new RemoveDeadNodesWork(ClusterControllerService.this));
         }
     }
+
+    private class HyracksClientInterfaceIPCI implements IIPCI {
+        @Override
+        public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+            HyracksClientInterfaceFunctions.Function fn = (HyracksClientInterfaceFunctions.Function) payload;
+            switch (fn.getFunctionId()) {
+                case GET_CLUSTER_CONTROLLER_INFO: {
+                    try {
+                        handle.send(mid, info, null);
+                    } catch (IPCException e) {
+                        e.printStackTrace();
+                    }
+                    return;
+                }
+
+                case CREATE_APPLICATION: {
+                    HyracksClientInterfaceFunctions.CreateApplicationFunction caf = (HyracksClientInterfaceFunctions.CreateApplicationFunction) fn;
+                    workQueue.schedule(new ApplicationCreateWork(ClusterControllerService.this, caf.getAppName(),
+                            new IPCResponder<Object>(handle, mid)));
+                    return;
+                }
+
+                case START_APPLICATION: {
+                    HyracksClientInterfaceFunctions.StartApplicationFunction saf = (HyracksClientInterfaceFunctions.StartApplicationFunction) fn;
+                    workQueue.schedule(new ApplicationStartWork(ClusterControllerService.this, saf.getAppName(),
+                            new IPCResponder<Object>(handle, mid)));
+                    return;
+                }
+
+                case DESTROY_APPLICATION: {
+                    HyracksClientInterfaceFunctions.DestroyApplicationFunction daf = (HyracksClientInterfaceFunctions.DestroyApplicationFunction) fn;
+                    workQueue.schedule(new ApplicationDestroyWork(ClusterControllerService.this, daf.getAppName(),
+                            new IPCResponder<Object>(handle, mid)));
+                    return;
+                }
+
+                case CREATE_JOB: {
+                    HyracksClientInterfaceFunctions.CreateJobFunction cjf = (HyracksClientInterfaceFunctions.CreateJobFunction) fn;
+                    JobId jobId = createJobId();
+                    workQueue.schedule(new JobCreateWork(ClusterControllerService.this, jobId, cjf.getAppName(), cjf
+                            .getJobSpec(), cjf.getJobFlags(), new IPCResponder<JobId>(handle, mid)));
+                    return;
+                }
+
+                case GET_JOB_STATUS: {
+                    HyracksClientInterfaceFunctions.GetJobStatusFunction gjsf = (HyracksClientInterfaceFunctions.GetJobStatusFunction) fn;
+                    workQueue.schedule(new GetJobStatusWork(ClusterControllerService.this, gjsf.getJobId(),
+                            new IPCResponder<JobStatus>(handle, mid)));
+                    return;
+                }
+
+                case START_JOB: {
+                    HyracksClientInterfaceFunctions.StartJobFunction sjf = (HyracksClientInterfaceFunctions.StartJobFunction) fn;
+                    workQueue.schedule(new JobStartWork(ClusterControllerService.this, sjf.getJobId(),
+                            new IPCResponder<Object>(handle, mid)));
+                    return;
+                }
+
+                case WAIT_FOR_COMPLETION: {
+                    HyracksClientInterfaceFunctions.WaitForCompletionFunction wfcf = (HyracksClientInterfaceFunctions.WaitForCompletionFunction) fn;
+                    workQueue.schedule(new WaitForJobCompletionWork(ClusterControllerService.this, wfcf.getJobId(),
+                            new IPCResponder<Object>(handle, mid)));
+                    return;
+                }
+
+                case GET_NODE_CONTROLLERS_INFO: {
+                    workQueue.schedule(new GetNodeControllersInfoWork(ClusterControllerService.this,
+                            new IPCResponder<Map<String, NodeControllerInfo>>(handle, mid)));
+                    return;
+                }
+            }
+            try {
+                handle.send(mid, null, new IllegalArgumentException("Unknown function " + fn.getFunctionId()));
+            } catch (IPCException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    private class ClusterControllerIPCI implements IIPCI {
+        @Override
+        public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+            ClusterControllerFunctions.Function fn = (Function) payload;
+            switch (fn.getFunctionId()) {
+                case REGISTER_NODE: {
+                    ClusterControllerFunctions.RegisterNodeFunction rnf = (ClusterControllerFunctions.RegisterNodeFunction) fn;
+                    workQueue.schedule(new RegisterNodeWork(ClusterControllerService.this, rnf.getNodeRegistration()));
+                    return;
+                }
+
+                case UNREGISTER_NODE: {
+                    ClusterControllerFunctions.UnregisterNodeFunction unf = (ClusterControllerFunctions.UnregisterNodeFunction) fn;
+                    workQueue.schedule(new UnregisterNodeWork(ClusterControllerService.this, unf.getNodeId()));
+                    return;
+                }
+
+                case NODE_HEARTBEAT: {
+                    ClusterControllerFunctions.NodeHeartbeatFunction nhf = (ClusterControllerFunctions.NodeHeartbeatFunction) fn;
+                    workQueue.schedule(new NodeHeartbeatWork(ClusterControllerService.this, nhf.getNodeId(), nhf
+                            .getHeartbeatData()));
+                    return;
+                }
+
+                case NOTIFY_JOBLET_CLEANUP: {
+                    ClusterControllerFunctions.NotifyJobletCleanupFunction njcf = (ClusterControllerFunctions.NotifyJobletCleanupFunction) fn;
+                    workQueue.schedule(new JobletCleanupNotificationWork(ClusterControllerService.this,
+                            njcf.getJobId(), njcf.getNodeId()));
+                    return;
+                }
+
+                case REPORT_PROFILE: {
+                    ClusterControllerFunctions.ReportProfileFunction rpf = (ClusterControllerFunctions.ReportProfileFunction) fn;
+                    workQueue.schedule(new ReportProfilesWork(ClusterControllerService.this, rpf.getProfiles()));
+                    return;
+                }
+
+                case NOTIFY_TASK_COMPLETE: {
+                    ClusterControllerFunctions.NotifyTaskCompleteFunction ntcf = (ClusterControllerFunctions.NotifyTaskCompleteFunction) fn;
+                    workQueue.schedule(new TaskCompleteWork(ClusterControllerService.this, ntcf.getJobId(), ntcf
+                            .getTaskId(), ntcf.getNodeId(), ntcf.getStatistics()));
+                    return;
+                }
+                case NOTIFY_TASK_FAILURE: {
+                    ClusterControllerFunctions.NotifyTaskFailureFunction ntff = (ClusterControllerFunctions.NotifyTaskFailureFunction) fn;
+                    workQueue.schedule(new TaskFailureWork(ClusterControllerService.this, ntff.getJobId(), ntff
+                            .getTaskId(), ntff.getDetails(), ntff.getDetails()));
+                    return;
+                }
+
+                case REGISTER_PARTITION_PROVIDER: {
+                    ClusterControllerFunctions.RegisterPartitionProviderFunction rppf = (ClusterControllerFunctions.RegisterPartitionProviderFunction) fn;
+                    workQueue.schedule(new RegisterPartitionAvailibilityWork(ClusterControllerService.this, rppf
+                            .getPartitionDescriptor()));
+                    return;
+                }
+
+                case REGISTER_PARTITION_REQUEST: {
+                    ClusterControllerFunctions.RegisterPartitionRequestFunction rprf = (ClusterControllerFunctions.RegisterPartitionRequestFunction) fn;
+                    workQueue.schedule(new RegisterPartitionRequestWork(ClusterControllerService.this, rprf
+                            .getPartitionRequest()));
+                    return;
+                }
+
+                case APPLICATION_STATE_CHANGE_RESPONSE: {
+                    ClusterControllerFunctions.ApplicationStateChangeResponseFunction astrf = (ClusterControllerFunctions.ApplicationStateChangeResponseFunction) fn;
+                    workQueue.schedule(new ApplicationStateChangeWork(ClusterControllerService.this, astrf));
+                    return;
+                }
+            }
+            LOGGER.warning("Unknown function: " + fn.getFunctionId());
+        }
+    }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
index 0b1fea0..2b7563a 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
@@ -80,6 +80,14 @@
 
     private final long[][] gcCollectionTimes;
 
+    private final long[] netPayloadBytesRead;
+
+    private final long[] netPayloadBytesWritten;
+
+    private final long[] netSignalingBytesRead;
+
+    private final long[] netSignalingBytesWritten;
+
     private int rrdPtr;
 
     private int lastHeartbeatDuration;
@@ -116,6 +124,10 @@
         }
         gcCollectionCounts = new long[gcN][RRD_SIZE];
         gcCollectionTimes = new long[gcN][RRD_SIZE];
+        netPayloadBytesRead = new long[RRD_SIZE];
+        netPayloadBytesWritten = new long[RRD_SIZE];
+        netSignalingBytesRead = new long[RRD_SIZE];
+        netSignalingBytesWritten = new long[RRD_SIZE];
         rrdPtr = 0;
     }
 
@@ -139,6 +151,10 @@
             gcCollectionCounts[i][rrdPtr] = hbData.gcCollectionCounts[i];
             gcCollectionTimes[i][rrdPtr] = hbData.gcCollectionTimes[i];
         }
+        netPayloadBytesRead[rrdPtr] = hbData.netPayloadBytesRead;
+        netPayloadBytesWritten[rrdPtr] = hbData.netPayloadBytesWritten;
+        netSignalingBytesRead[rrdPtr] = hbData.netSignalingBytesRead;
+        netSignalingBytesWritten[rrdPtr] = hbData.netSignalingBytesWritten;
         rrdPtr = (rrdPtr + 1) % RRD_SIZE;
     }
 
@@ -199,6 +215,10 @@
         o.put("gc-names", gcNames);
         o.put("gc-collection-counts", gcCollectionCounts);
         o.put("gc-collection-times", gcCollectionTimes);
+        o.put("net-payload-bytes-read", netPayloadBytesRead);
+        o.put("net-payload-bytes-written", netPayloadBytesWritten);
+        o.put("net-signaling-bytes-read", netSignalingBytesRead);
+        o.put("net-signaling-bytes-written", netSignalingBytesWritten);
 
         return o;
     }
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
index 6b7e143..585c7a1 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
@@ -15,17 +15,22 @@
 package edu.uci.ics.hyracks.control.cc.adminconsole.pages;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.wicket.markup.html.basic.Label;
-import org.apache.wicket.markup.html.panel.EmptyPanel;
 import org.apache.wicket.request.mapper.parameter.PageParameters;
 import org.apache.wicket.util.string.StringValue;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.work.GetJobActivityGraphJSONWork;
@@ -35,8 +40,7 @@
 public class JobDetailsPage extends AbstractPage {
     private static final long serialVersionUID = 1L;
 
-    private static final int HEIGHT = 9;
-    private static final int WIDTH = 9;
+    private static final int HEIGHT = 29;
 
     public JobDetailsPage(PageParameters params) throws Exception {
         ClusterControllerService ccs = getAdminConsoleApplication().getClusterControllerService();
@@ -47,19 +51,105 @@
 
         GetJobSpecificationJSONWork gjsw = new GetJobSpecificationJSONWork(ccs, jobId);
         ccs.getWorkQueue().scheduleAndSync(gjsw);
-        add(new Label("job-specification", gjsw.getJSON().toString()));
+        Label jobspec = new Label("job-specification", gjsw.getJSON().toString());
+        jobspec.setEscapeModelStrings(false);
+        add(jobspec);
 
         GetJobActivityGraphJSONWork gjagw = new GetJobActivityGraphJSONWork(ccs, jobId);
         ccs.getWorkQueue().scheduleAndSync(gjagw);
-        add(new Label("job-activity-graph", gjagw.getJSON().toString()));
+        Label jag = new Label("job-activity-graph", gjagw.getJSON().toString());
+        jag.setEscapeModelStrings(false);
+        add(jag);
+
+        JSONObject jagO = gjagw.getJSON();
+
+        Map<ActivityId, String> activityMap = new HashMap<ActivityId, String>();
+        if (jagO.has("activities")) {
+            JSONArray aArray = jagO.getJSONArray("activities");
+            for (int i = 0; i < aArray.length(); ++i) {
+                JSONObject aO = aArray.getJSONObject(i);
+                ActivityId aid = ActivityId.parse(aO.getString("id"));
+                String className = aO.getString("java-class");
+
+                activityMap.put(aid, className);
+            }
+        }
 
         GetJobRunJSONWork gjrw = new GetJobRunJSONWork(ccs, jobId);
         ccs.getWorkQueue().scheduleAndSync(gjrw);
-        add(new Label("job-run", gjrw.getJSON().toString()));
+        Label jobrun = new Label("job-run", gjrw.getJSON().toString());
+        jobrun.setEscapeModelStrings(false);
+        add(jobrun);
 
         JSONObject jrO = gjrw.getJSON();
 
-        List<TaskProfile> taskProfiles = new ArrayList<TaskProfile>();
+        List<TaskClusterAttempt[]> tcList = new ArrayList<TaskClusterAttempt[]>();
+        long minTime = Long.MAX_VALUE;
+        long maxTime = Long.MIN_VALUE;
+        if (jrO.has("activity-clusters")) {
+            JSONArray acA = jrO.getJSONArray("activity-clusters");
+            for (int i = 0; i < acA.length(); ++i) {
+                JSONObject acO = acA.getJSONObject(i);
+                if (acO.has("plan")) {
+                    JSONObject planO = acO.getJSONObject("plan");
+                    if (planO.has("task-clusters")) {
+                        JSONArray tcA = planO.getJSONArray("task-clusters");
+                        for (int j = 0; j < tcA.length(); ++j) {
+                            JSONObject tcO = tcA.getJSONObject(j);
+                            String tcId = tcO.getString("task-cluster-id");
+                            if (tcO.has("attempts")) {
+                                JSONArray tcaA = tcO.getJSONArray("attempts");
+                                TaskClusterAttempt[] tcAttempts = new TaskClusterAttempt[tcaA.length()];
+                                for (int k = 0; k < tcaA.length(); ++k) {
+                                    JSONObject tcaO = tcaA.getJSONObject(k);
+                                    int attempt = tcaO.getInt("attempt");
+                                    long startTime = tcaO.getLong("start-time");
+                                    long endTime = tcaO.getLong("end-time");
+
+                                    tcAttempts[k] = new TaskClusterAttempt(tcId, attempt, startTime, endTime);
+                                    if (startTime < minTime) {
+                                        minTime = startTime;
+                                    }
+                                    if (endTime > maxTime) {
+                                        maxTime = endTime;
+                                    }
+                                    if (tcaO.has("task-attempts")) {
+                                        JSONArray taArray = tcaO.getJSONArray("task-attempts");
+                                        tcAttempts[k].tasks = new TaskAttempt[taArray.length()];
+                                        for (int l = 0; l < taArray.length(); ++l) {
+                                            JSONObject taO = taArray.getJSONObject(l);
+                                            TaskAttemptId taId = TaskAttemptId.parse(taO.getString("task-attempt-id"));
+                                            TaskAttempt ta = new TaskAttempt(taId, taO.getLong("start-time"),
+                                                    taO.getLong("end-time"));
+                                            tcAttempts[k].tasks[l] = ta;
+                                            TaskId tid = taId.getTaskId();
+                                            ta.name = activityMap.get(tid.getActivityId());
+                                            ta.partition = tid.getPartition();
+                                        }
+                                        Arrays.sort(tcAttempts[k].tasks, new Comparator<TaskAttempt>() {
+                                            @Override
+                                            public int compare(TaskAttempt o1, TaskAttempt o2) {
+                                                return o1.startTime < o2.startTime ? -1
+                                                        : (o1.startTime > o2.startTime ? 1 : 0);
+                                            }
+                                        });
+                                    }
+                                }
+                                Arrays.sort(tcAttempts, new Comparator<TaskClusterAttempt>() {
+                                    @Override
+                                    public int compare(TaskClusterAttempt o1, TaskClusterAttempt o2) {
+                                        return o1.startTime < o2.startTime ? -1 : (o1.startTime > o2.startTime ? 1 : 0);
+                                    }
+                                });
+                                tcList.add(tcAttempts);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        Map<TaskAttemptId, TaskProfile> tpMap = new HashMap<TaskAttemptId, TaskProfile>();
         if (jrO.has("profile")) {
             JSONObject pO = jrO.getJSONObject("profile");
             if (pO.has("joblets")) {
@@ -70,22 +160,27 @@
                         JSONArray tasksA = jobletO.getJSONArray("tasks");
                         for (int j = 0; j < tasksA.length(); ++j) {
                             JSONObject taskO = tasksA.getJSONObject(j);
-                            String activityId = taskO.getString("activity-id");
+                            ActivityId activityId = ActivityId.parse(taskO.getString("activity-id"));
                             int partition = taskO.getInt("partition");
                             int attempt = taskO.getInt("attempt");
+                            TaskAttemptId taId = new TaskAttemptId(new TaskId(activityId, partition), attempt);
                             if (taskO.has("partition-send-profile")) {
                                 JSONArray taskProfilesA = taskO.getJSONArray("partition-send-profile");
                                 for (int k = 0; k < taskProfilesA.length(); ++k) {
                                     JSONObject ppO = taskProfilesA.getJSONObject(k);
                                     long openTime = ppO.getLong("open-time");
                                     long closeTime = ppO.getLong("close-time");
+                                    int resolution = ppO.getInt("resolution");
+                                    long offset = ppO.getLong("offset");
                                     JSONArray frameTimesA = ppO.getJSONArray("frame-times");
                                     long[] frameTimes = new long[frameTimesA.length()];
                                     for (int l = 0; l < frameTimes.length; ++l) {
-                                        frameTimes[l] = frameTimesA.getLong(l);
+                                        frameTimes[l] = frameTimesA.getInt(l) + offset;
                                     }
-                                    taskProfiles.add(new TaskProfile(activityId, partition, attempt, openTime,
-                                            closeTime, frameTimes));
+                                    TaskProfile tp = new TaskProfile(taId, openTime, closeTime, frameTimes, resolution);
+                                    if (!tpMap.containsKey(tp.taId)) {
+                                        tpMap.put(tp.taId, tp);
+                                    }
                                 }
                             }
                         }
@@ -93,68 +188,147 @@
                 }
             }
         }
-        if (!taskProfiles.isEmpty()) {
-            Collections.sort(taskProfiles, new Comparator<TaskProfile>() {
+
+        if (!tcList.isEmpty()) {
+            Collections.sort(tcList, new Comparator<TaskClusterAttempt[]>() {
                 @Override
-                public int compare(TaskProfile o1, TaskProfile o2) {
-                    return o1.openTime < o2.openTime ? -1 : (o1.openTime > o2.openTime ? 1 : 0);
+                public int compare(TaskClusterAttempt[] o1, TaskClusterAttempt[] o2) {
+                    if (o1.length == 0) {
+                        return o2.length == 0 ? 0 : -1;
+                    } else if (o2.length == 0) {
+                        return 1;
+                    }
+                    return o1[0].startTime < o2[0].startTime ? -1 : (o1[0].startTime > o2[0].startTime ? 1 : 0);
                 }
             });
-            long startTime = taskProfiles.get(0).openTime;
-            long timeRange = taskProfiles.get(taskProfiles.size() - 1).closeTime - startTime;
-            int n = taskProfiles.size();
+            long range = maxTime - minTime;
+
+            double leftOffset = 20;
+
+            int xWidth = 1024;
+            double width = ((double) xWidth) / range;
             StringBuilder buffer = new StringBuilder();
-            buffer.append("<svg viewBox=\"0 0 ").append((timeRange + 1) * (WIDTH + 1))
-                    .append(' ').append((n + 1) * (HEIGHT + 1)).append("\" version=\"1.1\"\n");
-            buffer.append("xmlns=\"http://www.w3.org/2000/svg\">\n");
-            for (int i = 0; i < n; ++i) {
-                TaskProfile tp = taskProfiles.get(i);
-                open(buffer, i, tp.openTime - startTime);
-                for (long ft : tp.frameTimes) {
-                    nextFrame(buffer, i, ft - startTime);
+            int y = 0;
+            for (TaskClusterAttempt[] tcAttempts : tcList) {
+                for (int i = 0; i < tcAttempts.length; ++i) {
+                    TaskClusterAttempt tca = tcAttempts[i];
+                    long startTime = tca.startTime - minTime;
+                    long endTime = tca.endTime - minTime;
+                    buffer.append("<rect x=\"").append(startTime * width + leftOffset).append("\" y=\"")
+                            .append(y * (HEIGHT + 1)).append("\" width=\"").append(width * (endTime - startTime))
+                            .append("\" height=\"").append(HEIGHT).append("\"/>\n");
+                    buffer.append("<text x=\"").append(endTime * width + leftOffset + 20).append("\" y=\"")
+                            .append(y * (HEIGHT + 1) + HEIGHT * 3 / 4).append("\">")
+                            .append((endTime - startTime) + " ms").append("</text>\n");
+                    ++y;
+                    for (int j = 0; j < tca.tasks.length; ++j) {
+                        TaskAttempt ta = tca.tasks[j];
+                        long tStartTime = ta.startTime - minTime;
+                        long tEndTime = ta.endTime - minTime;
+                        buffer.append("<rect x=\"").append(tStartTime * width + leftOffset).append("\" y=\"")
+                                .append(y * (HEIGHT + 1) + HEIGHT / 4).append("\" width=\"")
+                                .append(width * (tEndTime - tStartTime)).append("\" height=\"").append(HEIGHT / 2)
+                                .append("\" style=\"fill:rgb(255,255,255);stroke-width:1;stroke:rgb(0,0,0)\"/>\n");
+                        buffer.append("<text x=\"").append(tEndTime * width + leftOffset + 20).append("\" y=\"")
+                                .append(y * (HEIGHT + 1) + HEIGHT * 3 / 4).append("\">")
+                                .append((tEndTime - tStartTime) + " ms (" + ta.name + ":" + ta.partition + ")")
+                                .append("</text>\n");
+                        TaskProfile tp = tpMap.get(ta.taId);
+                        if (tp != null) {
+                            for (int k = 0; k < tp.frameTimes.length; ++k) {
+                                long taOpenTime = tp.openTime - minTime;
+                                buffer.append("<rect x=\"")
+                                        .append(taOpenTime * width + leftOffset)
+                                        .append("\" y=\"")
+                                        .append(y * (HEIGHT + 1) + HEIGHT / 4)
+                                        .append("\" width=\"1\" height=\"")
+                                        .append(HEIGHT / 2)
+                                        .append("\" style=\"fill:rgb(255,0,0);stroke-width:1;stroke:rgb(255,0,0)\"/>\n");
+                                for (int l = 0; l < tp.frameTimes.length; ++l) {
+                                    long ft = tp.frameTimes[l];
+                                    long ftn = l < tp.frameTimes.length - 1 ? tp.frameTimes[l + 1] : ft;
+                                    long taNextTime = ft - minTime;
+                                    long barWidth = ftn - ft;
+                                    buffer.append("<rect x=\"")
+                                            .append(taNextTime * width + leftOffset)
+                                            .append("\" y=\"")
+                                            .append(y * (HEIGHT + 1) + HEIGHT / 4)
+                                            .append("\" width=\"")
+                                            .append(barWidth == 0 ? 1 : (barWidth * width))
+                                            .append("\" height=\"")
+                                            .append(HEIGHT / 2)
+                                            .append("\" style=\"fill:rgb(0,255,0);stroke-width:1;stroke:rgb(0,255,0)\"/>\n");
+                                }
+                                long taCloseTime = tp.closeTime - minTime;
+                                buffer.append("<rect x=\"")
+                                        .append(taCloseTime * width + leftOffset)
+                                        .append("\" y=\"")
+                                        .append(y * (HEIGHT + 1) + HEIGHT / 4)
+                                        .append("\" width=\"1\" height=\"")
+                                        .append(HEIGHT / 2)
+                                        .append("\" style=\"fill:rgb(0,0,255);stroke-width:1;stroke:rgb(0,0,255)\"/>\n");
+                            }
+                        }
+                        ++y;
+                    }
                 }
-                close(buffer, i, tp.closeTime - startTime);
             }
+            buffer.append("<rect x=\"").append(leftOffset).append("\" y=\"").append(0).append("\" width=\"").append(1)
+                    .append("\" height=\"").append((y + 2) * (HEIGHT + 1)).append("\"/>\n");
+            buffer.append("<rect x=\"").append(0).append("\" y=\"").append((y + 1) * (HEIGHT + 1))
+                    .append("\" width=\"").append(xWidth + 2 * leftOffset).append("\" height=\"").append(1)
+                    .append("\"/>\n");
             buffer.append("</svg>");
-            Label markup = new Label("job-timeline", buffer.toString());
+            Label markup = new Label("job-timeline",
+                    "<svg version=\"1.1\"\nxmlns=\"http://www.w3.org/2000/svg\" width=\"" + (xWidth * 1.5)
+                            + "\" height=\"" + ((y + 2) * (HEIGHT + 1)) + "\">\n" + buffer.toString());
             markup.setEscapeModelStrings(false);
             add(markup);
-        } else {
-            add(new EmptyPanel("job-timeline"));
         }
     }
 
-    private void open(StringBuilder buffer, int i, long openTime) {
-        buffer.append("<rect x=\"").append(openTime * (WIDTH + 1)).append("\" y=\"").append(i * (HEIGHT + 1))
-                .append("\" width=\"").append(WIDTH).append("\" height=\"").append(HEIGHT).append("\"/>\n");
+    private static class TaskAttempt {
+        private TaskAttemptId taId;
+        private long startTime;
+        private long endTime;
+        private String name;
+        private int partition;
+
+        public TaskAttempt(TaskAttemptId taId, long startTime, long endTime) {
+            this.taId = taId;
+            this.startTime = startTime;
+            this.endTime = endTime;
+        }
     }
 
-    private void close(StringBuilder buffer, int i, long closeTime) {
-        buffer.append("<rect x=\"").append(closeTime * (WIDTH + 1)).append("\" y=\"").append(i * (HEIGHT + 1))
-                .append("\" width=\"").append(WIDTH).append("\" height=\"").append(HEIGHT).append("\"/>\n");
-    }
+    private static class TaskClusterAttempt {
+        private String tcId;
+        private int attempt;
+        private long startTime;
+        private long endTime;
+        private TaskAttempt[] tasks;
 
-    private void nextFrame(StringBuilder buffer, int i, long frameTime) {
-        buffer.append("<rect x=\"").append(frameTime * (WIDTH + 1)).append("\" y=\"").append(i * (HEIGHT + 1))
-                .append("\" width=\"").append(WIDTH).append("\" height=\"").append(HEIGHT).append("\"/>\n");
+        public TaskClusterAttempt(String tcId, int attempt, long startTime, long endTime) {
+            this.tcId = tcId;
+            this.attempt = attempt;
+            this.startTime = startTime;
+            this.endTime = endTime;
+        }
     }
 
     private static class TaskProfile {
-        private String activityId;
-        private int partition;
-        private int attempt;
+        private TaskAttemptId taId;
         private long openTime;
         private long closeTime;
         private long[] frameTimes;
+        private int resolution;
 
-        public TaskProfile(String activityId, int partition, int attempt, long openTime, long closeTime,
-                long[] frameTimes) {
-            this.activityId = activityId;
-            this.partition = partition;
-            this.activityId = activityId;
+        public TaskProfile(TaskAttemptId taId, long openTime, long closeTime, long[] frameTimes, int resolution) {
+            this.taId = taId;
             this.openTime = openTime;
             this.closeTime = closeTime;
             this.frameTimes = frameTimes;
+            this.resolution = resolution;
         }
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
index c1c0161..28f7c59 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
@@ -17,7 +17,9 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
 import edu.uci.ics.hyracks.api.application.ICCBootstrap;
@@ -30,10 +32,17 @@
 import edu.uci.ics.hyracks.control.cc.job.DeserializingJobSpecificationFactory;
 import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
 import edu.uci.ics.hyracks.control.common.context.ServerContext;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
 
 public class CCApplicationContext extends ApplicationContext implements ICCApplicationContext {
     private final ICCContext ccContext;
 
+    protected final Set<String> initPendingNodeIds;
+    protected final Set<String> deinitPendingNodeIds;
+
+    protected IResultCallback<Object> initializationCallback;
+    protected IResultCallback<Object> deinitializationCallback;
+
     private IJobSpecificationFactory jobSpecFactory;
 
     private List<IJobLifecycleListener> jobLifecycleListeners;
@@ -41,6 +50,8 @@
     public CCApplicationContext(ServerContext serverCtx, ICCContext ccContext, String appName) throws IOException {
         super(serverCtx, appName);
         this.ccContext = ccContext;
+        initPendingNodeIds = new HashSet<String>();
+        deinitPendingNodeIds = new HashSet<String>();
         jobSpecFactory = DeserializingJobSpecificationFactory.INSTANCE;
         jobLifecycleListeners = new ArrayList<IJobLifecycleListener>();
     }
@@ -98,4 +109,28 @@
             l.notifyJobCreation(jobId, specification);
         }
     }
+
+    public Set<String> getInitializationPendingNodeIds() {
+        return initPendingNodeIds;
+    }
+
+    public Set<String> getDeinitializationPendingNodeIds() {
+        return deinitPendingNodeIds;
+    }
+
+    public IResultCallback<Object> getInitializationCallback() {
+        return initializationCallback;
+    }
+
+    public void setInitializationCallback(IResultCallback<Object> initializationCallback) {
+        this.initializationCallback = initializationCallback;
+    }
+
+    public IResultCallback<Object> getDeinitializationCallback() {
+        return deinitializationCallback;
+    }
+
+    public void setDeinitializationCallback(IResultCallback<Object> deinitializationCallback) {
+        this.deinitializationCallback = deinitializationCallback;
+    }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
index df003b2..a72a866 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
@@ -316,6 +316,8 @@
                             JSONObject attempt = new JSONObject();
                             attempt.put("attempt", tca.getAttempt());
                             attempt.put("status", tca.getStatus());
+                            attempt.put("start-time", tca.getStartTime());
+                            attempt.put("end-time", tca.getEndTime());
 
                             JSONArray taskAttempts = new JSONArray();
                             for (TaskAttempt ta : tca.getTaskAttempts()) {
@@ -324,6 +326,8 @@
                                 taskAttempt.put("task-attempt-id", ta.getTaskAttemptId());
                                 taskAttempt.put("status", ta.getStatus());
                                 taskAttempt.put("node-id", ta.getNodeId());
+                                taskAttempt.put("start-time", ta.getStartTime());
+                                taskAttempt.put("end-time", ta.getEndTime());
                                 String failureDetails = ta.getFailureDetails();
                                 if (failureDetails != null) {
                                     taskAttempt.put("failure-details", failureDetails);
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java
index 43495db..7c0dd57 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java
@@ -29,7 +29,7 @@
 
     private final TaskAttemptId taskId;
 
-    private final Task taskState;
+    private final Task task;
 
     private String nodeId;
 
@@ -37,10 +37,16 @@
 
     private String failureDetails;
 
-    public TaskAttempt(TaskClusterAttempt tcAttempt, TaskAttemptId taskId, Task taskState) {
+    private long startTime;
+
+    private long endTime;
+
+    public TaskAttempt(TaskClusterAttempt tcAttempt, TaskAttemptId taskId, Task task) {
         this.tcAttempt = tcAttempt;
         this.taskId = taskId;
-        this.taskState = taskState;
+        this.task = task;
+        startTime = -1;
+        endTime = -1;
     }
 
     public TaskClusterAttempt getTaskClusterAttempt() {
@@ -51,8 +57,8 @@
         return taskId;
     }
 
-    public Task getTaskState() {
-        return taskState;
+    public Task getTask() {
+        return task;
     }
 
     public String getNodeId() {
@@ -75,4 +81,20 @@
         this.status = status;
         this.failureDetails = details;
     }
+
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public void setStartTime(long startTime) {
+        this.startTime = startTime;
+    }
+
+    public long getEndTime() {
+        return endTime;
+    }
+
+    public void setEndTime(long endTime) {
+        this.endTime = endTime;
+    }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java
index 9d33d12..84848bb 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java
@@ -32,9 +32,15 @@
 
     private int pendingTaskCounter;
 
+    private long startTime;
+
+    private long endTime;
+
     public TaskClusterAttempt(TaskCluster taskCluster, int attempt) {
         this.taskCluster = taskCluster;
         this.attempt = attempt;
+        startTime = -1;
+        endTime = -1;
     }
 
     public TaskCluster getTaskCluster() {
@@ -61,6 +67,22 @@
         return status;
     }
 
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public void setStartTime(long startTime) {
+        this.startTime = startTime;
+    }
+
+    public long getEndTime() {
+        return endTime;
+    }
+
+    public void setEndTime(long endTime) {
+        this.endTime = endTime;
+    }
+
     public void initializePendingTaskCounter() {
         pendingTaskCounter = taskAttempts.length;
     }
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java
index afa1946..2dcbb3e 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java
@@ -22,9 +22,10 @@
 import java.util.Set;
 import java.util.logging.Logger;
 
+import org.apache.commons.lang3.tuple.Pair;
+
 import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
 import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
 import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
 import edu.uci.ics.hyracks.control.common.job.PartitionState;
@@ -52,7 +53,7 @@
             while (i.hasNext()) {
                 PartitionRequest req = i.next();
                 if (partitionDescriptor.getState().isAtLeast(req.getMinimumState())) {
-                    matches.add(new Pair<PartitionDescriptor, PartitionRequest>(partitionDescriptor, req));
+                    matches.add(Pair.<PartitionDescriptor, PartitionRequest> of(partitionDescriptor, req));
                     i.remove();
                     matched = true;
                     if (!partitionDescriptor.isReusable()) {
@@ -88,7 +89,7 @@
             while (i.hasNext()) {
                 PartitionDescriptor descriptor = i.next();
                 if (descriptor.getState().isAtLeast(partitionRequest.getMinimumState())) {
-                    match = new Pair<PartitionDescriptor, PartitionRequest>(descriptor, partitionRequest);
+                    match = Pair.<PartitionDescriptor, PartitionRequest> of(descriptor, partitionRequest);
                     if (!descriptor.isReusable()) {
                         i.remove();
                     }
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java
index 90c6ad1..c36cf62 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java
@@ -1,8 +1,9 @@
 package edu.uci.ics.hyracks.control.cc.partitions;
 
+import org.apache.commons.lang3.tuple.Pair;
+
 import edu.uci.ics.hyracks.api.comm.NetworkAddress;
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.NodeControllerState;
 import edu.uci.ics.hyracks.control.common.base.INodeController;
@@ -11,23 +12,14 @@
 
 public class PartitionUtils {
     public static void reportPartitionMatch(ClusterControllerService ccs, final PartitionId pid,
-            Pair<PartitionDescriptor, PartitionRequest> match) {
-        PartitionDescriptor desc = match.first;
-        PartitionRequest req = match.second;
+            Pair<PartitionDescriptor, PartitionRequest> match) throws Exception {
+        PartitionDescriptor desc = match.getLeft();
+        PartitionRequest req = match.getRight();
 
         NodeControllerState producerNCS = ccs.getNodeMap().get(desc.getNodeId());
         NodeControllerState requestorNCS = ccs.getNodeMap().get(req.getNodeId());
         final NetworkAddress dataport = producerNCS.getDataPort();
         final INodeController requestorNC = requestorNCS.getNodeController();
-        ccs.getExecutor().execute(new Runnable() {
-            @Override
-            public void run() {
-                try {
-                    requestorNC.reportPartitionAvailability(pid, dataport);
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }
-            }
-        });
+        requestorNC.reportPartitionAvailability(pid, dataport);
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteOp.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteOp.java
deleted file mode 100644
index ec15186..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteOp.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote;
-
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public interface RemoteOp<T> {
-    public String getNodeId();
-
-    public T execute(INodeController node) throws Exception;
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteRunner.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteRunner.java
deleted file mode 100644
index a37c786..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteRunner.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote;
-
-import java.util.List;
-import java.util.Vector;
-import java.util.concurrent.Semaphore;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class RemoteRunner {
-    public static <T, R> R runRemote(ClusterControllerService ccs, final RemoteOp<T>[] remoteOps,
-            final Accumulator<T, R> accumulator) throws Exception {
-        final Semaphore installComplete = new Semaphore(remoteOps.length);
-        final List<Exception> errors = new Vector<Exception>();
-        for (final RemoteOp<T> remoteOp : remoteOps) {
-            NodeControllerState nodeState = ccs.getNodeMap().get(remoteOp.getNodeId());
-            final INodeController node = nodeState.getNodeController();
-
-            installComplete.acquire();
-            Runnable remoteRunner = new Runnable() {
-                @Override
-                public void run() {
-                    try {
-                        T t = remoteOp.execute(node);
-                        if (accumulator != null) {
-                            synchronized (accumulator) {
-                                accumulator.accumulate(t);
-                            }
-                        }
-                    } catch (Exception e) {
-                        errors.add(e);
-                    } finally {
-                        installComplete.release();
-                    }
-                }
-            };
-
-            ccs.getExecutor().execute(remoteRunner);
-        }
-        installComplete.acquire(remoteOps.length);
-        if (!errors.isEmpty()) {
-            throw errors.get(0);
-        }
-        return accumulator == null ? null : accumulator.getResult();
-    }
-}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationDestroyer.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationDestroyer.java
deleted file mode 100644
index 52e726c..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationDestroyer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote.ops;
-
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class ApplicationDestroyer implements RemoteOp<Void> {
-    private String nodeId;
-    private String appName;
-
-    public ApplicationDestroyer(String nodeId, String appName) {
-        this.nodeId = nodeId;
-        this.appName = appName;
-    }
-
-    @Override
-    public String getNodeId() {
-        return nodeId;
-    }
-
-    @Override
-    public Void execute(INodeController node) throws Exception {
-        node.destroyApplication(appName);
-        return null;
-    }
-
-    @Override
-    public String toString() {
-        return "Destroyed application: " + appName;
-    }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationStarter.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationStarter.java
deleted file mode 100644
index d6a1d26..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationStarter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote.ops;
-
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class ApplicationStarter implements RemoteOp<Void> {
-    private String nodeId;
-    private String appName;
-    private boolean deployHar;
-    private byte[] distributedState;
-
-    public ApplicationStarter(String nodeId, String appName, boolean deployHar, byte[] distributedState) {
-        this.nodeId = nodeId;
-        this.appName = appName;
-        this.deployHar = deployHar;
-        this.distributedState = distributedState;
-    }
-
-    @Override
-    public String getNodeId() {
-        return nodeId;
-    }
-
-    @Override
-    public Void execute(INodeController node) throws Exception {
-        node.createApplication(appName, deployHar, distributedState);
-        return null;
-    }
-
-    @Override
-    public String toString() {
-        return "Started application: " + appName;
-    }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/JobCompleteNotifier.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/JobCompleteNotifier.java
deleted file mode 100644
index d0f9151..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/JobCompleteNotifier.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote.ops;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class JobCompleteNotifier implements RemoteOp<Void> {
-    private String nodeId;
-    private JobId jobId;
-    private JobStatus status;
-
-    public JobCompleteNotifier(String nodeId, JobId jobId, JobStatus status) {
-        this.nodeId = nodeId;
-        this.jobId = jobId;
-        this.status = status;
-    }
-
-    @Override
-    public Void execute(INodeController node) throws Exception {
-        node.cleanUpJoblet(jobId, status);
-        return null;
-    }
-
-    @Override
-    public String toString() {
-        return jobId + " Cleaning Up";
-    }
-
-    @Override
-    public String getNodeId() {
-        return nodeId;
-    }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
index 0e59c8df..f407ae3 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
@@ -22,13 +22,14 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.commons.lang3.tuple.Pair;
+
 import edu.uci.ics.hyracks.api.dataflow.ActivityId;
 import edu.uci.ics.hyracks.api.dataflow.IActivity;
 import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
 import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
 import edu.uci.ics.hyracks.api.job.JobActivityGraph;
 import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.api.util.Pair;
 import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
 import edu.uci.ics.hyracks.control.cc.job.ActivityClusterId;
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
@@ -57,7 +58,7 @@
                         int producerOutputIndex = spec.getProducerOutputIndex(conn);
                         ActivityId inTask = jag.getOperatorOutputMap().get(producerId).get(producerOutputIndex);
                         if (!eqSet.getActivities().contains(inTask)) {
-                            return new Pair<ActivityId, ActivityId>(t, inTask);
+                            return Pair.<ActivityId, ActivityId> of(t, inTask);
                         }
                     }
                 }
@@ -70,7 +71,7 @@
                         int consumerInputIndex = spec.getConsumerInputIndex(conn);
                         ActivityId outTask = jag.getOperatorInputMap().get(consumerId).get(consumerInputIndex);
                         if (!eqSet.getActivities().contains(outTask)) {
-                            return new Pair<ActivityId, ActivityId>(t, outTask);
+                            return Pair.<ActivityId, ActivityId> of(t, outTask);
                         }
                     }
                 }
@@ -102,7 +103,7 @@
             changed = false;
             Pair<ActivityId, ActivityId> pair = findMergePair(jag, spec, stages);
             if (pair != null) {
-                merge(stageMap, stages, pair.first, pair.second);
+                merge(stageMap, stages, pair.getLeft(), pair.getRight());
                 changed = true;
             }
         }
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
index c721eca..929462b 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
@@ -25,6 +25,8 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.commons.lang3.tuple.Pair;
+
 import edu.uci.ics.hyracks.api.constraints.expressions.LValueConstraintExpression;
 import edu.uci.ics.hyracks.api.constraints.expressions.PartitionCountExpression;
 import edu.uci.ics.hyracks.api.dataflow.ActivityId;
@@ -38,7 +40,6 @@
 import edu.uci.ics.hyracks.api.exceptions.HyracksException;
 import edu.uci.ics.hyracks.api.job.JobActivityGraph;
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
 import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
 import edu.uci.ics.hyracks.control.cc.job.ActivityClusterPlan;
 import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
@@ -128,7 +129,7 @@
                         Set<TaskId> cluster = taskClusterMap.get(ac1TaskStates[i].getTaskId());
                         for (int j = targetBitmap.nextSetBit(0); j >= 0; j = targetBitmap.nextSetBit(j + 1)) {
                             TaskId targetTID = ac2TaskStates[j].getTaskId();
-                            cInfoList.add(new Pair<TaskId, ConnectorDescriptorId>(targetTID, cdId));
+                            cInfoList.add(Pair.<TaskId, ConnectorDescriptorId> of(targetTID, cdId));
                             IConnectorPolicy cPolicy = connectorPolicies.get(cdId);
                             if (cPolicy.requiresProducerConsumerCoscheduling()) {
                                 cluster.add(targetTID);
@@ -148,12 +149,13 @@
                 List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = taskConnectivity.get(tid);
                 if (cInfoList != null) {
                     for (Pair<TaskId, ConnectorDescriptorId> p : cInfoList) {
-                        Task targetTS = activityPlanMap.get(p.first.getActivityId()).getTasks()[p.first.getPartition()];
+                        Task targetTS = activityPlanMap.get(p.getLeft().getActivityId()).getTasks()[p.getLeft()
+                                .getPartition()];
                         TaskCluster targetTC = targetTS.getTaskCluster();
                         if (targetTC != tc) {
-                            ConnectorDescriptorId cdId = p.second;
-                            PartitionId pid = new PartitionId(jobRun.getJobId(), cdId, tid.getPartition(),
-                                    p.first.getPartition());
+                            ConnectorDescriptorId cdId = p.getRight();
+                            PartitionId pid = new PartitionId(jobRun.getJobId(), cdId, tid.getPartition(), p.getLeft()
+                                    .getPartition());
                             tc.getProducedPartitions().add(pid);
                             targetTC.getRequiredPartitions().add(pid);
                             partitionProducingTaskClusterMap.put(pid, tc);
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
index 3feb5a6..9a443d9 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
@@ -184,8 +184,8 @@
     private void startRunnableActivityClusters() throws HyracksException {
         Set<TaskCluster> taskClusterRoots = new HashSet<TaskCluster>();
         findRunnableTaskClusterRoots(taskClusterRoots, rootActivityClusters);
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: "
+        if (LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: "
                     + inProgressTaskClusters);
         }
         if (taskClusterRoots.isEmpty() && inProgressTaskClusters.isEmpty()) {
@@ -213,19 +213,19 @@
                 queue.add(new RankedRunnableTaskCluster(priority, tc));
             }
         }
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Ranked TCs: " + queue);
+        if (LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("Ranked TCs: " + queue);
         }
 
         Map<String, List<TaskAttemptDescriptor>> taskAttemptMap = new HashMap<String, List<TaskAttemptDescriptor>>();
         for (RankedRunnableTaskCluster rrtc : queue) {
             TaskCluster tc = rrtc.getTaskCluster();
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Found runnable TC: " + tc);
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("Found runnable TC: " + tc);
                 List<TaskClusterAttempt> attempts = tc.getAttempts();
-                LOGGER.info("Attempts so far:" + attempts.size());
+                LOGGER.fine("Attempts so far:" + attempts.size());
                 for (TaskClusterAttempt tcAttempt : attempts) {
-                    LOGGER.info("Status: " + tcAttempt.getStatus());
+                    LOGGER.fine("Status: " + tcAttempt.getStatus());
                 }
             }
             assignTaskLocations(tc, taskAttemptMap);
@@ -245,16 +245,16 @@
      * Runnability(Non-schedulable TaskCluster) = {NOT_RUNNABLE, _} 
      */
     private Runnability assignRunnabilityRank(TaskCluster goal, Map<TaskCluster, Runnability> runnabilityMap) {
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Computing runnability: " + goal);
+        if (LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("Computing runnability: " + goal);
         }
         if (runnabilityMap.containsKey(goal)) {
             return runnabilityMap.get(goal);
         }
         TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(goal);
         if (lastAttempt != null) {
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Last Attempt Status: " + lastAttempt.getStatus());
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("Last Attempt Status: " + lastAttempt.getStatus());
             }
             if (lastAttempt.getStatus() == TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
                 Runnability runnability = new Runnability(Runnability.Tag.COMPLETED, Integer.MIN_VALUE);
@@ -271,15 +271,15 @@
         PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
         Runnability aggregateRunnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
         for (PartitionId pid : goal.getRequiredPartitions()) {
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Inspecting required partition: " + pid);
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("Inspecting required partition: " + pid);
             }
             Runnability runnability;
             ConnectorDescriptorId cdId = pid.getConnectorDescriptorId();
             IConnectorPolicy cPolicy = connectorPolicyMap.get(cdId);
             PartitionState maxState = pmm.getMaximumAvailableState(pid);
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Policy: " + cPolicy + " maxState: " + maxState);
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("Policy: " + cPolicy + " maxState: " + maxState);
             }
             if (PartitionState.COMMITTED.equals(maxState)) {
                 runnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
@@ -309,8 +309,12 @@
                 }
             }
             aggregateRunnability = Runnability.getWorstCase(aggregateRunnability, runnability);
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("aggregateRunnability: " + aggregateRunnability);
+            if (aggregateRunnability.getTag() == Runnability.Tag.NOT_RUNNABLE) {
+                // already not runnable -- cannot get better. bail.
+                break;
+            }
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("aggregateRunnability: " + aggregateRunnability);
             }
         }
         runnabilityMap.put(goal, aggregateRunnability);
@@ -387,6 +391,7 @@
             }
             taskAttempt.setNodeId(nodeId);
             taskAttempt.setStatus(TaskAttempt.TaskStatus.RUNNING, null);
+            taskAttempt.setStartTime(System.currentTimeMillis());
             List<TaskAttemptDescriptor> tads = taskAttemptMap.get(nodeId);
             if (tads == null) {
                 tads = new ArrayList<TaskAttemptDescriptor>();
@@ -399,6 +404,7 @@
         tcAttempt.initializePendingTaskCounter();
         tcAttempts.add(tcAttempt);
         tcAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.RUNNING);
+        tcAttempt.setStartTime(System.currentTimeMillis());
         inProgressTaskClusters.add(tc);
     }
 
@@ -446,7 +452,7 @@
                     public void run() {
                         try {
                             node.getNodeController().startTasks(appName, jobId, JavaSerializationUtils.serialize(jag),
-                                    taskDescriptors, connectorPolicies, null);
+                                    taskDescriptors, connectorPolicies);
                         } catch (IOException e) {
                             e.printStackTrace();
                         } catch (Exception e) {
@@ -459,24 +465,26 @@
     }
 
     private void abortJob(Exception exception) {
-        for (TaskCluster tc : inProgressTaskClusters) {
+        Set<TaskCluster> inProgressTaskClustersCopy = new HashSet<TaskCluster>(inProgressTaskClusters);
+        for (TaskCluster tc : inProgressTaskClustersCopy) {
             abortTaskCluster(findLastTaskClusterAttempt(tc));
         }
-        inProgressTaskClusters.clear();
+        assert inProgressTaskClusters.isEmpty();
         ccs.getWorkQueue().schedule(new JobCleanupWork(ccs, jobRun.getJobId(), JobStatus.FAILURE, exception));
     }
 
     private void abortTaskCluster(TaskClusterAttempt tcAttempt) {
-        LOGGER.info("Aborting task cluster: " + tcAttempt.getAttempt());
+        LOGGER.fine("Aborting task cluster: " + tcAttempt.getAttempt());
         Set<TaskAttemptId> abortTaskIds = new HashSet<TaskAttemptId>();
         Map<String, List<TaskAttemptId>> abortTaskAttemptMap = new HashMap<String, List<TaskAttemptId>>();
         for (TaskAttempt ta : tcAttempt.getTaskAttempts()) {
             TaskAttemptId taId = ta.getTaskAttemptId();
             TaskAttempt.TaskStatus status = ta.getStatus();
             abortTaskIds.add(taId);
-            LOGGER.info("Checking " + taId + ": " + ta.getStatus());
+            LOGGER.fine("Checking " + taId + ": " + ta.getStatus());
             if (status == TaskAttempt.TaskStatus.RUNNING || status == TaskAttempt.TaskStatus.COMPLETED) {
                 ta.setStatus(TaskAttempt.TaskStatus.ABORTED, null);
+                ta.setEndTime(System.currentTimeMillis());
                 List<TaskAttemptId> abortTaskAttempts = abortTaskAttemptMap.get(ta.getNodeId());
                 if (abortTaskAttempts == null) {
                     abortTaskAttempts = new ArrayList<TaskAttemptId>();
@@ -486,12 +494,12 @@
             }
         }
         final JobId jobId = jobRun.getJobId();
-        LOGGER.info("Abort map for job: " + jobId + ": " + abortTaskAttemptMap);
+        LOGGER.fine("Abort map for job: " + jobId + ": " + abortTaskAttemptMap);
         for (Map.Entry<String, List<TaskAttemptId>> e : abortTaskAttemptMap.entrySet()) {
             final NodeControllerState node = ccs.getNodeMap().get(e.getKey());
             final List<TaskAttemptId> abortTaskAttempts = e.getValue();
             if (node != null) {
-                LOGGER.info("Aborting: " + abortTaskAttempts + " at " + e.getKey());
+                LOGGER.fine("Aborting: " + abortTaskAttempts + " at " + e.getKey());
                 ccs.getExecutor().execute(new Runnable() {
                     @Override
                     public void run() {
@@ -524,6 +532,7 @@
             TaskClusterAttempt tca = findLastTaskClusterAttempt(tc);
             if (tca != null) {
                 abortTaskCluster(tca);
+                tca.setEndTime(System.currentTimeMillis());
                 tca.setStatus(TaskClusterAttempt.TaskClusterStatus.ABORTED);
             }
         }
@@ -571,14 +580,16 @@
 
     public void notifyTaskComplete(TaskAttempt ta) throws HyracksException {
         TaskAttemptId taId = ta.getTaskAttemptId();
-        TaskCluster tc = ta.getTaskState().getTaskCluster();
+        TaskCluster tc = ta.getTask().getTaskCluster();
         TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc);
         if (lastAttempt != null && taId.getAttempt() == lastAttempt.getAttempt()) {
             TaskAttempt.TaskStatus taStatus = ta.getStatus();
             if (taStatus == TaskAttempt.TaskStatus.RUNNING) {
                 ta.setStatus(TaskAttempt.TaskStatus.COMPLETED, null);
+                ta.setEndTime(System.currentTimeMillis());
                 if (lastAttempt.decrementPendingTasksCounter() == 0) {
                     lastAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.COMPLETED);
+                    lastAttempt.setEndTime(System.currentTimeMillis());
                     inProgressTaskClusters.remove(tc);
                     startRunnableActivityClusters();
                 }
@@ -602,15 +613,16 @@
      */
     public void notifyTaskFailure(TaskAttempt ta, ActivityCluster ac, String details) {
         try {
-            LOGGER.info("Received failure notification for TaskAttempt " + ta.getTaskAttemptId());
+            LOGGER.fine("Received failure notification for TaskAttempt " + ta.getTaskAttemptId());
             TaskAttemptId taId = ta.getTaskAttemptId();
-            TaskCluster tc = ta.getTaskState().getTaskCluster();
+            TaskCluster tc = ta.getTask().getTaskCluster();
             TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc);
             if (lastAttempt != null && taId.getAttempt() == lastAttempt.getAttempt()) {
-                LOGGER.info("Marking TaskAttempt " + ta.getTaskAttemptId() + " as failed");
+                LOGGER.fine("Marking TaskAttempt " + ta.getTaskAttemptId() + " as failed");
                 ta.setStatus(TaskAttempt.TaskStatus.FAILED, details);
                 abortTaskCluster(lastAttempt);
                 lastAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.FAILED);
+                lastAttempt.setEndTime(System.currentTimeMillis());
                 abortDoomedTaskClusters();
                 if (lastAttempt.getAttempt() >= ac.getMaxTaskClusterAttempts()) {
                     abortJob(new HyracksException(details));
@@ -648,6 +660,7 @@
                                 assert (ta.getStatus() == TaskAttempt.TaskStatus.COMPLETED || ta.getStatus() == TaskAttempt.TaskStatus.RUNNING);
                                 if (deadNodes.contains(ta.getNodeId())) {
                                     ta.setStatus(TaskAttempt.TaskStatus.FAILED, "Node " + ta.getNodeId() + " failed");
+                                    ta.setEndTime(System.currentTimeMillis());
                                     abort = true;
                                 }
                             }
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationCreateWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationCreateWork.java
new file mode 100644
index 0000000..15d6d1f
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationCreateWork.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.io.IOException;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+
+public class ApplicationCreateWork extends AbstractWork {
+    private final ClusterControllerService ccs;
+    private final String appName;
+    private IResultCallback<Object> callback;
+
+    public ApplicationCreateWork(ClusterControllerService ccs, String appName, IResultCallback<Object> callback) {
+        this.ccs = ccs;
+        this.appName = appName;
+        this.callback = callback;
+    }
+
+    @Override
+    public void run() {
+        try {
+            Map<String, CCApplicationContext> applications = ccs.getApplicationMap();
+            if (applications.containsKey(appName)) {
+                callback.setException(new HyracksException("Duplicate application with name: " + appName
+                        + " being created."));
+                return;
+            }
+            CCApplicationContext appCtx;
+            try {
+                appCtx = new CCApplicationContext(ccs.getServerContext(), ccs.getCCContext(), appName);
+            } catch (IOException e) {
+                callback.setException(e);
+                return;
+            }
+            appCtx.setStatus(ApplicationStatus.CREATED);
+            applications.put(appName, appCtx);
+            callback.setValue(null);
+        } catch (Exception e) {
+            callback.setException(e);
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationDestroyWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationDestroyWork.java
index c6af1b9..ac780c4 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationDestroyWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationDestroyWork.java
@@ -14,62 +14,51 @@
  */
 package edu.uci.ics.hyracks.control.cc.work;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Map;
 
 import edu.uci.ics.hyracks.api.exceptions.HyracksException;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteRunner;
-import edu.uci.ics.hyracks.control.cc.remote.ops.ApplicationDestroyer;
-import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
 import edu.uci.ics.hyracks.control.common.work.AbstractWork;
-import edu.uci.ics.hyracks.control.common.work.FutureValue;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
 
 public class ApplicationDestroyWork extends AbstractWork {
     private final ClusterControllerService ccs;
     private final String appName;
-    private FutureValue<Object> fv;
+    private IResultCallback<Object> callback;
 
-    public ApplicationDestroyWork(ClusterControllerService ccs, String appName, FutureValue<Object> fv) {
+    public ApplicationDestroyWork(ClusterControllerService ccs, String appName, IResultCallback<Object> callback) {
         this.ccs = ccs;
         this.appName = appName;
-        this.fv = fv;
+        this.callback = callback;
     }
 
     @Override
     public void run() {
-        final ApplicationContext appCtx = ccs.getApplicationMap().remove(appName);
-        if (appCtx == null) {
-            fv.setException(new HyracksException("No application with name: " + appName));
-            return;
-        }
-        List<RemoteOp<Void>> opList = new ArrayList<RemoteOp<Void>>();
-        for (final String nodeId : ccs.getNodeMap().keySet()) {
-            opList.add(new ApplicationDestroyer(nodeId, appName));
-        }
-        final RemoteOp[] ops = opList.toArray(new RemoteOp[opList.size()]);
-        ccs.getExecutor().execute(new Runnable() {
-            @Override
-            public void run() {
-                try {
-                    RemoteRunner.runRemote(ccs, ops, null);
-                } catch (Exception e) {
-                    fv.setException(e);
-                    return;
-                }
-                ccs.getWorkQueue().schedule(new AbstractWork() {
-                    @Override
-                    public void run() {
-                        try {
-                            appCtx.deinitialize();
-                        } catch (Exception e) {
-                            fv.setException(e);
-                        }
-                        fv.setValue(null);
-                    }
-                });
+        try {
+            final CCApplicationContext appCtx = ccs.getApplicationMap().remove(appName);
+            if (appCtx == null) {
+                callback.setException(new HyracksException("No application with name: " + appName));
+                return;
             }
-        });
+            if (appCtx.getStatus() == ApplicationStatus.IN_DEINITIALIZATION
+                    || appCtx.getStatus() == ApplicationStatus.DEINITIALIZED) {
+                return;
+            }
+            Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+            appCtx.getDeinitializationPendingNodeIds().addAll(nodeMap.keySet());
+            appCtx.setStatus(ApplicationStatus.IN_DEINITIALIZATION);
+            appCtx.setDeinitializationCallback(callback);
+            for (String nodeId : ccs.getNodeMap().keySet()) {
+                NodeControllerState nodeState = nodeMap.get(nodeId);
+                final INodeController node = nodeState.getNodeController();
+                node.destroyApplication(appName);
+            }
+        } catch (Exception e) {
+            callback.setException(e);
+        }
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStartWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStartWork.java
index 2022c7e..e4ad56c 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStartWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStartWork.java
@@ -14,60 +14,67 @@
  */
 package edu.uci.ics.hyracks.control.cc.work;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
 
 import edu.uci.ics.hyracks.api.exceptions.HyracksException;
 import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteRunner;
-import edu.uci.ics.hyracks.control.cc.remote.ops.ApplicationStarter;
-import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
 import edu.uci.ics.hyracks.control.common.work.AbstractWork;
-import edu.uci.ics.hyracks.control.common.work.FutureValue;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
 
 public class ApplicationStartWork extends AbstractWork {
     private final ClusterControllerService ccs;
     private final String appName;
-    private final FutureValue<Object> fv;
+    private final IResultCallback<Object> callback;
 
-    public ApplicationStartWork(ClusterControllerService ccs, String appName, FutureValue<Object> fv) {
+    public ApplicationStartWork(ClusterControllerService ccs, String appName, IResultCallback<Object> callback) {
         this.ccs = ccs;
         this.appName = appName;
-        this.fv = fv;
+        this.callback = callback;
     }
 
     @Override
     public void run() {
-        ApplicationContext appCtx = ccs.getApplicationMap().get(appName);
-        if (appCtx == null) {
-            fv.setException(new HyracksException("No application with name: " + appName));
-            return;
-        }
         try {
-            appCtx.initializeClassPath();
-            appCtx.initialize();
-            final byte[] distributedState = JavaSerializationUtils.serialize(appCtx.getDistributedState());
-            final boolean deployHar = appCtx.containsHar();
-            List<RemoteOp<Void>> opList = new ArrayList<RemoteOp<Void>>();
-            for (final String nodeId : ccs.getNodeMap().keySet()) {
-                opList.add(new ApplicationStarter(nodeId, appName, deployHar, distributedState));
+            final CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
+            if (appCtx == null) {
+                callback.setException(new HyracksException("No application with name: " + appName));
+                return;
             }
-            final RemoteOp[] ops = opList.toArray(new RemoteOp[opList.size()]);
+            if (appCtx.getStatus() != ApplicationStatus.CREATED) {
+                callback.setException(new HyracksException("Application in incorrect state for starting: "
+                        + appCtx.getStatus()));
+            }
+            final Map<String, NodeControllerState> nodeMapCopy = new HashMap<String, NodeControllerState>(
+                    ccs.getNodeMap());
+            appCtx.getInitializationPendingNodeIds().addAll(nodeMapCopy.keySet());
+            appCtx.setStatus(ApplicationStatus.IN_INITIALIZATION);
+            appCtx.setInitializationCallback(callback);
             ccs.getExecutor().execute(new Runnable() {
                 @Override
                 public void run() {
                     try {
-                        RemoteRunner.runRemote(ccs, ops, null);
-                        fv.setValue(null);
+                        appCtx.initializeClassPath();
+                        appCtx.initialize();
+                        final byte[] distributedState = JavaSerializationUtils.serialize(appCtx.getDistributedState());
+                        final boolean deployHar = appCtx.containsHar();
+                        for (final String nodeId : nodeMapCopy.keySet()) {
+                            NodeControllerState nodeState = nodeMapCopy.get(nodeId);
+                            final INodeController node = nodeState.getNodeController();
+                            node.createApplication(appName, deployHar, distributedState);
+                        }
                     } catch (Exception e) {
-                        fv.setException(e);
+                        callback.setException(e);
                     }
                 }
             });
         } catch (Exception e) {
-            fv.setException(e);
+            callback.setException(e);
         }
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStateChangeWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStateChangeWork.java
new file mode 100644
index 0000000..2d8ae85
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStateChangeWork.java
@@ -0,0 +1,85 @@
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Set;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.ipc.ClusterControllerFunctions;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+
+public class ApplicationStateChangeWork extends AbstractWork {
+    private static final Logger LOGGER = Logger.getLogger(ApplicationStateChangeWork.class.getName());
+
+    private final ClusterControllerService ccs;
+    private final ClusterControllerFunctions.ApplicationStateChangeResponseFunction ascrf;
+
+    public ApplicationStateChangeWork(ClusterControllerService ccs,
+            ClusterControllerFunctions.ApplicationStateChangeResponseFunction ascrf) {
+        this.ccs = ccs;
+        this.ascrf = ascrf;
+    }
+
+    @Override
+    public void run() {
+        final CCApplicationContext appCtx = ccs.getApplicationMap().get(ascrf.getApplicationName());
+        if (appCtx == null) {
+            LOGGER.warning("Got ApplicationStateChangeResponse for application " + ascrf.getApplicationName()
+                    + " that does not exist");
+            return;
+        }
+        switch (ascrf.getStatus()) {
+            case INITIALIZED: {
+                Set<String> pendingNodeIds = appCtx.getInitializationPendingNodeIds();
+                boolean changed = pendingNodeIds.remove(ascrf.getNodeId());
+                if (!changed) {
+                    LOGGER.warning("Got ApplicationStateChangeResponse for application " + ascrf.getApplicationName()
+                            + " from unexpected node " + ascrf.getNodeId() + " to state " + ascrf.getStatus());
+                    return;
+                }
+                if (pendingNodeIds.isEmpty()) {
+                    appCtx.setStatus(ApplicationStatus.INITIALIZED);
+                    IResultCallback<Object> callback = appCtx.getInitializationCallback();
+                    appCtx.setInitializationCallback(null);
+                    callback.setValue(null);
+                }
+                return;
+            }
+
+            case DEINITIALIZED: {
+                Set<String> pendingNodeIds = appCtx.getDeinitializationPendingNodeIds();
+                boolean changed = pendingNodeIds.remove(ascrf.getNodeId());
+                if (!changed) {
+                    LOGGER.warning("Got ApplicationStateChangeResponse for application " + ascrf.getApplicationName()
+                            + " from unexpected node " + ascrf.getNodeId() + " to state " + ascrf.getStatus());
+                    return;
+                }
+                if (pendingNodeIds.isEmpty()) {
+                    appCtx.setStatus(ApplicationStatus.DEINITIALIZED);
+                    ccs.getExecutor().execute(new Runnable() {
+                        @Override
+                        public void run() {
+                            try {
+                                appCtx.deinitialize();
+                            } catch (Exception e) {
+                                e.printStackTrace();
+                            }
+                            ccs.getWorkQueue().schedule(new AbstractWork() {
+                                @Override
+                                public void run() {
+                                    ccs.getApplicationMap().remove(ascrf.getApplicationName());
+                                    IResultCallback<Object> callback = appCtx.getDeinitializationCallback();
+                                    appCtx.setDeinitializationCallback(null);
+                                    callback.setValue(null);
+                                }
+                            });
+                        }
+                    });
+                }
+                return;
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusConditionVariableWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetIpAddressNodeNameMapWork.java
similarity index 60%
rename from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusConditionVariableWork.java
rename to hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetIpAddressNodeNameMapWork.java
index ff67928..1f29903 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusConditionVariableWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetIpAddressNodeNameMapWork.java
@@ -14,30 +14,23 @@
  */
 package edu.uci.ics.hyracks.control.cc.work;
 
-import edu.uci.ics.hyracks.api.job.JobId;
+import java.util.Map;
+import java.util.Set;
+
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.IJobStatusConditionVariable;
 import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
 
-public class GetJobStatusConditionVariableWork extends SynchronizableWork {
+public class GetIpAddressNodeNameMapWork extends SynchronizableWork {
     private final ClusterControllerService ccs;
-    private final JobId jobId;
-    private IJobStatusConditionVariable cVar;
+    private Map<String, Set<String>> map;
 
-    public GetJobStatusConditionVariableWork(ClusterControllerService ccs, JobId jobId) {
+    public GetIpAddressNodeNameMapWork(ClusterControllerService ccs, Map<String, Set<String>> map) {
         this.ccs = ccs;
-        this.jobId = jobId;
+        this.map = map;
     }
 
     @Override
     protected void doRun() throws Exception {
-        cVar = ccs.getActiveRunMap().get(jobId);
-        if (cVar == null) {
-            cVar = ccs.getRunMapArchive().get(jobId);
-        }
-    }
-
-    public IJobStatusConditionVariable getConditionVariable() {
-        return cVar;
+        map.putAll(ccs.getIpAddressNodeNameMap());
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusWork.java
index 7a8943a..c70e24f 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusWork.java
@@ -18,28 +18,31 @@
 import edu.uci.ics.hyracks.api.job.JobStatus;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
 import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
 
 public class GetJobStatusWork extends SynchronizableWork {
     private final ClusterControllerService ccs;
     private final JobId jobId;
-    private JobStatus status;
+    private final IResultCallback<JobStatus> callback;
 
-    public GetJobStatusWork(ClusterControllerService ccs, JobId jobId) {
+    public GetJobStatusWork(ClusterControllerService ccs, JobId jobId, IResultCallback<JobStatus> callback) {
         this.ccs = ccs;
         this.jobId = jobId;
+        this.callback = callback;
     }
 
     @Override
     protected void doRun() throws Exception {
-        JobRun run = ccs.getActiveRunMap().get(jobId);
-        if (run == null) {
-            run = ccs.getRunMapArchive().get(jobId);
+        try {
+            JobRun run = ccs.getActiveRunMap().get(jobId);
+            if (run == null) {
+                run = ccs.getRunMapArchive().get(jobId);
+            }
+            JobStatus status = run == null ? null : run.getStatus();
+            callback.setValue(status);
+        } catch (Exception e) {
+            callback.setException(e);
         }
-        status = run == null ? null : run.getStatus();
-    }
-
-    public JobStatus getStatus() {
-        return status;
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeControllersInfoWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeControllersInfoWork.java
index 48d9b84..49a3be9 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeControllersInfoWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeControllersInfoWork.java
@@ -22,15 +22,16 @@
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.NodeControllerState;
 import edu.uci.ics.hyracks.control.common.work.AbstractWork;
-import edu.uci.ics.hyracks.control.common.work.FutureValue;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
 
 public class GetNodeControllersInfoWork extends AbstractWork {
     private final ClusterControllerService ccs;
-    private FutureValue<Map<String, NodeControllerInfo>> fv;
+    private IResultCallback<Map<String, NodeControllerInfo>> callback;
 
-    public GetNodeControllersInfoWork(ClusterControllerService ccs, FutureValue<Map<String, NodeControllerInfo>> fv) {
+    public GetNodeControllersInfoWork(ClusterControllerService ccs,
+            IResultCallback<Map<String, NodeControllerInfo>> callback) {
         this.ccs = ccs;
-        this.fv = fv;
+        this.callback = callback;
     }
 
     @Override
@@ -41,6 +42,6 @@
             result.put(e.getKey(), new NodeControllerInfo(e.getKey(), e.getValue().getDataPort().getIpAddress(),
                     NodeStatus.ALIVE));
         }
-        fv.setValue(result);
+        callback.setValue(result);
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
index e30a718..07088bb 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
@@ -20,9 +20,8 @@
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.api.job.JobStatus;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteRunner;
-import edu.uci.ics.hyracks.control.cc.remote.ops.JobCompleteNotifier;
 import edu.uci.ics.hyracks.control.common.work.AbstractWork;
 
 public class JobCleanupWork extends AbstractWork {
@@ -54,22 +53,13 @@
         Set<String> targetNodes = run.getParticipatingNodeIds();
         run.getCleanupPendingNodeIds().addAll(targetNodes);
         run.setPendingStatus(status, exception);
-        final JobCompleteNotifier[] jcns = new JobCompleteNotifier[targetNodes.size()];
-        int i = 0;
         for (String n : targetNodes) {
-            jcns[i++] = new JobCompleteNotifier(n, jobId, status);
-        }
-        ccs.getExecutor().execute(new Runnable() {
-            @Override
-            public void run() {
-                if (jcns.length > 0) {
-                    try {
-                        RemoteRunner.runRemote(ccs, jcns, null);
-                    } catch (Exception e) {
-                        e.printStackTrace();
-                    }
-                }
+            NodeControllerState ncs = ccs.getNodeMap().get(n);
+            try {
+                ncs.getNodeController().cleanUpJoblet(jobId, status);
+            } catch (Exception e) {
+                e.printStackTrace();
             }
-        });
+        }
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java
index 14947cd..b7cf629 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java
@@ -30,6 +30,7 @@
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
 import edu.uci.ics.hyracks.control.cc.job.PlanUtils;
 import edu.uci.ics.hyracks.control.cc.scheduler.JobScheduler;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
 import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
 
 public class JobCreateWork extends SynchronizableWork {
@@ -38,42 +39,50 @@
     private final EnumSet<JobFlag> jobFlags;
     private final JobId jobId;
     private final String appName;
+    private final IResultCallback<JobId> callback;
 
     public JobCreateWork(ClusterControllerService ccs, JobId jobId, String appName, byte[] jobSpec,
-            EnumSet<JobFlag> jobFlags) {
+            EnumSet<JobFlag> jobFlags, IResultCallback<JobId> callback) {
         this.jobId = jobId;
         this.ccs = ccs;
         this.jobSpec = jobSpec;
         this.jobFlags = jobFlags;
         this.appName = appName;
+        this.callback = callback;
     }
 
     @Override
     protected void doRun() throws Exception {
-        CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
-        if (appCtx == null) {
-            throw new HyracksException("No application with id " + appName + " found");
-        }
-        JobSpecification spec = appCtx.createJobSpecification(jobSpec);
-
-        final JobActivityGraphBuilder builder = new JobActivityGraphBuilder();
-        builder.init(appName, spec, jobFlags);
-        PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
-            @Override
-            public void visit(IOperatorDescriptor op) {
-                op.contributeActivities(builder);
+        try {
+            CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
+            if (appCtx == null) {
+                throw new HyracksException("No application with id " + appName + " found");
             }
-        });
-        final JobActivityGraph jag = builder.getActivityGraph();
+            JobSpecification spec = appCtx.createJobSpecification(jobSpec);
 
-        JobRun run = new JobRun(jobId, jag);
+            final JobActivityGraphBuilder builder = new JobActivityGraphBuilder();
+            builder.init(appName, spec, jobFlags);
+            PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
+                @Override
+                public void visit(IOperatorDescriptor op) {
+                    op.contributeActivities(builder);
+                }
+            });
+            final JobActivityGraph jag = builder.getActivityGraph();
 
-        run.setStatus(JobStatus.INITIALIZED, null);
+            JobRun run = new JobRun(jobId, jag);
 
-        ccs.getActiveRunMap().put(jobId, run);
-        JobScheduler jrs = new JobScheduler(ccs, run);
-        run.setScheduler(jrs);
-        appCtx.notifyJobCreation(jobId, spec);
+            run.setStatus(JobStatus.INITIALIZED, null);
+
+            ccs.getActiveRunMap().put(jobId, run);
+            JobScheduler jrs = new JobScheduler(ccs, run);
+            run.setScheduler(jrs);
+            appCtx.notifyJobCreation(jobId, spec);
+            callback.setValue(jobId);
+        } catch (Exception e) {
+            callback.setException(e);
+            return;
+        }
     }
 
     public JobId getJobId() {
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
index 855e3e7..dd834ed 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
@@ -18,31 +18,39 @@
 import edu.uci.ics.hyracks.api.job.JobStatus;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
 import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
 
 public class JobStartWork extends SynchronizableWork {
     private final ClusterControllerService ccs;
     private final JobId jobId;
+    private final IResultCallback<Object> callback;
 
-    public JobStartWork(ClusterControllerService ccs, JobId jobId) {
+    public JobStartWork(ClusterControllerService ccs, JobId jobId, IResultCallback<Object> callback) {
         this.ccs = ccs;
         this.jobId = jobId;
+        this.callback = callback;
     }
 
     @Override
     protected void doRun() throws Exception {
-        JobRun run = ccs.getActiveRunMap().get(jobId);
-        if (run == null) {
-            throw new Exception("Unable to find job with id = " + jobId);
-        }
-        if (run.getStatus() != JobStatus.INITIALIZED) {
-            throw new Exception("Job already started");
-        }
-        run.setStatus(JobStatus.RUNNING, null);
         try {
-            run.getScheduler().startJob();
+            JobRun run = ccs.getActiveRunMap().get(jobId);
+            if (run == null) {
+                throw new Exception("Unable to find job with id = " + jobId);
+            }
+            if (run.getStatus() != JobStatus.INITIALIZED) {
+                throw new Exception("Job already started");
+            }
+            run.setStatus(JobStatus.RUNNING, null);
+            try {
+                run.getScheduler().startJob();
+            } catch (Exception e) {
+                ccs.getWorkQueue().schedule(new JobCleanupWork(ccs, run.getJobId(), JobStatus.FAILURE, e));
+            }
+            callback.setValue(null);
         } catch (Exception e) {
-            ccs.getWorkQueue().schedule(new JobCleanupWork(ccs, run.getJobId(), JobStatus.FAILURE, e));
+            callback.setException(e);
         }
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterNodeWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterNodeWork.java
index 15daff5..cba4b4b 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterNodeWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterNodeWork.java
@@ -14,29 +14,65 @@
  */
 package edu.uci.ics.hyracks.control.cc.work;
 
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
+import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
+import edu.uci.ics.hyracks.control.common.ipc.NodeControllerFunctions;
+import edu.uci.ics.hyracks.control.common.ipc.NodeControllerRemoteProxy;
 import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
 
 public class RegisterNodeWork extends SynchronizableWork {
-    private final ClusterControllerService ccs;
-    private final String nodeId;
-    private final NodeControllerState state;
+    private static final Logger LOGGER = Logger.getLogger(RegisterNodeWork.class.getName());
 
-    public RegisterNodeWork(ClusterControllerService ccs, String nodeId, NodeControllerState state) {
+    private final ClusterControllerService ccs;
+    private final NodeRegistration reg;
+
+    public RegisterNodeWork(ClusterControllerService ccs, NodeRegistration reg) {
         this.ccs = ccs;
-        this.nodeId = nodeId;
-        this.state = state;
+        this.reg = reg;
     }
 
     @Override
     protected void doRun() throws Exception {
-        Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
-        if (nodeMap.containsKey(nodeId)) {
-            throw new Exception("Node with this name already registered.");
+        String id = reg.getNodeId();
+
+        IIPCHandle ncIPCHandle = ccs.getClusterIPC().getHandle(reg.getNodeControllerAddress());
+        NodeControllerFunctions.NodeRegistrationResult result = null;
+        try {
+            INodeController nodeController = new NodeControllerRemoteProxy(ncIPCHandle);
+
+            NodeControllerState state = new NodeControllerState(nodeController, reg);
+            Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+            if (nodeMap.containsKey(id)) {
+                throw new Exception("Node with this name already registered.");
+            }
+            nodeMap.put(id, state);
+            Map<String, Set<String>> ipAddressNodeNameMap = ccs.getIpAddressNodeNameMap();
+            String ipAddress = state.getNCConfig().dataIPAddress;
+            Set<String> nodes = ipAddressNodeNameMap.get(ipAddress);
+            if (nodes == null) {
+                nodes = new HashSet<String>();
+                ipAddressNodeNameMap.put(ipAddress, nodes);
+            }
+            nodes.add(id);
+            LOGGER.log(Level.INFO, "Registered INodeController: id = " + id);
+            NodeParameters params = new NodeParameters();
+            params.setClusterControllerInfo(ccs.getClusterControllerInfo());
+            params.setHeartbeatPeriod(ccs.getCCConfig().heartbeatPeriod);
+            params.setProfileDumpPeriod(ccs.getCCConfig().profileDumpPeriod);
+            result = new NodeControllerFunctions.NodeRegistrationResult(params, null);
+        } catch (Exception e) {
+            result = new NodeControllerFunctions.NodeRegistrationResult(null, e);
         }
-        nodeMap.put(nodeId, state);
+        ncIPCHandle.send(-1, result, null);
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
index 87b5878..2a844ad 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
@@ -16,8 +16,9 @@
 
 import java.util.List;
 
+import org.apache.commons.lang3.tuple.Pair;
+
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
 import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
@@ -46,7 +47,11 @@
         List<Pair<PartitionDescriptor, PartitionRequest>> matches = pmm
                 .registerPartitionDescriptor(partitionDescriptor);
         for (Pair<PartitionDescriptor, PartitionRequest> match : matches) {
-            PartitionUtils.reportPartitionMatch(ccs, pid, match);
+            try {
+                PartitionUtils.reportPartitionMatch(ccs, pid, match);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
         }
     }
 
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionRequestWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionRequestWork.java
index 19716a4..a71875f 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionRequestWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionRequestWork.java
@@ -14,8 +14,9 @@
  */
 package edu.uci.ics.hyracks.control.cc.work;
 
+import org.apache.commons.lang3.tuple.Pair;
+
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
 import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
 import edu.uci.ics.hyracks.control.cc.job.JobRun;
 import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
@@ -43,7 +44,11 @@
         PartitionMatchMaker pmm = run.getPartitionMatchMaker();
         Pair<PartitionDescriptor, PartitionRequest> match = pmm.matchPartitionRequest(partitionRequest);
         if (match != null) {
-            PartitionUtils.reportPartitionMatch(ccs, pid, match);
+            try {
+                PartitionUtils.reportPartitionMatch(ccs, pid, match);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
         }
     }
 
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
index 2838ed4..2694af4 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
@@ -39,7 +39,7 @@
     @Override
     protected void performEvent(TaskAttempt ta) {
         try {
-            ActivityCluster ac = ta.getTaskState().getTaskCluster().getActivityCluster();
+            ActivityCluster ac = ta.getTask().getTaskCluster().getActivityCluster();
             JobRun run = ac.getJobRun();
             if (statistics != null) {
                 JobProfile jobProfile = run.getJobProfile();
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
index b80d928..3201bec 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
@@ -31,7 +31,7 @@
 
     @Override
     protected void performEvent(TaskAttempt ta) {
-        ActivityCluster ac = ta.getTaskState().getTaskCluster().getActivityCluster();
+        ActivityCluster ac = ta.getTask().getTaskCluster().getActivityCluster();
         ac.getJobRun().getScheduler().notifyTaskFailure(ta, ac, details);
     }
 
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/WaitForJobCompletionWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/WaitForJobCompletionWork.java
new file mode 100644
index 0000000..6cfe025
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/WaitForJobCompletionWork.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.IJobStatusConditionVariable;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class WaitForJobCompletionWork extends SynchronizableWork {
+    private final ClusterControllerService ccs;
+    private final JobId jobId;
+    private final IResultCallback<Object> callback;
+
+    public WaitForJobCompletionWork(ClusterControllerService ccs, JobId jobId, IResultCallback<Object> callback) {
+        this.ccs = ccs;
+        this.jobId = jobId;
+        this.callback = callback;
+    }
+
+    @Override
+    protected void doRun() throws Exception {
+        final IJobStatusConditionVariable cRunningVar = ccs.getActiveRunMap().get(jobId);
+        if (cRunningVar != null) {
+            ccs.getExecutor().execute(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        cRunningVar.waitForCompletion();
+                        callback.setValue(null);
+                    } catch (Exception e) {
+                        callback.setException(e);
+                    }
+                }
+            });
+        } else {
+            final IJobStatusConditionVariable cArchivedVar = ccs.getRunMapArchive().get(jobId);
+            ccs.getExecutor().execute(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        cArchivedVar.waitForCompletion();
+                        callback.setValue(null);
+                    } catch (Exception e) {
+                        callback.setException(e);
+                    }
+                }
+            });
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.html b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.html
index f11e480..d2068fd 100644
--- a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.html
+++ b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.html
@@ -5,5 +5,5 @@
     </div>
     <div id="job-run" wicket:id="job-run" style="display: none;">
     </div>
-    <div wicket:id="job-timeline"></div>
+    <div wicket:id="job-timeline" style="overflow: auto;"></div>
 </wicket:extend>
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java
index cf64554..21e3e9d 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java
@@ -14,14 +14,9 @@
  */
 package edu.uci.ics.hyracks.control.common;
 
-import java.rmi.RemoteException;
-import java.rmi.server.UnicastRemoteObject;
-
 import edu.uci.ics.hyracks.control.common.service.IService;
 
-public abstract class AbstractRemoteService extends UnicastRemoteObject implements IService {
-    private static final long serialVersionUID = 1L;
-
-    public AbstractRemoteService() throws RemoteException {
+public abstract class AbstractRemoteService implements IService {
+    public AbstractRemoteService() {
     }
-}
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
index d2a06be..9f29fbd 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
@@ -59,7 +59,6 @@
         this.serverCtx = serverCtx;
         this.appName = appName;
         this.applicationRootDir = new File(new File(serverCtx.getBaseDir(), APPLICATION_ROOT), appName);
-        status = ApplicationStatus.CREATED;
         FileUtils.deleteDirectory(applicationRootDir);
         applicationRootDir.mkdirs();
     }
@@ -87,9 +86,6 @@
     }
 
     public void initialize() throws Exception {
-        if (status != ApplicationStatus.CREATED) {
-            throw new IllegalStateException();
-        }
         if (deploymentDescriptor != null) {
             String bootstrapClass = null;
             switch (serverCtx.getServerType()) {
@@ -107,7 +103,6 @@
                 start();
             }
         }
-        status = ApplicationStatus.INITIALIZED;
     }
 
     protected abstract void start() throws Exception;
@@ -168,7 +163,6 @@
     }
 
     public void deinitialize() throws Exception {
-        status = ApplicationStatus.DEINITIALIZED;
         stop();
         File expandedFolder = getExpandedFolder();
         FileUtils.deleteDirectory(expandedFolder);
@@ -203,4 +197,12 @@
     public ClassLoader getClassLoader() {
         return classLoader;
     }
-}
+
+    public void setStatus(ApplicationStatus status) {
+        this.status = status;
+    }
+
+    public ApplicationStatus getStatus() {
+        return status;
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java
index 8a97ad6..b45c4f1 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java
@@ -16,6 +16,8 @@
 
 public enum ApplicationStatus {
     CREATED,
+    IN_INITIALIZATION,
     INITIALIZED,
+    IN_DEINITIALIZATION,
     DEINITIALIZED
 }
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
index 53003d0..a03f5c9 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
@@ -14,12 +14,11 @@
  */
 package edu.uci.ics.hyracks.control.common.base;
 
-import java.rmi.Remote;
 import java.util.List;
 
 import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
 import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
 import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
 import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
 import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
@@ -27,10 +26,10 @@
 import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
 import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
 
-public interface IClusterController extends Remote {
-    public NodeParameters registerNode(NodeRegistration reg) throws Exception;
+public interface IClusterController {
+    public void registerNode(NodeRegistration reg) throws Exception;
 
-    public void unregisterNode(INodeController nodeController) throws Exception;
+    public void unregisterNode(String nodeId) throws Exception;
 
     public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics)
             throws Exception;
@@ -46,4 +45,6 @@
     public void registerPartitionProvider(PartitionDescriptor partitionDescriptor) throws Exception;
 
     public void registerPartitionRequest(PartitionRequest partitionRequest) throws Exception;
+
+    public void notifyApplicationStateChange(String nodeId, String appName, ApplicationStatus status) throws Exception;
 }
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
index fdf49e9..47f4ceb 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
@@ -14,7 +14,6 @@
  */
 package edu.uci.ics.hyracks.control.common.base;
 
-import java.rmi.Remote;
 import java.util.List;
 import java.util.Map;
 
@@ -25,23 +24,16 @@
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.api.job.JobStatus;
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
 import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
 
-public interface INodeController extends Remote {
-    public String getId() throws Exception;
-
-    public NCConfig getConfiguration() throws Exception;
-
+public interface INodeController {
     public void startTasks(String appName, JobId jobId, byte[] planBytes, List<TaskAttemptDescriptor> taskDescriptors,
-            Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies, byte[] ctxVarBytes) throws Exception;
+            Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) throws Exception;
 
     public void abortTasks(JobId jobId, List<TaskAttemptId> tasks) throws Exception;
 
     public void cleanUpJoblet(JobId jobId, JobStatus status) throws Exception;
 
-    public void notifyRegistration(IClusterController ccs) throws Exception;
-
     public void createApplication(String appName, boolean deployHar, byte[] serializedDistributedState)
             throws Exception;
 
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
index 5091d12..4536fc4 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
@@ -19,8 +19,17 @@
 import org.kohsuke.args4j.Option;
 
 public class CCConfig {
-    @Option(name = "-port", usage = "Sets the port to listen for connections from node controllers (default 1099)")
-    public int port = 1099;
+    @Option(name = "-client-net-ip-address", usage = "Sets the IP Address to listen for connections from clients", required = true)
+    public String clientNetIpAddress;
+
+    @Option(name = "-client-net-port", usage = "Sets the port to listen for connections from clients (default 1098)")
+    public int clientNetPort = 1098;
+
+    @Option(name = "-cluster-net-ip-address", usage = "Sets the IP Address to listen for connections from ", required = true)
+    public String clusterNetIpAddress;
+
+    @Option(name = "-cluster-net-port", usage = "Sets the port to listen for connections from node controllers (default 1099)")
+    public int clusterNetPort = 1099;
 
     @Option(name = "-http-port", usage = "Sets the http port for the Cluster Controller (default: 19001)")
     public int httpPort = 19001;
@@ -44,8 +53,14 @@
     public String ccRoot = "ClusterControllerService";
 
     public void toCommandLine(List<String> cList) {
-        cList.add("-port");
-        cList.add(String.valueOf(port));
+        cList.add("-client-net-ip-address");
+        cList.add(clientNetIpAddress);
+        cList.add("-client-net-port");
+        cList.add(String.valueOf(clientNetPort));
+        cList.add("-cluster-net-ip-address");
+        cList.add(clusterNetIpAddress);
+        cList.add("-cluster-net-port");
+        cList.add(String.valueOf(clusterNetPort));
         cList.add("-http-port");
         cList.add(String.valueOf(httpPort));
         cList.add("-heartbeat-period");
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
index 02ce0f8..574f552 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
@@ -22,16 +22,19 @@
 public class NCConfig implements Serializable {
     private static final long serialVersionUID = 1L;
 
-    @Option(name = "-cc-host", usage = "Cluster Controller host name")
+    @Option(name = "-cc-host", usage = "Cluster Controller host name", required = true)
     public String ccHost;
 
     @Option(name = "-cc-port", usage = "Cluster Controller port (default: 1099)")
     public int ccPort = 1099;
 
-    @Option(name = "-node-id", usage = "Logical name of node controller unique within the cluster")
+    @Option(name = "-cluster-net-ip-address", usage = "IP Address to bind cluster listener", required = true)
+    public String clusterNetIPAddress;
+
+    @Option(name = "-node-id", usage = "Logical name of node controller unique within the cluster", required = true)
     public String nodeId;
 
-    @Option(name = "-data-ip-address", usage = "IP Address to bind data listener")
+    @Option(name = "-data-ip-address", usage = "IP Address to bind data listener", required = true)
     public String dataIPAddress;
 
     @Option(name = "-frame-size", usage = "Frame Size to use for data communication (default: 32768)")
@@ -49,11 +52,16 @@
     @Option(name = "-dcache-client-path", usage = "Sets the path to store the files retrieved from the DCache server (default /tmp/dcache-client)")
     public String dcacheClientPath = "/tmp/dcache-client";
 
+    @Option(name = "-net-thread-count", usage = "Number of threads to use for Network I/O (default: 1)")
+    public int nNetThreads = 1;
+
     public void toCommandLine(List<String> cList) {
         cList.add("-cc-host");
         cList.add(ccHost);
         cList.add("-cc-port");
         cList.add(String.valueOf(ccPort));
+        cList.add("-cluster-net-ip-address");
+        cList.add(clusterNetIPAddress);
         cList.add("-node-id");
         cList.add(nodeId);
         cList.add("-data-ip-address");
@@ -70,5 +78,7 @@
         }
         cList.add("-dcache-client-path");
         cList.add(dcacheClientPath);
+        cList.add("-net-thread-count");
+        cList.add(String.valueOf(nNetThreads));
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
index ff92c2d..f6dde10 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
@@ -15,15 +15,15 @@
 package edu.uci.ics.hyracks.control.common.controllers;
 
 import java.io.Serializable;
+import java.net.InetSocketAddress;
 
 import edu.uci.ics.hyracks.api.comm.NetworkAddress;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
 import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
 
 public final class NodeRegistration implements Serializable {
     private static final long serialVersionUID = 1L;
 
-    private final INodeController nc;
+    private final InetSocketAddress ncAddress;
 
     private final String nodeId;
 
@@ -41,9 +41,9 @@
 
     private final HeartbeatSchema hbSchema;
 
-    public NodeRegistration(INodeController nc, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
+    public NodeRegistration(InetSocketAddress ncAddress, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
             String osName, String arch, String osVersion, int nProcessors, HeartbeatSchema hbSchema) {
-        this.nc = nc;
+        this.ncAddress = ncAddress;
         this.nodeId = nodeId;
         this.ncConfig = ncConfig;
         this.dataPort = dataPort;
@@ -54,8 +54,8 @@
         this.hbSchema = hbSchema;
     }
 
-    public INodeController getNodeController() {
-        return nc;
+    public InetSocketAddress getNodeControllerAddress() {
+        return ncAddress;
     }
 
     public String getNodeId() {
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
index cd9086f..8ca0fa4 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
@@ -33,4 +33,8 @@
     public double systemLoadAverage;
     public long[] gcCollectionCounts;
     public long[] gcCollectionTimes;
+    public long netPayloadBytesRead;
+    public long netPayloadBytesWritten;
+    public long netSignalingBytesRead;
+    public long netSignalingBytesWritten;
 }
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerFunctions.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerFunctions.java
new file mode 100644
index 0000000..2b6792e
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerFunctions.java
@@ -0,0 +1,305 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.ipc;
+
+import java.io.Serializable;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
+import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+
+public class ClusterControllerFunctions {
+    public enum FunctionId {
+        REGISTER_NODE,
+        UNREGISTER_NODE,
+        NOTIFY_TASK_COMPLETE,
+        NOTIFY_TASK_FAILURE,
+        NOTIFY_JOBLET_CLEANUP,
+        NODE_HEARTBEAT,
+        REPORT_PROFILE,
+        REGISTER_PARTITION_PROVIDER,
+        REGISTER_PARTITION_REQUEST,
+        APPLICATION_STATE_CHANGE_RESPONSE,
+    }
+
+    public static abstract class Function implements Serializable {
+        private static final long serialVersionUID = 1L;
+
+        public abstract FunctionId getFunctionId();
+    }
+
+    public static class RegisterNodeFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final NodeRegistration reg;
+
+        public RegisterNodeFunction(NodeRegistration reg) {
+            this.reg = reg;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.REGISTER_NODE;
+        }
+
+        public NodeRegistration getNodeRegistration() {
+            return reg;
+        }
+    }
+
+    public static class UnregisterNodeFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String nodeId;
+
+        public UnregisterNodeFunction(String nodeId) {
+            this.nodeId = nodeId;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.UNREGISTER_NODE;
+        }
+
+        public String getNodeId() {
+            return nodeId;
+        }
+    }
+
+    public static class NotifyTaskCompleteFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+        private final TaskAttemptId taskId;
+        private final String nodeId;
+        private final TaskProfile statistics;
+
+        public NotifyTaskCompleteFunction(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics) {
+            this.jobId = jobId;
+            this.taskId = taskId;
+            this.nodeId = nodeId;
+            this.statistics = statistics;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.NOTIFY_TASK_COMPLETE;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+
+        public TaskAttemptId getTaskId() {
+            return taskId;
+        }
+
+        public String getNodeId() {
+            return nodeId;
+        }
+
+        public TaskProfile getStatistics() {
+            return statistics;
+        }
+    }
+
+    public static class NotifyTaskFailureFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+        private final TaskAttemptId taskId;
+        private final String nodeId;
+        private final String details;
+
+        public NotifyTaskFailureFunction(JobId jobId, TaskAttemptId taskId, String nodeId, String details) {
+            this.jobId = jobId;
+            this.taskId = taskId;
+            this.nodeId = nodeId;
+            this.details = details;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.NOTIFY_TASK_FAILURE;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+
+        public TaskAttemptId getTaskId() {
+            return taskId;
+        }
+
+        public String getNodeId() {
+            return nodeId;
+        }
+
+        public String getDetails() {
+            return details;
+        }
+    }
+
+    public static class NotifyJobletCleanupFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+        private final String nodeId;
+
+        public NotifyJobletCleanupFunction(JobId jobId, String nodeId) {
+            this.jobId = jobId;
+            this.nodeId = nodeId;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.NOTIFY_JOBLET_CLEANUP;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+
+        public String getNodeId() {
+            return nodeId;
+        }
+    }
+
+    public static class NodeHeartbeatFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String nodeId;
+        private final HeartbeatData hbData;
+
+        public NodeHeartbeatFunction(String nodeId, HeartbeatData hbData) {
+            this.nodeId = nodeId;
+            this.hbData = hbData;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.NODE_HEARTBEAT;
+        }
+
+        public String getNodeId() {
+            return nodeId;
+        }
+
+        public HeartbeatData getHeartbeatData() {
+            return hbData;
+        }
+    }
+
+    public static class ReportProfileFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String nodeId;
+        private final List<JobProfile> profiles;
+
+        public ReportProfileFunction(String nodeId, List<JobProfile> profiles) {
+            this.nodeId = nodeId;
+            this.profiles = profiles;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.REPORT_PROFILE;
+        }
+
+        public String getNodeId() {
+            return nodeId;
+        }
+
+        public List<JobProfile> getProfiles() {
+            return profiles;
+        }
+    }
+
+    public static class RegisterPartitionProviderFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final PartitionDescriptor partitionDescriptor;
+
+        public RegisterPartitionProviderFunction(PartitionDescriptor partitionDescriptor) {
+            this.partitionDescriptor = partitionDescriptor;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.REGISTER_PARTITION_PROVIDER;
+        }
+
+        public PartitionDescriptor getPartitionDescriptor() {
+            return partitionDescriptor;
+        }
+    }
+
+    public static class RegisterPartitionRequestFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final PartitionRequest partitionRequest;
+
+        public RegisterPartitionRequestFunction(PartitionRequest partitionRequest) {
+            this.partitionRequest = partitionRequest;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.REGISTER_PARTITION_REQUEST;
+        }
+
+        public PartitionRequest getPartitionRequest() {
+            return partitionRequest;
+        }
+    }
+
+    public static class ApplicationStateChangeResponseFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String nodeId;
+        private final String appName;
+        private final ApplicationStatus status;
+
+        public ApplicationStateChangeResponseFunction(String nodeId, String appName, ApplicationStatus status) {
+            this.nodeId = nodeId;
+            this.appName = appName;
+            this.status = status;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.APPLICATION_STATE_CHANGE_RESPONSE;
+        }
+
+        public String getNodeId() {
+            return nodeId;
+        }
+
+        public String getApplicationName() {
+            return appName;
+        }
+
+        public ApplicationStatus getStatus() {
+            return status;
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
new file mode 100644
index 0000000..b7dd0af
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.ipc;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.base.IClusterController;
+import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
+import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+
+public class ClusterControllerRemoteProxy implements IClusterController {
+    private final IIPCHandle ipcHandle;
+
+    public ClusterControllerRemoteProxy(IIPCHandle ipcHandle) {
+        this.ipcHandle = ipcHandle;
+    }
+
+    @Override
+    public void registerNode(NodeRegistration reg) throws Exception {
+        ClusterControllerFunctions.RegisterNodeFunction fn = new ClusterControllerFunctions.RegisterNodeFunction(reg);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void unregisterNode(String nodeId) throws Exception {
+        ClusterControllerFunctions.UnregisterNodeFunction fn = new ClusterControllerFunctions.UnregisterNodeFunction(
+                nodeId);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics)
+            throws Exception {
+        ClusterControllerFunctions.NotifyTaskCompleteFunction fn = new ClusterControllerFunctions.NotifyTaskCompleteFunction(
+                jobId, taskId, nodeId, statistics);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void notifyTaskFailure(JobId jobId, TaskAttemptId taskId, String nodeId, String details) throws Exception {
+        ClusterControllerFunctions.NotifyTaskFailureFunction fn = new ClusterControllerFunctions.NotifyTaskFailureFunction(
+                jobId, taskId, nodeId, details);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void notifyJobletCleanup(JobId jobId, String nodeId) throws Exception {
+        ClusterControllerFunctions.NotifyJobletCleanupFunction fn = new ClusterControllerFunctions.NotifyJobletCleanupFunction(
+                jobId, nodeId);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void nodeHeartbeat(String id, HeartbeatData hbData) throws Exception {
+        ClusterControllerFunctions.NodeHeartbeatFunction fn = new ClusterControllerFunctions.NodeHeartbeatFunction(id,
+                hbData);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void reportProfile(String id, List<JobProfile> profiles) throws Exception {
+        ClusterControllerFunctions.ReportProfileFunction fn = new ClusterControllerFunctions.ReportProfileFunction(id,
+                profiles);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void registerPartitionProvider(PartitionDescriptor partitionDescriptor) throws Exception {
+        ClusterControllerFunctions.RegisterPartitionProviderFunction fn = new ClusterControllerFunctions.RegisterPartitionProviderFunction(
+                partitionDescriptor);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void registerPartitionRequest(PartitionRequest partitionRequest) throws Exception {
+        ClusterControllerFunctions.RegisterPartitionRequestFunction fn = new ClusterControllerFunctions.RegisterPartitionRequestFunction(
+                partitionRequest);
+        ipcHandle.send(-1, fn, null);
+    }
+
+    @Override
+    public void notifyApplicationStateChange(String nodeId, String appName, ApplicationStatus status) throws Exception {
+        ClusterControllerFunctions.ApplicationStateChangeResponseFunction fn = new ClusterControllerFunctions.ApplicationStateChangeResponseFunction(
+                nodeId, appName, status);
+        ipcHandle.send(-1, fn, null);
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerFunctions.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerFunctions.java
new file mode 100644
index 0000000..b72d4e5
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerFunctions.java
@@ -0,0 +1,243 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.ipc;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
+import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
+
+public class NodeControllerFunctions {
+    public enum FunctionId {
+        NODE_REGISTRATION_RESULT,
+        START_TASKS,
+        ABORT_TASKS,
+        CLEANUP_JOBLET,
+        CREATE_APPLICATION,
+        DESTROY_APPLICATION,
+        REPORT_PARTITION_AVAILABILITY
+    }
+
+    public static abstract class Function implements Serializable {
+        private static final long serialVersionUID = 1L;
+
+        public abstract FunctionId getFunctionId();
+    }
+
+    public static class NodeRegistrationResult extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final NodeParameters params;
+
+        private final Exception exception;
+
+        public NodeRegistrationResult(NodeParameters params, Exception exception) {
+            this.params = params;
+            this.exception = exception;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.NODE_REGISTRATION_RESULT;
+        }
+
+        public NodeParameters getNodeParameters() {
+            return params;
+        }
+
+        public Exception getException() {
+            return exception;
+        }
+    }
+
+    public static class StartTasksFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String appName;
+        private final JobId jobId;
+        private final byte[] planBytes;
+        private final List<TaskAttemptDescriptor> taskDescriptors;
+        private final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies;
+
+        public StartTasksFunction(String appName, JobId jobId, byte[] planBytes,
+                List<TaskAttemptDescriptor> taskDescriptors,
+                Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) {
+            this.appName = appName;
+            this.jobId = jobId;
+            this.planBytes = planBytes;
+            this.taskDescriptors = taskDescriptors;
+            this.connectorPolicies = connectorPolicies;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.START_TASKS;
+        }
+
+        public String getAppName() {
+            return appName;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+
+        public byte[] getPlanBytes() {
+            return planBytes;
+        }
+
+        public List<TaskAttemptDescriptor> getTaskDescriptors() {
+            return taskDescriptors;
+        }
+
+        public Map<ConnectorDescriptorId, IConnectorPolicy> getConnectorPolicies() {
+            return connectorPolicies;
+        }
+    }
+
+    public static class AbortTasksFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+        private final List<TaskAttemptId> tasks;
+
+        public AbortTasksFunction(JobId jobId, List<TaskAttemptId> tasks) {
+            this.jobId = jobId;
+            this.tasks = tasks;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.ABORT_TASKS;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+
+        public List<TaskAttemptId> getTasks() {
+            return tasks;
+        }
+    }
+
+    public static class CleanupJobletFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+        private final JobStatus status;
+
+        public CleanupJobletFunction(JobId jobId, JobStatus status) {
+            this.jobId = jobId;
+            this.status = status;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.CLEANUP_JOBLET;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+
+        public JobStatus getStatus() {
+            return status;
+        }
+    }
+
+    public static class CreateApplicationFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String appName;
+        private final boolean deployHar;
+        private final byte[] serializedDistributedState;
+
+        public CreateApplicationFunction(String appName, boolean deployHar, byte[] serializedDistributedState) {
+            this.appName = appName;
+            this.deployHar = deployHar;
+            this.serializedDistributedState = serializedDistributedState;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.CREATE_APPLICATION;
+        }
+
+        public String getAppName() {
+            return appName;
+        }
+
+        public boolean isDeployHar() {
+            return deployHar;
+        }
+
+        public byte[] getSerializedDistributedState() {
+            return serializedDistributedState;
+        }
+    }
+
+    public static class DestroyApplicationFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final String appName;
+
+        public DestroyApplicationFunction(String appName) {
+            this.appName = appName;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.DESTROY_APPLICATION;
+        }
+
+        public String getAppName() {
+            return appName;
+        }
+    }
+
+    public static class ReportPartitionAvailabilityFunction extends Function {
+        private static final long serialVersionUID = 1L;
+
+        private final PartitionId pid;
+        private final NetworkAddress networkAddress;
+
+        public ReportPartitionAvailabilityFunction(PartitionId pid, NetworkAddress networkAddress) {
+            this.pid = pid;
+            this.networkAddress = networkAddress;
+        }
+
+        @Override
+        public FunctionId getFunctionId() {
+            return FunctionId.REPORT_PARTITION_AVAILABILITY;
+        }
+
+        public PartitionId getPartitionId() {
+            return pid;
+        }
+
+        public NetworkAddress getNetworkAddress() {
+            return networkAddress;
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
new file mode 100644
index 0000000..474fb93
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.ipc;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+
+public class NodeControllerRemoteProxy implements INodeController {
+    private final IIPCHandle ipcHandle;
+
+    public NodeControllerRemoteProxy(IIPCHandle ipcHandle) {
+        this.ipcHandle = ipcHandle;
+    }
+
+    @Override
+    public void startTasks(String appName, JobId jobId, byte[] planBytes, List<TaskAttemptDescriptor> taskDescriptors,
+            Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) throws Exception {
+        NodeControllerFunctions.StartTasksFunction stf = new NodeControllerFunctions.StartTasksFunction(appName, jobId,
+                planBytes, taskDescriptors, connectorPolicies);
+        ipcHandle.send(-1, stf, null);
+    }
+
+    @Override
+    public void abortTasks(JobId jobId, List<TaskAttemptId> tasks) throws Exception {
+        NodeControllerFunctions.AbortTasksFunction atf = new NodeControllerFunctions.AbortTasksFunction(jobId, tasks);
+        ipcHandle.send(-1, atf, null);
+    }
+
+    @Override
+    public void cleanUpJoblet(JobId jobId, JobStatus status) throws Exception {
+        NodeControllerFunctions.CleanupJobletFunction cjf = new NodeControllerFunctions.CleanupJobletFunction(jobId,
+                status);
+        ipcHandle.send(-1, cjf, null);
+    }
+
+    @Override
+    public void createApplication(String appName, boolean deployHar, byte[] serializedDistributedState)
+            throws Exception {
+        NodeControllerFunctions.CreateApplicationFunction caf = new NodeControllerFunctions.CreateApplicationFunction(
+                appName, deployHar, serializedDistributedState);
+        ipcHandle.send(-1, caf, null);
+    }
+
+    @Override
+    public void destroyApplication(String appName) throws Exception {
+        NodeControllerFunctions.DestroyApplicationFunction daf = new NodeControllerFunctions.DestroyApplicationFunction(
+                appName);
+        ipcHandle.send(-1, daf, null);
+    }
+
+    @Override
+    public void reportPartitionAvailability(PartitionId pid, NetworkAddress networkAddress) throws Exception {
+        NodeControllerFunctions.ReportPartitionAvailabilityFunction rpaf = new NodeControllerFunctions.ReportPartitionAvailabilityFunction(
+                pid, networkAddress);
+        ipcHandle.send(-1, rpaf, null);
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/counters/MultiResolutionEventProfiler.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/counters/MultiResolutionEventProfiler.java
new file mode 100644
index 0000000..f8fba7a
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/counters/MultiResolutionEventProfiler.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.job.profiling.counters;
+
+import java.io.Serializable;
+
+public class MultiResolutionEventProfiler implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+    private final int[] times;
+
+    private long offset;
+
+    private int ptr;
+
+    private int resolution;
+
+    private int eventCounter;
+
+    public MultiResolutionEventProfiler(int nSamples) {
+        times = new int[nSamples];
+        offset = -1;
+        ptr = 0;
+        resolution = 1;
+        eventCounter = 0;
+    }
+
+    public void reportEvent() {
+        ++eventCounter;
+        if (eventCounter % resolution != 0) {
+            return;
+        }
+        if (ptr >= times.length) {
+            compact();
+            return;
+        }
+        eventCounter = 0;
+        long time = System.currentTimeMillis();
+        if (offset < 0) {
+            offset = time;
+        }
+        int value = (int) (time - offset);
+        times[ptr++] = value;
+    }
+
+    private void compact() {
+        for (int i = 1; i < ptr / 2; ++i) {
+            times[i] = times[i * 2];
+        }
+        resolution <<= 1;
+        ptr >>= 1;
+    }
+
+    public int getResolution() {
+        return resolution;
+    }
+
+    public int getCount() {
+        return ptr;
+    }
+
+    public int[] getSamples() {
+        return times;
+    }
+
+    public long getOffset() {
+        return offset;
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java
index 5457ac5..46a964a 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java
@@ -3,6 +3,7 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
@@ -32,12 +33,13 @@
     public JSONObject toJSON() throws JSONException {
         JSONObject json = new JSONObject();
 
-        json.put("type", "job-profile");
         json.put("job-id", jobId.toString());
         populateCounters(json);
+        JSONArray jobletsArray = new JSONArray();
         for (JobletProfile p : jobletProfiles.values()) {
-            json.accumulate("joblets", p.toJSON());
+            jobletsArray.put(p.toJSON());
         }
+        json.put("joblets", jobletsArray);
 
         return json;
     }
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java
index 78f885d..0c60006 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java
@@ -47,7 +47,6 @@
     public JSONObject toJSON() throws JSONException {
         JSONObject json = new JSONObject();
 
-        json.put("type", "joblet-profile");
         json.put("node-id", nodeId.toString());
         populateCounters(json);
         JSONArray tasks = new JSONArray();
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/PartitionProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/PartitionProfile.java
index f6568d9..ef61796 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/PartitionProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/PartitionProfile.java
@@ -17,6 +17,7 @@
 import java.io.Serializable;
 
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.profiling.counters.MultiResolutionEventProfiler;
 
 public class PartitionProfile implements Serializable {
     private static final long serialVersionUID = 1L;
@@ -27,13 +28,13 @@
 
     private final long closeTime;
 
-    private final byte[] frameTimes;
+    private final MultiResolutionEventProfiler mrep;
 
-    public PartitionProfile(PartitionId pid, long openTime, long closeTime, byte[] frameTimes) {
+    public PartitionProfile(PartitionId pid, long openTime, long closeTime, MultiResolutionEventProfiler mrep) {
         this.pid = pid;
         this.openTime = openTime;
         this.closeTime = closeTime;
-        this.frameTimes = frameTimes;
+        this.mrep = mrep;
     }
 
     public PartitionId getPartitionId() {
@@ -48,7 +49,7 @@
         return closeTime;
     }
 
-    public byte[] getFrameTimes() {
-        return frameTimes;
+    public MultiResolutionEventProfiler getSamples() {
+        return mrep;
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
index 3764a01..2116f61 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
@@ -14,7 +14,6 @@
  */
 package edu.uci.ics.hyracks.control.common.job.profiling.om;
 
-import java.io.ByteArrayInputStream;
 import java.util.Map;
 
 import org.json.JSONArray;
@@ -23,6 +22,7 @@
 
 import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.profiling.counters.MultiResolutionEventProfiler;
 
 public class TaskProfile extends AbstractProfile {
     private static final long serialVersionUID = 1L;
@@ -48,7 +48,6 @@
     public JSONObject toJSON() throws JSONException {
         JSONObject json = new JSONObject();
 
-        json.put("type", "task-profile");
         json.put("activity-id", taskAttemptId.getTaskId().getActivityId().toString());
         json.put("partition", taskAttemptId.getTaskId().getPartition());
         json.put("attempt", taskAttemptId.getAttempt());
@@ -65,24 +64,17 @@
                 ppObj.put("partition-id", pidObj);
                 ppObj.put("open-time", pp.getOpenTime());
                 ppObj.put("close-time", pp.getCloseTime());
-                JSONArray ftArray = new JSONArray();
-                byte[] ftb = pp.getFrameTimes();
-                ByteArrayInputStream bais = new ByteArrayInputStream(ftb);
-                long value = 0;
-                int vLen = 0;
-                long time = pp.getOpenTime();
-                for (int i = 0; i < ftb.length; ++i) {
-                    byte b = (byte) bais.read();
-                    ++vLen;
-                    value += (((long) (b & 0xef)) << ((vLen - 1) * 7));
-                    if ((b & 0x80) == 0) {
-                        time += value;
-                        ftArray.put(time);
-                        vLen = 0;
-                        value = 0;
-                    }
+                MultiResolutionEventProfiler samples = pp.getSamples();
+                ppObj.put("offset", samples.getOffset());
+                int resolution = samples.getResolution();
+                int sampleCount = samples.getCount();
+                JSONArray ftA = new JSONArray();
+                int[] ft = samples.getSamples();
+                for (int i = 0; i < sampleCount; ++i) {
+                    ftA.put(ft[i]);
                 }
-                ppObj.put("frame-times", ftArray);
+                ppObj.put("frame-times", ftA);
+                ppObj.put("resolution", resolution);
                 pspArray.put(ppObj);
             }
             json.put("partition-send-profile", pspArray);
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/FutureValue.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/FutureValue.java
index 00565b6..7eb4ff6 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/FutureValue.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/FutureValue.java
@@ -14,7 +14,7 @@
  */
 package edu.uci.ics.hyracks.control.common.work;
 
-public class FutureValue<T> {
+public class FutureValue<T> implements IResultCallback<T> {
     private boolean done;
 
     private T value;
@@ -27,6 +27,7 @@
         e = null;
     }
 
+    @Override
     public synchronized void setValue(T value) {
         done = true;
         this.value = value;
@@ -34,6 +35,7 @@
         notifyAll();
     }
 
+    @Override
     public synchronized void setException(Exception e) {
         done = true;
         this.e = e;
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IPCResponder.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IPCResponder.java
new file mode 100644
index 0000000..dcea864
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IPCResponder.java
@@ -0,0 +1,33 @@
+package edu.uci.ics.hyracks.control.common.work;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+
+public class IPCResponder<T> implements IResultCallback<T> {
+    private final IIPCHandle handle;
+
+    private final long rmid;
+
+    public IPCResponder(IIPCHandle handle, long rmid) {
+        this.handle = handle;
+        this.rmid = rmid;
+    }
+
+    @Override
+    public void setValue(T result) {
+        try {
+            handle.send(rmid, result, null);
+        } catch (IPCException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void setException(Exception e) {
+        try {
+            handle.send(rmid, null, e);
+        } catch (IPCException e1) {
+            e1.printStackTrace();
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IResultCallback.java
similarity index 78%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IResultCallback.java
index c728b0b..80c3d76 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IResultCallback.java
@@ -12,10 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.control.common.work;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
+public interface IResultCallback<T> {
+    public void setValue(T result);
 
-    public R getResult();
+    public void setException(Exception e);
 }
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
index f5ae1f4..36f0c49 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
@@ -16,6 +16,8 @@
 
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import edu.uci.ics.hyracks.api.exceptions.HyracksException;
@@ -27,15 +29,21 @@
     private final WorkerThread thread;
     private final Semaphore stopSemaphore;
     private boolean stopped;
+    private final AtomicInteger enqueueCount;
+    private final AtomicInteger dequeueCount;
 
     public WorkQueue() {
         queue = new LinkedBlockingQueue<AbstractWork>();
         thread = new WorkerThread();
         stopSemaphore = new Semaphore(1);
+        enqueueCount = new AtomicInteger();
+        dequeueCount = new AtomicInteger();
     }
 
     public void start() throws HyracksException {
         stopped = false;
+        enqueueCount.set(0);
+        dequeueCount.set(0);
         try {
             stopSemaphore.acquire();
         } catch (InterruptedException e) {
@@ -61,6 +69,10 @@
     }
 
     public void schedule(AbstractWork event) {
+        enqueueCount.incrementAndGet();
+        if (LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("Enqueue: " + enqueueCount);
+        }
         if (LOGGER.isLoggable(event.logLevel())) {
             LOGGER.log(event.logLevel(), "Scheduling: " + event);
         }
@@ -80,7 +92,7 @@
         @Override
         public void run() {
             try {
-                Runnable r;
+                AbstractWork r;
                 while (true) {
                     synchronized (WorkQueue.this) {
                         if (stopped) {
@@ -92,7 +104,14 @@
                     } catch (InterruptedException e) {
                         continue;
                     }
+                    dequeueCount.incrementAndGet();
+                    if (LOGGER.isLoggable(Level.FINE)) {
+                        LOGGER.fine("Dequeue: " + dequeueCount + "/" + enqueueCount);
+                    }
                     try {
+                        if (LOGGER.isLoggable(r.logLevel())) {
+                            LOGGER.log(r.logLevel(), "Executing: " + r);
+                        }
                         r.run();
                     } catch (Exception e) {
                         e.printStackTrace();
diff --git a/hyracks-control-nc/pom.xml b/hyracks-control-nc/pom.xml
index 64a409c..8c56958 100644
--- a/hyracks-control-nc/pom.xml
+++ b/hyracks-control-nc/pom.xml
@@ -1,9 +1,6 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
-  <groupId>edu.uci.ics.hyracks</groupId>
   <artifactId>hyracks-control-nc</artifactId>
-  <version>0.2.0-SNAPSHOT</version>
-
   <parent>
     <groupId>edu.uci.ics.hyracks</groupId>
     <artifactId>hyracks</artifactId>
@@ -37,6 +34,11 @@
   		<type>jar</type>
   		<scope>compile</scope>
   	</dependency>
+  	<dependency>
+  		<groupId>edu.uci.ics.hyracks</groupId>
+  		<artifactId>hyracks-net</artifactId>
+  		<version>0.2.0-SNAPSHOT</version>
+  	</dependency>
   </dependencies>
   <reporting>
     <plugins>
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
index 1a98ebd..07d8ad7 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
@@ -145,7 +145,7 @@
     }
 
     public synchronized void notifyTaskComplete(Task task) throws Exception {
-        taskMap.remove(task);
+        taskMap.remove(task.getTaskAttemptId());
         try {
             TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId(), task.getPartitionSendProfile());
             task.dumpProfile(taskProfile);
@@ -159,7 +159,7 @@
     }
 
     public synchronized void notifyTaskFailed(Task task, String details) throws Exception {
-        taskMap.remove(task);
+        taskMap.remove(task.getTaskAttemptId());
         try {
             nodeController.getClusterController().notifyTaskFailure(jobId, task.getTaskAttemptId(),
                     nodeController.getId(), details);
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
index 3fd10a8..ba1d970 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
@@ -23,8 +23,7 @@
 import java.lang.management.RuntimeMXBean;
 import java.lang.management.ThreadMXBean;
 import java.net.InetAddress;
-import java.rmi.registry.LocateRegistry;
-import java.rmi.registry.Registry;
+import java.net.InetSocketAddress;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.Hashtable;
@@ -40,31 +39,25 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import edu.uci.ics.hyracks.api.comm.NetworkAddress;
 import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
-import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
 import edu.uci.ics.hyracks.api.io.IODeviceHandle;
 import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
 import edu.uci.ics.hyracks.control.common.AbstractRemoteService;
 import edu.uci.ics.hyracks.control.common.base.IClusterController;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
 import edu.uci.ics.hyracks.control.common.context.ServerContext;
 import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
 import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
 import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
 import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
 import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
-import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
+import edu.uci.ics.hyracks.control.common.ipc.ClusterControllerRemoteProxy;
+import edu.uci.ics.hyracks.control.common.ipc.NodeControllerFunctions;
 import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
 import edu.uci.ics.hyracks.control.common.work.FutureValue;
 import edu.uci.ics.hyracks.control.common.work.WorkQueue;
 import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
 import edu.uci.ics.hyracks.control.nc.io.IOManager;
-import edu.uci.ics.hyracks.control.nc.net.ConnectionManager;
+import edu.uci.ics.hyracks.control.nc.net.NetworkManager;
 import edu.uci.ics.hyracks.control.nc.partitions.PartitionManager;
 import edu.uci.ics.hyracks.control.nc.runtime.RootHyracksContext;
 import edu.uci.ics.hyracks.control.nc.work.AbortTasksWork;
@@ -74,26 +67,34 @@
 import edu.uci.ics.hyracks.control.nc.work.DestroyApplicationWork;
 import edu.uci.ics.hyracks.control.nc.work.ReportPartitionAvailabilityWork;
 import edu.uci.ics.hyracks.control.nc.work.StartTasksWork;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.PerformanceCounters;
 
-public class NodeControllerService extends AbstractRemoteService implements INodeController {
+public class NodeControllerService extends AbstractRemoteService {
     private static Logger LOGGER = Logger.getLogger(NodeControllerService.class.getName());
 
-    private static final long serialVersionUID = 1L;
-
     private NCConfig ncConfig;
 
     private final String id;
 
     private final IHyracksRootContext ctx;
 
+    private final IPCSystem ipc;
+
     private final PartitionManager partitionManager;
 
-    private final ConnectionManager connectionManager;
+    private final NetworkManager netManager;
 
     private final WorkQueue queue;
 
     private final Timer timer;
 
+    private boolean registrationPending;
+
+    private Exception registrationException;
+
     private IClusterController ccs;
 
     private final Map<JobId, Joblet> jobletMap;
@@ -122,13 +123,14 @@
         this.ncConfig = ncConfig;
         id = ncConfig.nodeId;
         executor = Executors.newCachedThreadPool();
+        NodeControllerIPCI ipci = new NodeControllerIPCI();
+        ipc = new IPCSystem(new InetSocketAddress(ncConfig.clusterNetIPAddress, 0), ipci);
         this.ctx = new RootHyracksContext(ncConfig.frameSize, new IOManager(getDevices(ncConfig.ioDevices), executor));
         if (id == null) {
             throw new Exception("id not set");
         }
-        connectionManager = new ConnectionManager(ctx, getIpAddress(ncConfig));
         partitionManager = new PartitionManager(this);
-        connectionManager.setPartitionRequestListener(partitionManager);
+        netManager = new NetworkManager(ctx, getIpAddress(ncConfig), partitionManager, ncConfig.nNetThreads);
 
         queue = new WorkQueue();
         jobletMap = new Hashtable<JobId, Joblet>();
@@ -141,6 +143,7 @@
         threadMXBean = ManagementFactory.getThreadMXBean();
         runtimeMXBean = ManagementFactory.getRuntimeMXBean();
         osMXBean = ManagementFactory.getOperatingSystemMXBean();
+        registrationPending = true;
     }
 
     public IHyracksRootContext getRootContext() {
@@ -157,30 +160,48 @@
         return devices;
     }
 
+    private synchronized void setNodeRegistrationResult(NodeParameters parameters, Exception exception) {
+        this.nodeParameters = parameters;
+        this.registrationException = exception;
+        this.registrationPending = false;
+        notifyAll();
+    }
+
     @Override
     public void start() throws Exception {
         LOGGER.log(Level.INFO, "Starting NodeControllerService");
-        connectionManager.start();
-        Registry registry = LocateRegistry.getRegistry(ncConfig.ccHost, ncConfig.ccPort);
-        IClusterController cc = (IClusterController) registry.lookup(IClusterController.class.getName());
+        ipc.start();
+        netManager.start();
+        IIPCHandle ccIPCHandle = ipc.getHandle(new InetSocketAddress(ncConfig.ccHost, ncConfig.ccPort));
+        this.ccs = new ClusterControllerRemoteProxy(ccIPCHandle);
         HeartbeatSchema.GarbageCollectorInfo[] gcInfos = new HeartbeatSchema.GarbageCollectorInfo[gcMXBeans.size()];
         for (int i = 0; i < gcInfos.length; ++i) {
             gcInfos[i] = new HeartbeatSchema.GarbageCollectorInfo(gcMXBeans.get(i).getName());
         }
         HeartbeatSchema hbSchema = new HeartbeatSchema(gcInfos);
-        this.nodeParameters = cc.registerNode(new NodeRegistration(this, id, ncConfig, connectionManager
-                .getNetworkAddress(), osMXBean.getName(), osMXBean.getArch(), osMXBean.getVersion(), osMXBean
-                .getAvailableProcessors(), hbSchema));
+        ccs.registerNode(new NodeRegistration(ipc.getSocketAddress(), id, ncConfig, netManager.getNetworkAddress(),
+                osMXBean.getName(), osMXBean.getArch(), osMXBean.getVersion(), osMXBean.getAvailableProcessors(),
+                hbSchema));
+
+        synchronized (this) {
+            while (registrationPending) {
+                wait();
+            }
+        }
+        if (registrationException != null) {
+            throw registrationException;
+        }
+
         queue.start();
 
-        heartbeatTask = new HeartbeatTask(cc);
+        heartbeatTask = new HeartbeatTask(ccs);
 
         // Schedule heartbeat generator.
         timer.schedule(heartbeatTask, 0, nodeParameters.getHeartbeatPeriod());
 
         if (nodeParameters.getProfileDumpPeriod() > 0) {
             // Schedule profile dump generator.
-            timer.schedule(new ProfileDumpTask(cc), 0, nodeParameters.getProfileDumpPeriod());
+            timer.schedule(new ProfileDumpTask(ccs), 0, nodeParameters.getProfileDumpPeriod());
         }
 
         LOGGER.log(Level.INFO, "Started NodeControllerService");
@@ -191,12 +212,11 @@
         LOGGER.log(Level.INFO, "Stopping NodeControllerService");
         partitionManager.close();
         heartbeatTask.cancel();
-        connectionManager.stop();
+        netManager.stop();
         queue.stop();
         LOGGER.log(Level.INFO, "Stopped NodeControllerService");
     }
 
-    @Override
     public String getId() {
         return id;
     }
@@ -213,8 +233,8 @@
         return jobletMap;
     }
 
-    public ConnectionManager getConnectionManager() {
-        return connectionManager;
+    public NetworkManager getNetworkManager() {
+        return netManager;
     }
 
     public PartitionManager getPartitionManager() {
@@ -233,55 +253,10 @@
         return executor;
     }
 
-    @Override
-    public void startTasks(String appName, final JobId jobId, byte[] jagBytes,
-            List<TaskAttemptDescriptor> taskDescriptors,
-            Map<ConnectorDescriptorId, IConnectorPolicy> connectorPoliciesMap, byte[] ctxVarBytes) throws Exception {
-        StartTasksWork stw = new StartTasksWork(this, appName, jobId, jagBytes, taskDescriptors, connectorPoliciesMap);
-        queue.scheduleAndSync(stw);
-    }
-
-    @Override
-    public void cleanUpJoblet(JobId jobId, JobStatus status) throws Exception {
-        CleanupJobletWork cjw = new CleanupJobletWork(this, jobId, status);
-        queue.scheduleAndSync(cjw);
-    }
-
-    @Override
-    public void notifyRegistration(IClusterController ccs) throws Exception {
-        this.ccs = ccs;
-    }
-
-    @Override
     public NCConfig getConfiguration() throws Exception {
         return ncConfig;
     }
 
-    @Override
-    public synchronized void abortTasks(JobId jobId, List<TaskAttemptId> tasks) throws Exception {
-        AbortTasksWork atw = new AbortTasksWork(this, jobId, tasks);
-        queue.scheduleAndSync(atw);
-    }
-
-    @Override
-    public void createApplication(String appName, boolean deployHar, byte[] serializedDistributedState)
-            throws Exception {
-        CreateApplicationWork caw = new CreateApplicationWork(this, appName, deployHar, serializedDistributedState);
-        queue.scheduleAndSync(caw);
-    }
-
-    @Override
-    public void destroyApplication(String appName) throws Exception {
-        DestroyApplicationWork daw = new DestroyApplicationWork(this, appName);
-        queue.scheduleAndSync(daw);
-    }
-
-    @Override
-    public void reportPartitionAvailability(PartitionId pid, NetworkAddress networkAddress) throws Exception {
-        ReportPartitionAvailabilityWork rpaw = new ReportPartitionAvailabilityWork(this, pid, networkAddress);
-        queue.scheduleAndSync(rpaw);
-    }
-
     private static InetAddress getIpAddress(NCConfig ncConfig) throws Exception {
         String ipaddrStr = ncConfig.dataIPAddress;
         ipaddrStr = ipaddrStr.trim();
@@ -333,6 +308,11 @@
                 hbData.gcCollectionCounts[i] = gcMXBean.getCollectionCount();
                 hbData.gcCollectionTimes[i] = gcMXBean.getCollectionTime();
             }
+            PerformanceCounters netPC = netManager.getPerformanceCounters();
+            hbData.netPayloadBytesRead = netPC.getPayloadBytesRead();
+            hbData.netPayloadBytesWritten = netPC.getPayloadBytesWritten();
+            hbData.netSignalingBytesRead = netPC.getSignalingBytesRead();
+            hbData.netSignalingBytesWritten = netPC.getSignalingBytesWritten();
             try {
                 cc.nodeHeartbeat(id, hbData);
             } catch (Exception e) {
@@ -363,4 +343,59 @@
             }
         }
     }
+
+    private final class NodeControllerIPCI implements IIPCI {
+        @Override
+        public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+            NodeControllerFunctions.Function fn = (NodeControllerFunctions.Function) payload;
+            switch (fn.getFunctionId()) {
+                case START_TASKS: {
+                    NodeControllerFunctions.StartTasksFunction stf = (NodeControllerFunctions.StartTasksFunction) fn;
+                    queue.schedule(new StartTasksWork(NodeControllerService.this, stf.getAppName(), stf.getJobId(), stf
+                            .getPlanBytes(), stf.getTaskDescriptors(), stf.getConnectorPolicies()));
+                    return;
+                }
+
+                case ABORT_TASKS: {
+                    NodeControllerFunctions.AbortTasksFunction atf = (NodeControllerFunctions.AbortTasksFunction) fn;
+                    queue.schedule(new AbortTasksWork(NodeControllerService.this, atf.getJobId(), atf.getTasks()));
+                    return;
+                }
+
+                case CLEANUP_JOBLET: {
+                    NodeControllerFunctions.CleanupJobletFunction cjf = (NodeControllerFunctions.CleanupJobletFunction) fn;
+                    queue.schedule(new CleanupJobletWork(NodeControllerService.this, cjf.getJobId(), cjf.getStatus()));
+                    return;
+                }
+
+                case CREATE_APPLICATION: {
+                    NodeControllerFunctions.CreateApplicationFunction caf = (NodeControllerFunctions.CreateApplicationFunction) fn;
+                    queue.schedule(new CreateApplicationWork(NodeControllerService.this, caf.getAppName(), caf
+                            .isDeployHar(), caf.getSerializedDistributedState()));
+                    return;
+                }
+
+                case DESTROY_APPLICATION: {
+                    NodeControllerFunctions.DestroyApplicationFunction daf = (NodeControllerFunctions.DestroyApplicationFunction) fn;
+                    queue.schedule(new DestroyApplicationWork(NodeControllerService.this, daf.getAppName()));
+                    return;
+                }
+
+                case REPORT_PARTITION_AVAILABILITY: {
+                    NodeControllerFunctions.ReportPartitionAvailabilityFunction rpaf = (NodeControllerFunctions.ReportPartitionAvailabilityFunction) fn;
+                    queue.schedule(new ReportPartitionAvailabilityWork(NodeControllerService.this, rpaf
+                            .getPartitionId(), rpaf.getNetworkAddress()));
+                    return;
+                }
+
+                case NODE_REGISTRATION_RESULT: {
+                    NodeControllerFunctions.NodeRegistrationResult nrrf = (NodeControllerFunctions.NodeRegistrationResult) fn;
+                    setNodeRegistrationResult(nrrf.getNodeParameters(), nrrf.getException());
+                    return;
+                }
+            }
+            throw new IllegalArgumentException("Unknown function: " + fn.getFunctionId());
+
+        }
+    }
 }
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/ConnectionManager.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/ConnectionManager.java
deleted file mode 100644
index b9eaf1f..0000000
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/ConnectionManager.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.nc.net;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.nio.ByteBuffer;
-import java.nio.channels.AsynchronousCloseException;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.Selector;
-import java.nio.channels.ServerSocketChannel;
-import java.nio.channels.SocketChannel;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.api.comm.NetworkAddress;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
-import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.control.nc.partitions.IPartitionRequestListener;
-
-public class ConnectionManager {
-    private static final Logger LOGGER = Logger.getLogger(ConnectionManager.class.getName());
-
-    static final int INITIAL_MESSAGE_SIZE = 20;
-
-    private final IHyracksRootContext ctx;
-
-    private IPartitionRequestListener partitionRequestListener;
-
-    private final ServerSocketChannel serverChannel;
-
-    private volatile boolean stopped;
-
-    private final ConnectionListenerThread connectionListener;
-
-    private final DataListenerThread dataListener;
-
-    private final NetworkAddress networkAddress;
-
-    public ConnectionManager(IHyracksRootContext ctx, InetAddress inetAddress) throws IOException {
-        this.ctx = ctx;
-        serverChannel = ServerSocketChannel.open();
-        ServerSocket serverSocket = serverChannel.socket();
-        serverSocket.bind(new InetSocketAddress(inetAddress, 0));
-        stopped = false;
-        connectionListener = new ConnectionListenerThread();
-        dataListener = new DataListenerThread();
-        networkAddress = new NetworkAddress(serverSocket.getInetAddress(), serverSocket.getLocalPort());
-
-    }
-
-    public void setPartitionRequestListener(IPartitionRequestListener partitionRequestListener) {
-        this.partitionRequestListener = partitionRequestListener;
-    }
-
-    public void start() {
-        connectionListener.start();
-        dataListener.start();
-    }
-
-    public void stop() {
-        try {
-            stopped = true;
-            serverChannel.close();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-    }
-
-    public void connect(INetworkChannel channel) throws IOException {
-        dataListener.addOutgoingConnection(channel);
-    }
-
-    private final class ConnectionListenerThread extends Thread {
-        public ConnectionListenerThread() {
-            super("Hyracks NC Connection Listener");
-            setDaemon(true);
-        }
-
-        @Override
-        public void run() {
-            while (!stopped) {
-                try {
-                    SocketChannel sc = serverChannel.accept();
-                    dataListener.addIncomingConnection(sc);
-                } catch (AsynchronousCloseException e) {
-                    // do nothing
-                    if (!stopped) {
-                        e.printStackTrace();
-                    }
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-    }
-
-    private final class DataListenerThread extends Thread {
-        private Selector selector;
-
-        private final List<SocketChannel> pendingIncomingConnections;
-        private final Set<SocketChannel> pendingNegotiations;
-        private final List<INetworkChannel> pendingOutgoingConnections;
-
-        public DataListenerThread() {
-            super("Hyracks Data Listener Thread");
-            setDaemon(true);
-            try {
-                selector = Selector.open();
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-            pendingIncomingConnections = new ArrayList<SocketChannel>();
-            pendingNegotiations = new HashSet<SocketChannel>();
-            pendingOutgoingConnections = new ArrayList<INetworkChannel>();
-        }
-
-        synchronized void addIncomingConnection(SocketChannel sc) throws IOException {
-            pendingIncomingConnections.add(sc);
-            selector.wakeup();
-        }
-
-        synchronized void addOutgoingConnection(INetworkChannel channel) throws IOException {
-            pendingOutgoingConnections.add(channel);
-            selector.wakeup();
-        }
-
-        @Override
-        public void run() {
-            while (!stopped) {
-                try {
-                    if (LOGGER.isLoggable(Level.FINE)) {
-                        LOGGER.fine("Starting Select");
-                    }
-                    int n = selector.select();
-                    synchronized (this) {
-                        if (!pendingIncomingConnections.isEmpty()) {
-                            for (SocketChannel sc : pendingIncomingConnections) {
-                                sc.configureBlocking(false);
-                                SelectionKey scKey = sc.register(selector, SelectionKey.OP_READ);
-                                ByteBuffer buffer = ByteBuffer.allocate(INITIAL_MESSAGE_SIZE);
-                                scKey.attach(buffer);
-                                pendingNegotiations.add(sc);
-                            }
-                            pendingIncomingConnections.clear();
-                        }
-                        if (!pendingOutgoingConnections.isEmpty()) {
-                            for (INetworkChannel nc : pendingOutgoingConnections) {
-                                SocketChannel sc = SocketChannel.open();
-                                sc.configureBlocking(false);
-                                SelectionKey scKey = sc.register(selector, 0);
-                                scKey.attach(nc);
-                                nc.setSelectionKey(scKey);
-                                nc.notifyConnectionManagerRegistration();
-                            }
-                            pendingOutgoingConnections.clear();
-                        }
-                        if (LOGGER.isLoggable(Level.FINE)) {
-                            LOGGER.fine("Selector: " + n);
-                        }
-                        if (n > 0) {
-                            for (Iterator<SelectionKey> i = selector.selectedKeys().iterator(); i.hasNext();) {
-                                SelectionKey key = i.next();
-                                i.remove();
-                                SocketChannel sc = (SocketChannel) key.channel();
-                                if (pendingNegotiations.contains(sc)) {
-                                    if (key.isReadable()) {
-                                        ByteBuffer buffer = (ByteBuffer) key.attachment();
-                                        sc.read(buffer);
-                                        buffer.flip();
-                                        if (buffer.remaining() >= INITIAL_MESSAGE_SIZE) {
-                                            PartitionId pid = readInitialMessage(buffer);
-                                            pendingNegotiations.remove(sc);
-                                            key.interestOps(0);
-                                            NetworkOutputChannel channel = new NetworkOutputChannel(ctx, 5);
-                                            channel.setSelectionKey(key);
-                                            key.attach(channel);
-                                            try {
-                                                partitionRequestListener.registerPartitionRequest(pid, channel);
-                                            } catch (HyracksException e) {
-                                                key.cancel();
-                                                sc.close();
-                                            }
-                                        } else {
-                                            buffer.compact();
-                                        }
-                                    }
-                                } else {
-                                    INetworkChannel channel = (INetworkChannel) key.attachment();
-                                    boolean close = false;
-                                    try {
-                                        close = channel.dispatchNetworkEvent();
-                                    } catch (IOException e) {
-                                        e.printStackTrace();
-                                        close = true;
-                                    }
-                                    if (close) {
-                                        key.cancel();
-                                        sc.close();
-                                    }
-                                }
-                            }
-                        }
-                    }
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-
-        private PartitionId readInitialMessage(ByteBuffer buffer) {
-            JobId jobId = new JobId(buffer.getLong());
-            ConnectorDescriptorId cdid = new ConnectorDescriptorId(buffer.getInt());
-            int senderIndex = buffer.getInt();
-            int receiverIndex = buffer.getInt();
-            return new PartitionId(jobId, cdid, senderIndex, receiverIndex);
-        }
-    }
-
-    public NetworkAddress getNetworkAddress() {
-        return networkAddress;
-    }
-}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/INetworkChannel.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/INetworkChannel.java
deleted file mode 100644
index 61cd91f..0000000
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/INetworkChannel.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package edu.uci.ics.hyracks.control.nc.net;
-
-import java.io.IOException;
-import java.net.SocketAddress;
-import java.nio.channels.SelectionKey;
-
-public interface INetworkChannel {
-    public boolean dispatchNetworkEvent() throws IOException;
-
-    public void setSelectionKey(SelectionKey key);
-
-    public SelectionKey getSelectionKey();
-
-    public SocketAddress getRemoteAddress();
-
-    public void abort();
-
-    public void notifyConnectionManagerRegistration() throws IOException;
-}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
index 58f7088..ae2cd37 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
@@ -14,11 +14,8 @@
  */
 package edu.uci.ics.hyracks.control.nc.net;
 
-import java.io.IOException;
 import java.net.SocketAddress;
 import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.SocketChannel;
 import java.util.ArrayDeque;
 import java.util.Queue;
 import java.util.logging.Level;
@@ -26,53 +23,43 @@
 
 import edu.uci.ics.hyracks.api.channels.IInputChannel;
 import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
-import edu.uci.ics.hyracks.api.comm.FrameHelper;
 import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.api.exceptions.HyracksException;
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
 
-public class NetworkInputChannel implements IInputChannel, INetworkChannel {
+public class NetworkInputChannel implements IInputChannel {
     private static final Logger LOGGER = Logger.getLogger(NetworkInputChannel.class.getName());
 
-    private final ConnectionManager connectionManager;
+    private IHyracksRootContext ctx;
+
+    private final NetworkManager netManager;
 
     private final SocketAddress remoteAddress;
 
     private final PartitionId partitionId;
 
-    private final Queue<ByteBuffer> emptyQueue;
-
     private final Queue<ByteBuffer> fullQueue;
 
-    private SocketChannel socketChannel;
+    private final int nBuffers;
 
-    private SelectionKey key;
-
-    private ByteBuffer currentBuffer;
-
-    private boolean eos;
-
-    private boolean aborted;
+    private ChannelControlBlock ccb;
 
     private IInputChannelMonitor monitor;
 
     private Object attachment;
 
-    private ByteBuffer writeBuffer;
-
-    public NetworkInputChannel(IHyracksRootContext ctx, ConnectionManager connectionManager,
-            SocketAddress remoteAddress, PartitionId partitionId, int nBuffers) {
-        this.connectionManager = connectionManager;
+    public NetworkInputChannel(IHyracksRootContext ctx, NetworkManager netManager, SocketAddress remoteAddress,
+            PartitionId partitionId, int nBuffers) {
+        this.ctx = ctx;
+        this.netManager = netManager;
         this.remoteAddress = remoteAddress;
         this.partitionId = partitionId;
-        this.emptyQueue = new ArrayDeque<ByteBuffer>(nBuffers);
-        for (int i = 0; i < nBuffers; ++i) {
-            emptyQueue.add(ctx.allocateFrame());
-        }
         fullQueue = new ArrayDeque<ByteBuffer>(nBuffers);
-        aborted = false;
-        eos = false;
+        this.nBuffers = nBuffers;
     }
 
     @Override
@@ -96,29 +83,34 @@
     }
 
     @Override
-    public synchronized void recycleBuffer(ByteBuffer buffer) {
+    public void recycleBuffer(ByteBuffer buffer) {
         buffer.clear();
-        emptyQueue.add(buffer);
-        if (!eos && !aborted) {
-            int ops = key.interestOps();
-            if ((ops & SelectionKey.OP_READ) == 0) {
-                key.interestOps(ops | SelectionKey.OP_READ);
-                key.selector().wakeup();
-                if (currentBuffer == null) {
-                    currentBuffer = emptyQueue.poll();
-                }
-            }
-        }
+        ccb.getReadInterface().getEmptyBufferAcceptor().accept(buffer);
     }
 
     @Override
     public void open() throws HyracksDataException {
-        currentBuffer = emptyQueue.poll();
         try {
-            connectionManager.connect(this);
-        } catch (IOException e) {
+            ccb = netManager.connect(remoteAddress);
+        } catch (Exception e) {
             throw new HyracksDataException(e);
         }
+        ccb.getReadInterface().setFullBufferAcceptor(new ReadFullBufferAcceptor());
+        ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
+        for (int i = 0; i < nBuffers; ++i) {
+            ccb.getReadInterface().getEmptyBufferAcceptor().accept(ctx.allocateFrame());
+        }
+        ByteBuffer writeBuffer = ByteBuffer.allocate(NetworkManager.INITIAL_MESSAGE_SIZE);
+        writeBuffer.putLong(partitionId.getJobId().getId());
+        writeBuffer.putInt(partitionId.getConnectorDescriptorId().getId());
+        writeBuffer.putInt(partitionId.getSenderIndex());
+        writeBuffer.putInt(partitionId.getReceiverIndex());
+        writeBuffer.flip();
+        if (LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("Sending partition request: " + partitionId + " on channel: " + ccb);
+        }
+        ccb.getWriteInterface().getFullBufferAcceptor().accept(writeBuffer);
+        ccb.getWriteInterface().getFullBufferAcceptor().close();
     }
 
     @Override
@@ -126,110 +118,28 @@
 
     }
 
-    @Override
-    public synchronized boolean dispatchNetworkEvent() throws IOException {
-        if (aborted) {
-            eos = true;
-            monitor.notifyEndOfStream(this);
-            return true;
+    private class ReadFullBufferAcceptor implements ICloseableBufferAcceptor {
+        @Override
+        public void accept(ByteBuffer buffer) {
+            fullQueue.add(buffer);
+            monitor.notifyDataAvailability(NetworkInputChannel.this, 1);
         }
-        if (key.isConnectable()) {
-            if (socketChannel.finishConnect()) {
-                key.interestOps(key.interestOps() & ~SelectionKey.OP_CONNECT);
-                prepareForWrite();
-            }
-        } else if (key.isWritable()) {
-            socketChannel.write(writeBuffer);
-            if (writeBuffer.remaining() == 0) {
-                key.interestOps(SelectionKey.OP_READ);
-            }
-        } else if (key.isReadable()) {
-            if (LOGGER.isLoggable(Level.FINER)) {
-                LOGGER.finer("Before read: " + currentBuffer.position() + " " + currentBuffer.limit());
-            }
-            int bytesRead = socketChannel.read(currentBuffer);
-            if (bytesRead < 0) {
-                eos = true;
-                monitor.notifyEndOfStream(this);
-                return true;
-            }
-            if (LOGGER.isLoggable(Level.FINER)) {
-                LOGGER.finer("After read: " + currentBuffer.position() + " " + currentBuffer.limit());
-            }
-            currentBuffer.flip();
-            int dataLen = currentBuffer.remaining();
-            if (dataLen >= currentBuffer.capacity() || aborted()) {
-                if (LOGGER.isLoggable(Level.FINEST)) {
-                    LOGGER.finest("NetworkInputChannel: frame received: sender = " + partitionId.getSenderIndex());
-                }
-                if (currentBuffer.getInt(FrameHelper.getTupleCountOffset(currentBuffer.capacity())) == 0) {
-                    eos = true;
-                    monitor.notifyEndOfStream(this);
-                    return true;
-                }
-                fullQueue.add(currentBuffer);
-                currentBuffer = emptyQueue.poll();
-                if (currentBuffer == null && key.isValid()) {
-                    int ops = key.interestOps();
-                    key.interestOps(ops & ~SelectionKey.OP_READ);
-                }
-                monitor.notifyDataAvailability(this, 1);
-                return false;
-            }
-            currentBuffer.compact();
+
+        @Override
+        public void close() {
+            monitor.notifyEndOfStream(NetworkInputChannel.this);
         }
-        return false;
+
+        @Override
+        public void error(int ecode) {
+            monitor.notifyFailure(NetworkInputChannel.this);
+        }
     }
 
-    private void prepareForConnect() {
-        key.interestOps(SelectionKey.OP_CONNECT);
-    }
-
-    private void prepareForWrite() {
-        writeBuffer = ByteBuffer.allocate(ConnectionManager.INITIAL_MESSAGE_SIZE);
-        writeBuffer.putLong(partitionId.getJobId().getId());
-        writeBuffer.putInt(partitionId.getConnectorDescriptorId().getId());
-        writeBuffer.putInt(partitionId.getSenderIndex());
-        writeBuffer.putInt(partitionId.getReceiverIndex());
-        writeBuffer.flip();
-
-        key.interestOps(SelectionKey.OP_WRITE);
-    }
-
-    @Override
-    public void setSelectionKey(SelectionKey key) {
-        this.key = key;
-        socketChannel = (SocketChannel) key.channel();
-    }
-
-    @Override
-    public SocketAddress getRemoteAddress() {
-        return remoteAddress;
-    }
-
-    @Override
-    public SelectionKey getSelectionKey() {
-        return key;
-    }
-
-    public PartitionId getPartitionId() {
-        return partitionId;
-    }
-
-    public void abort() {
-        aborted = true;
-    }
-
-    public boolean aborted() {
-        return aborted;
-    }
-
-    @Override
-    public void notifyConnectionManagerRegistration() throws IOException {
-        if (socketChannel.connect(remoteAddress)) {
-            prepareForWrite();
-        } else {
-            prepareForConnect();
+    private class WriteEmptyBufferAcceptor implements IBufferAcceptor {
+        @Override
+        public void accept(ByteBuffer buffer) {
+            // do nothing
         }
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java
new file mode 100644
index 0000000..f94be22
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.net;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.nc.partitions.IPartitionRequestListener;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.IChannelOpenListener;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MultiplexedConnection;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MuxDemux;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.PerformanceCounters;
+
+public class NetworkManager {
+    private static final Logger LOGGER = Logger.getLogger(NetworkManager.class.getName());
+
+    static final int INITIAL_MESSAGE_SIZE = 20;
+
+    private final IHyracksRootContext ctx;
+
+    private final IPartitionRequestListener partitionRequestListener;
+
+    private final MuxDemux md;
+
+    private NetworkAddress networkAddress;
+
+    public NetworkManager(IHyracksRootContext ctx, InetAddress inetAddress,
+            IPartitionRequestListener partitionRequestListener, int nThreads) throws IOException {
+        this.ctx = ctx;
+        this.partitionRequestListener = partitionRequestListener;
+        md = new MuxDemux(new InetSocketAddress(inetAddress, 0), new ChannelOpenListener(), nThreads);
+    }
+
+    public void start() throws IOException {
+        md.start();
+        InetSocketAddress sockAddr = md.getLocalAddress();
+        networkAddress = new NetworkAddress(sockAddr.getAddress(), sockAddr.getPort());
+    }
+
+    public NetworkAddress getNetworkAddress() {
+        return networkAddress;
+    }
+
+    public void stop() {
+
+    }
+
+    public ChannelControlBlock connect(SocketAddress remoteAddress) throws InterruptedException, NetException {
+        MultiplexedConnection mConn = md.connect((InetSocketAddress) remoteAddress);
+        return mConn.openChannel();
+    }
+
+    private class ChannelOpenListener implements IChannelOpenListener {
+        @Override
+        public void channelOpened(ChannelControlBlock channel) {
+            channel.getReadInterface().setFullBufferAcceptor(new InitialBufferAcceptor(channel));
+            channel.getReadInterface().getEmptyBufferAcceptor().accept(ByteBuffer.allocate(INITIAL_MESSAGE_SIZE));
+        }
+    }
+
+    private class InitialBufferAcceptor implements ICloseableBufferAcceptor {
+        private final ChannelControlBlock ccb;
+
+        private NetworkOutputChannel noc;
+
+        public InitialBufferAcceptor(ChannelControlBlock ccb) {
+            this.ccb = ccb;
+        }
+
+        @Override
+        public void accept(ByteBuffer buffer) {
+            PartitionId pid = readInitialMessage(buffer);
+            if (LOGGER.isLoggable(Level.FINE)) {
+                LOGGER.fine("Received initial partition request: " + pid + " on channel: " + ccb);
+            }
+            noc = new NetworkOutputChannel(ctx, ccb, 5);
+            try {
+                partitionRequestListener.registerPartitionRequest(pid, noc);
+            } catch (HyracksException e) {
+                noc.abort();
+            }
+        }
+
+        @Override
+        public void close() {
+
+        }
+
+        @Override
+        public void error(int ecode) {
+            if (noc != null) {
+                noc.abort();
+            }
+        }
+    }
+
+    private static PartitionId readInitialMessage(ByteBuffer buffer) {
+        JobId jobId = new JobId(buffer.getLong());
+        ConnectorDescriptorId cdid = new ConnectorDescriptorId(buffer.getInt());
+        int senderIndex = buffer.getInt();
+        int receiverIndex = buffer.getInt();
+        return new PartitionId(jobId, cdid, senderIndex, receiverIndex);
+    }
+
+    public PerformanceCounters getPerformanceCounters() {
+        return md.getPerformanceCounters();
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
index 31ce924..8d5f475 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
@@ -14,124 +14,44 @@
  */
 package edu.uci.ics.hyracks.control.nc.net;
 
-import java.io.IOException;
-import java.net.SocketAddress;
 import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.SocketChannel;
 import java.util.ArrayDeque;
 import java.util.Queue;
 
-import edu.uci.ics.hyracks.api.comm.FrameHelper;
 import edu.uci.ics.hyracks.api.comm.IFrameWriter;
 import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
 
-public class NetworkOutputChannel implements INetworkChannel, IFrameWriter {
-    private final IHyracksRootContext ctx;
+public class NetworkOutputChannel implements IFrameWriter {
+    private final ChannelControlBlock ccb;
 
     private final Queue<ByteBuffer> emptyQueue;
 
-    private final Queue<ByteBuffer> fullQueue;
-
-    private SelectionKey key;
-
     private boolean aborted;
 
-    private boolean eos;
-
-    private boolean eosSent;
-
-    private boolean failed;
-
-    private ByteBuffer currentBuffer;
-
-    public NetworkOutputChannel(IHyracksRootContext ctx, int nBuffers) {
-        this.ctx = ctx;
+    public NetworkOutputChannel(IHyracksRootContext ctx, ChannelControlBlock ccb, int nBuffers) {
+        this.ccb = ccb;
         emptyQueue = new ArrayDeque<ByteBuffer>(nBuffers);
         for (int i = 0; i < nBuffers; ++i) {
-            emptyQueue.add(ctx.allocateFrame());
+            emptyQueue.add(ByteBuffer.allocateDirect(ctx.getFrameSize()));
         }
-        fullQueue = new ArrayDeque<ByteBuffer>(nBuffers);
-    }
-
-    @Override
-    public synchronized boolean dispatchNetworkEvent() throws IOException {
-        if (failed || aborted) {
-            eos = true;
-            return true;
-        } else if (key.isWritable()) {
-            while (true) {
-                if (currentBuffer == null) {
-                    if (eosSent) {
-                        return true;
-                    }
-                    currentBuffer = fullQueue.poll();
-                    if (currentBuffer == null) {
-                        if (eos) {
-                            currentBuffer = emptyQueue.poll();
-                            currentBuffer.clear();
-                            currentBuffer.putInt(FrameHelper.getTupleCountOffset(ctx.getFrameSize()), 0);
-                            eosSent = true;
-                        } else {
-                            key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE);
-                            return false;
-                        }
-                    }
-                }
-                int bytesWritten = ((SocketChannel) key.channel()).write(currentBuffer);
-                if (bytesWritten < 0) {
-                    eos = true;
-                    return true;
-                }
-                if (currentBuffer.remaining() == 0) {
-                    emptyQueue.add(currentBuffer);
-                    notifyAll();
-                    currentBuffer = null;
-                    if (eosSent) {
-                        return true;
-                    }
-                } else {
-                    return false;
-                }
-            }
-        }
-        return false;
-    }
-
-    @Override
-    public void setSelectionKey(SelectionKey key) {
-        this.key = key;
-    }
-
-    @Override
-    public SelectionKey getSelectionKey() {
-        return key;
-    }
-
-    @Override
-    public SocketAddress getRemoteAddress() {
-        return ((SocketChannel) key.channel()).socket().getRemoteSocketAddress();
-    }
-
-    @Override
-    public synchronized void abort() {
-        aborted = true;
+        ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
     }
 
     @Override
     public void open() throws HyracksDataException {
-        currentBuffer = null;
     }
 
     @Override
     public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
         ByteBuffer destBuffer = null;
         synchronized (this) {
-            if (aborted) {
-                throw new HyracksDataException("Connection has been aborted");
-            }
             while (true) {
+                if (aborted) {
+                    throw new HyracksDataException("Connection has been aborted");
+                }
                 destBuffer = emptyQueue.poll();
                 if (destBuffer != null) {
                     break;
@@ -148,26 +68,34 @@
         destBuffer.clear();
         destBuffer.put(buffer);
         destBuffer.flip();
-        synchronized (this) {
-            fullQueue.add(destBuffer);
-        }
-        key.interestOps(SelectionKey.OP_WRITE);
-        key.selector().wakeup();
+        ccb.getWriteInterface().getFullBufferAcceptor().accept(destBuffer);
     }
 
     @Override
     public void fail() throws HyracksDataException {
-        failed = true;
+        ccb.getWriteInterface().getFullBufferAcceptor().error(1);
     }
 
     @Override
-    public synchronized void close() throws HyracksDataException {
-        eos = true;
-        key.interestOps(SelectionKey.OP_WRITE);
-        key.selector().wakeup();
+    public void close() throws HyracksDataException {
+        ccb.getWriteInterface().getFullBufferAcceptor().close();
     }
 
-    @Override
-    public void notifyConnectionManagerRegistration() throws IOException {
+    void abort() {
+        ccb.getWriteInterface().getFullBufferAcceptor().error(1);
+        synchronized (NetworkOutputChannel.this) {
+            aborted = true;
+            NetworkOutputChannel.this.notifyAll();
+        }
+    }
+
+    private class WriteEmptyBufferAcceptor implements IBufferAcceptor {
+        @Override
+        public void accept(ByteBuffer buffer) {
+            synchronized (NetworkOutputChannel.this) {
+                emptyQueue.add(buffer);
+                NetworkOutputChannel.this.notifyAll();
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
index 9403736..44e61f1 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
@@ -32,6 +32,8 @@
 
     private IFrameWriter delegate;
 
+    private boolean pendingConnection;
+
     private boolean failed;
 
     public PipelinedPartition(PartitionManager manager, PartitionId pid, TaskAttemptId taId) {
@@ -57,32 +59,44 @@
     }
 
     @Override
-    public synchronized void open() throws HyracksDataException {
+    public void open() throws HyracksDataException {
         manager.registerPartition(pid, taId, this, PartitionState.STARTED);
         failed = false;
-        while (delegate == null) {
-            try {
-                wait();
-            } catch (InterruptedException e) {
-                throw new HyracksDataException(e);
-            }
-        }
-        delegate.open();
+        pendingConnection = true;
     }
 
     @Override
     public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+        ensureConnected();
         delegate.nextFrame(buffer);
     }
 
+    private void ensureConnected() throws HyracksDataException {
+        if (pendingConnection) {
+            synchronized (this) {
+                while (delegate == null) {
+                    try {
+                        wait();
+                    } catch (InterruptedException e) {
+                        throw new HyracksDataException(e);
+                    }
+                }
+            }
+            delegate.open();
+        }
+        pendingConnection = false;
+    }
+
     @Override
     public void fail() throws HyracksDataException {
+        ensureConnected();
         failed = true;
         delegate.fail();
     }
 
     @Override
     public void close() throws HyracksDataException {
+        ensureConnected();
         if (!failed) {
             manager.updatePartitionState(pid, taId, this, PartitionState.COMMITTED);
         }
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ProfilingPartitionWriterFactory.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ProfilingPartitionWriterFactory.java
index ea4dbd6..affa01c 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ProfilingPartitionWriterFactory.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ProfilingPartitionWriterFactory.java
@@ -14,7 +14,6 @@
  */
 package edu.uci.ics.hyracks.control.nc.profiling;
 
-import java.io.ByteArrayOutputStream;
 import java.nio.ByteBuffer;
 
 import edu.uci.ics.hyracks.api.comm.IFrameWriter;
@@ -23,10 +22,13 @@
 import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
 import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.profiling.counters.MultiResolutionEventProfiler;
 import edu.uci.ics.hyracks.control.common.job.profiling.om.PartitionProfile;
 import edu.uci.ics.hyracks.control.nc.Task;
 
 public class ProfilingPartitionWriterFactory implements IPartitionWriterFactory {
+    private static final int N_SAMPLES = 64;
+
     private final IHyracksTaskContext ctx;
 
     private final IConnectorDescriptor cd;
@@ -52,31 +54,17 @@
 
             private long closeTime;
 
-            private long prevTime;
-
-            private ByteArrayOutputStream baos = new ByteArrayOutputStream();
+            MultiResolutionEventProfiler mrep = new MultiResolutionEventProfiler(N_SAMPLES);
 
             @Override
             public void open() throws HyracksDataException {
-                baos.reset();
                 openTime = System.currentTimeMillis();
-                prevTime = openTime;
                 writer.open();
             }
 
             @Override
             public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
-                long time = System.currentTimeMillis();
-                long diff = time - prevTime;
-                prevTime = time;
-                do {
-                    byte b = (byte) (diff & 0x7f);
-                    diff >>= 7;
-                    if (diff != 0) {
-                        b |= 0x80;
-                    }
-                    baos.write(b);
-                } while (diff != 0);
+                mrep.reportEvent();
                 writer.nextFrame(buffer);
             }
 
@@ -89,8 +77,7 @@
             public void close() throws HyracksDataException {
                 closeTime = System.currentTimeMillis();
                 ((Task) ctx).setPartitionSendProfile(new PartitionProfile(new PartitionId(ctx.getJobletContext()
-                        .getJobId(), cd.getConnectorId(), senderIndex, receiverIndex), openTime, closeTime, baos
-                        .toByteArray()));
+                        .getJobId(), cd.getConnectorId(), senderIndex, receiverIndex), openTime, closeTime, mrep));
                 writer.close();
             }
         };
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
index dce7c13..7bebb53 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
@@ -50,6 +50,15 @@
         if (joblet != null) {
             joblet.cleanup(status);
         }
-        ncs.getClusterController().notifyJobletCleanup(jobId, ncs.getId());
+        ncs.getExecutor().execute(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    ncs.getClusterController().notifyJobletCleanup(jobId, ncs.getId());
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
index 120e376..eb982df 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
@@ -27,6 +27,7 @@
 import org.apache.http.impl.client.DefaultHttpClient;
 
 import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
 import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
 import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
 import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
@@ -54,31 +55,37 @@
 
     @Override
     protected void doRun() throws Exception {
-        NCApplicationContext appCtx;
-        Map<String, NCApplicationContext> applications = ncs.getApplications();
-        if (applications.containsKey(appName)) {
-            throw new HyracksException("Duplicate application with name: " + appName + " being created.");
-        }
-        appCtx = new NCApplicationContext(ncs.getServerContext(), ncs.getRootContext(), appName, ncs.getId());
-        applications.put(appName, appCtx);
-        if (deployHar) {
-            NCConfig ncConfig = ncs.getConfiguration();
-            NodeParameters nodeParameters = ncs.getNodeParameters();
-            HttpClient hc = new DefaultHttpClient();
-            HttpGet get = new HttpGet("http://" + ncConfig.ccHost + ":"
-                    + nodeParameters.getClusterControllerInfo().getWebPort() + "/applications/" + appName);
-            HttpResponse response = hc.execute(get);
-            InputStream is = response.getEntity().getContent();
-            OutputStream os = appCtx.getHarOutputStream();
-            try {
-                IOUtils.copyLarge(is, os);
-            } finally {
-                os.close();
-                is.close();
+        try {
+            NCApplicationContext appCtx;
+            Map<String, NCApplicationContext> applications = ncs.getApplications();
+            if (applications.containsKey(appName)) {
+                throw new HyracksException("Duplicate application with name: " + appName + " being created.");
             }
+            appCtx = new NCApplicationContext(ncs.getServerContext(), ncs.getRootContext(), appName, ncs.getId());
+            applications.put(appName, appCtx);
+            if (deployHar) {
+                NCConfig ncConfig = ncs.getConfiguration();
+                NodeParameters nodeParameters = ncs.getNodeParameters();
+                HttpClient hc = new DefaultHttpClient();
+                HttpGet get = new HttpGet("http://" + ncConfig.ccHost + ":"
+                        + nodeParameters.getClusterControllerInfo().getWebPort() + "/applications/" + appName);
+                HttpResponse response = hc.execute(get);
+                InputStream is = response.getEntity().getContent();
+                OutputStream os = appCtx.getHarOutputStream();
+                try {
+                    IOUtils.copyLarge(is, os);
+                } finally {
+                    os.close();
+                    is.close();
+                }
+            }
+            appCtx.initializeClassPath();
+            appCtx.setDistributedState((Serializable) appCtx.deserialize(serializedDistributedState));
+            appCtx.initialize();
+            ncs.getClusterController()
+                    .notifyApplicationStateChange(ncs.getId(), appName, ApplicationStatus.INITIALIZED);
+        } catch (Exception e) {
+            LOGGER.warning("Error creating application: " + e.getMessage());
         }
-        appCtx.initializeClassPath();
-        appCtx.setDistributedState((Serializable) appCtx.deserialize(serializedDistributedState));
-        appCtx.initialize();
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
index 6a5fbfc..cfe00f6 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
@@ -18,6 +18,7 @@
 import java.util.logging.Logger;
 
 import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
 import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
 import edu.uci.ics.hyracks.control.nc.NodeControllerService;
 import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
@@ -36,10 +37,15 @@
 
     @Override
     protected void doRun() throws Exception {
-        Map<String, NCApplicationContext> applications = ncs.getApplications();
-        ApplicationContext appCtx = applications.remove(appName);
-        if (appCtx != null) {
-            appCtx.deinitialize();
+        try {
+            Map<String, NCApplicationContext> applications = ncs.getApplications();
+            ApplicationContext appCtx = applications.remove(appName);
+            if (appCtx != null) {
+                appCtx.deinitialize();
+            }
+        } catch (Exception e) {
+            LOGGER.warning("Error destroying application: " + e.getMessage());
         }
+        ncs.getClusterController().notifyApplicationStateChange(ncs.getId(), appName, ApplicationStatus.DEINITIALIZED);
     }
 }
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
index bfa21c9..ce3f4e0 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
@@ -48,8 +48,8 @@
         Joblet ji = jobletMap.get(pid.getJobId());
         if (ji != null) {
             PartitionChannel channel = new PartitionChannel(pid, new NetworkInputChannel(ncs.getRootContext(),
-                    ncs.getConnectionManager(), new InetSocketAddress(networkAddress.getIpAddress(),
-                            networkAddress.getPort()), pid, 1));
+                    ncs.getNetworkManager(), new InetSocketAddress(networkAddress.getIpAddress(),
+                            networkAddress.getPort()), pid, 5));
             ji.reportPartitionAvailability(channel);
         }
     }
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java
index 61638d1..beaf862 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java
@@ -62,7 +62,7 @@
     public Object[] deserializeRecord() throws HyracksDataException {
         int start = frameTupleAccessor.getTupleStartOffset(tIndex) + frameTupleAccessor.getFieldSlotsLength();
         bbis.setByteBuffer(buffer, start);
-        Object[] record = new Object[recordDescriptor.getFields().length];
+        Object[] record = new Object[recordDescriptor.getFieldCount()];
         for (int i = 0; i < record.length; ++i) {
             Object instance = recordDescriptor.getFields()[i].deserialize(di);
             if (LOGGER.isLoggable(Level.FINEST)) {
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java
index 8ab5d4f..6c51d14 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java
@@ -24,11 +24,12 @@
 import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
 
 /**
- * FrameTupleCursor is used to navigate over tuples in a Frame.
- * A frame is formatted with tuple data concatenated starting at offset 0, one tuple after another.
- * Offset FS - 4 holds an int indicating the number of tuples (N) in the frame. FS - ((i + 1) * 4) for i from
- * 0 to N - 1 holds an int indicating the offset of the (i + 1)^th tuple.
- * Every tuple is organized as a sequence of ints indicating the end of each field in the tuple relative to the end of the
+ * FrameTupleCursor is used to navigate over tuples in a Frame. A frame is
+ * formatted with tuple data concatenated starting at offset 0, one tuple after
+ * another. Offset FS - 4 holds an int indicating the number of tuples (N) in
+ * the frame. FS - ((i + 1) * 4) for i from 0 to N - 1 holds an int indicating
+ * the offset of the (i + 1)^th tuple. Every tuple is organized as a sequence of
+ * ints indicating the end of each field in the tuple relative to the end of the
  * field slots.
  * 
  * @author vinayakb
@@ -86,7 +87,7 @@
 
     @Override
     public int getFieldSlotsLength() {
-        return recordDescriptor.getFields().length * 4;
+        return getFieldCount() * 4;
     }
 
     public void prettyPrint() {
@@ -96,7 +97,7 @@
         System.err.println("TC: " + tc);
         for (int i = 0; i < tc; ++i) {
             System.err.print(i + ":(" + getTupleStartOffset(i) + ", " + getTupleEndOffset(i) + ")[");
-            for (int j = 0; j < recordDescriptor.getFields().length; ++j) {
+            for (int j = 0; j < getFieldCount(); ++j) {
                 System.err.print(j + ":(" + getFieldStartOffset(i, j) + ", " + getFieldEndOffset(i, j) + ") ");
                 System.err.print("{");
                 bbis.setByteBuffer(buffer, getTupleStartOffset(i) + getFieldSlotsLength() + getFieldStartOffset(i, j));
@@ -113,6 +114,6 @@
 
     @Override
     public int getFieldCount() {
-        return recordDescriptor.getFields().length;
+        return recordDescriptor.getFieldCount();
     }
 }
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
index e8f43cd..cbe2ae6 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
@@ -41,7 +41,7 @@
 
     public SerializingDataWriter(IHyracksTaskContext ctx, RecordDescriptor recordDescriptor, IFrameWriter frameWriter) {
         buffer = ctx.allocateFrame();
-        tb = new ArrayTupleBuilder(recordDescriptor.getFields().length);
+        tb = new ArrayTupleBuilder(recordDescriptor.getFieldCount());
         this.recordDescriptor = recordDescriptor;
         this.frameWriter = frameWriter;
         tupleAppender = new FrameTupleAppender(ctx.getFrameSize());
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
index 8d69b95..7769539 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
@@ -39,6 +39,9 @@
         return new ITuplePartitionComputer() {
             @Override
             public int partition(IFrameTupleAccessor accessor, int tIndex, int nParts) {
+                if (nParts == 1) {
+                    return 0;
+                }
                 int h = 0;
                 int startOffset = accessor.getTupleStartOffset(tIndex);
                 int slotLength = accessor.getFieldSlotsLength();
@@ -49,7 +52,7 @@
                     int fEnd = accessor.getFieldEndOffset(tIndex, fIdx);
                     int fh = hashFn
                             .hash(accessor.getBuffer().array(), startOffset + slotLength + fStart, fEnd - fStart);
-                    h += fh;
+                    h = h * 31 + fh;
                 }
                 if (h < 0) {
                     h = -h;
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java
index ef2b4cb..2841509 100644
--- a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java
@@ -142,8 +142,8 @@
 
     @SuppressWarnings("deprecation")
     @Override
-    public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IRecordDescriptorProvider recordDescProvider,
-            final int partition, int nPartitions)
+    public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+            final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
             throws HyracksDataException {
         return new AbstractUnaryOutputSourceOperatorNodePushable() {
             @Override
@@ -191,7 +191,7 @@
                     RecordDescriptor outputRecordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor(
                             (Class<? extends Writable>) hadoopRecordReader.createKey().getClass(),
                             (Class<? extends Writable>) hadoopRecordReader.createValue().getClass());
-                    int nFields = outputRecordDescriptor.getFields().length;
+                    int nFields = outputRecordDescriptor.getFieldCount();
                     ArrayTupleBuilder tb = new ArrayTupleBuilder(nFields);
                     writer.open();
                     try {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorFactory.java
index 8237110..4fec049 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorFactory.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorFactory.java
@@ -41,7 +41,7 @@
         for (int i = 0; i < aFactories.length; ++i) {
             aggregators[i] = aFactories[i].createFieldValueResultingAggregator();
         }
-        final ArrayTupleBuilder tb = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
+        final ArrayTupleBuilder tb = new ArrayTupleBuilder(outRecordDescriptor.getFieldCount());
         return new IAccumulatingAggregator() {
 
             private boolean pending;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
index f3535f8..4ffc060 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
@@ -41,7 +41,6 @@
     public JSONObject toJSON() throws JSONException {
         JSONObject jconn = new JSONObject();
 
-        jconn.put("type", "connector");
         jconn.put("id", getConnectorId().getId());
         jconn.put("java-class", getClass().getName());
 
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
index 15fc75d..1bb13ca 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
@@ -75,7 +75,6 @@
     @Override
     public JSONObject toJSON() throws JSONException {
         JSONObject jop = new JSONObject();
-        jop.put("type", "operator");
         jop.put("id", getOperatorId().getId());
         jop.put("java-class", getClass().getName());
         jop.put("in-arity", getInputArity());
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionCollector.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionCollector.java
index 33819a8..d5f4fd8 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionCollector.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionCollector.java
@@ -18,6 +18,8 @@
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collection;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import edu.uci.ics.hyracks.api.channels.IInputChannel;
 import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
@@ -31,6 +33,8 @@
 import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
 
 public class NonDeterministicPartitionCollector extends AbstractPartitionCollector {
+    private static final Logger LOGGER = Logger.getLogger(NonDeterministicPartitionCollector.class.getName());
+
     private final FrameReader reader;
 
     private final BitSet expectedPartitions;
@@ -172,6 +176,11 @@
                 PartitionId pid = (PartitionId) channel.getAttachment();
                 int senderIndex = pid.getSenderIndex();
                 failSenders.set(senderIndex);
+                eosSenders.set(senderIndex);
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.fine("Failure: " + connectorId + " sender: " + senderIndex + " receiver: "
+                            + receiverIndex);
+                }
                 NonDeterministicPartitionCollector.this.notifyAll();
             }
         }
@@ -183,6 +192,10 @@
                 int senderIndex = pid.getSenderIndex();
                 availableFrameCounts[senderIndex] += nFrames;
                 frameAvailability.set(senderIndex);
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.fine("Data available: " + connectorId + " sender: " + senderIndex + " receiver: "
+                            + receiverIndex);
+                }
                 NonDeterministicPartitionCollector.this.notifyAll();
             }
         }
@@ -193,6 +206,10 @@
                 PartitionId pid = (PartitionId) channel.getAttachment();
                 int senderIndex = pid.getSenderIndex();
                 eosSenders.set(senderIndex);
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.fine("EOS: " + connectorId + " sender: " + senderIndex + " receiver: "
+                            + receiverIndex);
+                }
                 NonDeterministicPartitionCollector.this.notifyAll();
             }
         }
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
index 5bc6ee5..8544891 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
@@ -75,7 +75,7 @@
             writer.open();
             try {
                 while (true) {
-                    Object[] record = new Object[desc.getFields().length];
+                    Object[] record = new Object[desc.getFieldCount()];
                     if (!reader.read(record)) {
                         break;
                     }
@@ -107,8 +107,8 @@
     }
 
     @Override
-    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
-            int partition, int nPartitions) {
+    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+            IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
         return new DeserializedOperatorNodePushable(ctx, new DeserializedFileScanOperator(partition), null);
     }
 }
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
index e0f5a65..47a3158 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
@@ -60,8 +60,8 @@
      * @see edu.uci.ics.hyracks.api.dataflow.IActivityNode#createPushRuntime(edu.uci.ics.hyracks.api.context.IHyracksContext, edu.uci.ics.hyracks.api.job.IOperatorEnvironment, edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider, int, int)
      */
     @Override
-    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
-            final int partition, int nPartitions)
+    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+            IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
             throws HyracksDataException {
         // Output files
         final FileSplit[] splits = fileSplitProvider.getFileSplits();
@@ -96,7 +96,7 @@
                         int start = frameTupleAccessor.getTupleStartOffset(tIndex)
                                 + frameTupleAccessor.getFieldSlotsLength();
                         bbis.setByteBuffer(buffer, start);
-                        Object[] record = new Object[recordDescriptor.getFields().length];
+                        Object[] record = new Object[recordDescriptor.getFieldCount()];
                         for (int i = 0; i < record.length; ++i) {
                             Object instance = recordDescriptor.getFields()[i].deserialize(di);
                             if (i == 0) {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
index aa3f892..0680f69 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
@@ -256,7 +256,7 @@
             /**
              * Tuple builder
              */
-            final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFields().length);
+            final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFieldCount());
 
             IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
                 /**
@@ -457,7 +457,7 @@
 
                 private void flushOutFrame(IFrameWriter writer, boolean isFinal) throws HyracksDataException {
                     if (finalTupleBuilder == null) {
-                        finalTupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFields().length);
+                        finalTupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFieldCount());
                     }
                     if (writerFrame == null) {
                         writerFrame = ctx.allocateFrame();
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
index 8255c58..5a6d8cd 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -18,7 +18,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
 import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
@@ -63,7 +63,7 @@
         public String host;
 
         @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
-        public int port = 1099;
+        public int port = 1098;
 
         @Option(name = "-app", usage = "Hyracks Application name", required = true)
         public String app;
@@ -86,7 +86,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job = createJob(options);
 
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index ae988cd..0b1cd6c 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -17,7 +17,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
 import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
@@ -61,7 +61,7 @@
         public String host;
 
         @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
-        public int port = 1099;
+        public int port = 1098;
 
         @Option(name = "-app", usage = "Hyracks Application name", required = true)
         public String app;
@@ -84,7 +84,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job = createJob(options);
 
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index dbc0bca..aa73a62 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -19,7 +19,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -60,7 +60,7 @@
         public String host;
 
         @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
-        public int port = 1099;
+        public int port = 1098;
 
         @Option(name = "-app", usage = "Hyracks Application name", required = true)
         public String app;
@@ -77,7 +77,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job = createJob(options);
 
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index bf2c2e1..ad9ad3a 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -17,7 +17,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -56,7 +56,7 @@
         public String host;
 
         @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
-        public int port = 1099;
+        public int port = 1098;
 
         @Option(name = "-app", usage = "Hyracks Application name", required = true)
         public String app;
@@ -79,7 +79,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job = createJob(options);
 
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index 29f8257..2719397 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -19,7 +19,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -60,7 +60,7 @@
         public String host;
 
         @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
-        public int port = 1099;
+        public int port = 1098;
 
         @Option(name = "-app", usage = "Hyracks Application name", required = true)
         public String app;
@@ -80,7 +80,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job = createJob(options);
 
diff --git a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
index 35deb57..36e315d 100644
--- a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
+++ b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
@@ -58,8 +58,8 @@
     }
 
     @Override
-    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
-            int partition, int nPartitions) {
+    public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+            IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
 
         final ByteBuffer outputFrame = ctx.allocateFrame();
         final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
@@ -82,7 +82,7 @@
                     appender.reset(outputFrame, true);
                     for (int i = 0; i < numRecords; i++) {
                         tb.reset();
-                        for (int j = 0; j < recDesc.getFields().length; j++) {
+                        for (int j = 0; j < recDesc.getFieldCount(); j++) {
                             genField(tb, j);
                         }
 
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
index 15d31b6..fe71db4 100644
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
+++ b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
@@ -77,7 +77,7 @@
       <plugin>
       	<groupId>edu.uci.ics.hyracks</groupId>
       	<artifactId>hyracks-maven-plugin</artifactId>
-      	<version>0.0.1</version>
+      	<version>0.0.2</version>
         <configuration>
           <hyracksServerHome>${basedir}/../../../hyracks-server/target/hyracks-server-${project.version}-binary-assembly</hyracksServerHome>
           <hyracksCLIHome>${basedir}/../../../hyracks-cli/target/hyracks-cli-${project.version}-binary-assembly</hyracksCLIHome>
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
index abedf4a..4ed8361 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
@@ -28,7 +28,7 @@
 import org.junit.Rule;
 import org.junit.rules.TemporaryFolder;
 
-import edu.uci.ics.hyracks.api.client.HyracksLocalConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.job.JobFlag;
 import edu.uci.ics.hyracks.api.job.JobId;
@@ -61,7 +61,10 @@
     @BeforeClass
     public static void init() throws Exception {
         CCConfig ccConfig = new CCConfig();
-        ccConfig.port = 39001;
+        ccConfig.clientNetIpAddress = "127.0.0.1";
+        ccConfig.clientNetPort = 39000;
+        ccConfig.clusterNetIpAddress = "127.0.0.1";
+        ccConfig.clusterNetPort = 39001;
         ccConfig.profileDumpPeriod = 10000;
         File outDir = new File("target/ClusterController");
         outDir.mkdirs();
@@ -75,6 +78,7 @@
         NCConfig ncConfig1 = new NCConfig();
         ncConfig1.ccHost = "localhost";
         ncConfig1.ccPort = 39001;
+        ncConfig1.clusterNetIPAddress = "127.0.0.1";
         ncConfig1.dataIPAddress = "127.0.0.1";
         ncConfig1.nodeId = NC1_ID;
         nc1 = new NodeControllerService(ncConfig1);
@@ -83,12 +87,13 @@
         NCConfig ncConfig2 = new NCConfig();
         ncConfig2.ccHost = "localhost";
         ncConfig2.ccPort = 39001;
+        ncConfig2.clusterNetIPAddress = "127.0.0.1";
         ncConfig2.dataIPAddress = "127.0.0.1";
         ncConfig2.nodeId = NC2_ID;
         nc2 = new NodeControllerService(ncConfig2);
         nc2.start();
 
-        hcc = new HyracksLocalConnection(cc);
+        hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
         hcc.createApplication("test", null);
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath());
@@ -111,7 +116,7 @@
         if (LOGGER.isLoggable(Level.INFO)) {
             LOGGER.info(jobId.toString());
         }
-        cc.waitForCompletion(jobId);
+        hcc.waitForCompletion(jobId);
         dumpOutputFiles();
     }
 
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
index 5571a4f..61d4696 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
@@ -378,7 +378,7 @@
                         UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
         PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
 
-        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFields().length];
+        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFieldCount()];
         for (int j = 0; j < nullWriterFactories.length; j++) {
             nullWriterFactories[j] = NoopNullWriterFactory.INSTANCE;
         }
@@ -460,7 +460,7 @@
                         UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
         PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
 
-        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFields().length];
+        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFieldCount()];
         for (int j = 0; j < nullWriterFactories.length; j++) {
             nullWriterFactories[j] = NoopNullWriterFactory.INSTANCE;
         }
@@ -546,7 +546,7 @@
                         UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
         PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
 
-        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFields().length];
+        INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFieldCount()];
         for (int j = 0; j < nullWriterFactories.length; j++) {
             nullWriterFactories[j] = NoopNullWriterFactory.INSTANCE;
         }
diff --git a/hyracks-examples/text-example/textapp/pom.xml b/hyracks-examples/text-example/textapp/pom.xml
index 3b3b9fd..913a95a 100644
--- a/hyracks-examples/text-example/textapp/pom.xml
+++ b/hyracks-examples/text-example/textapp/pom.xml
@@ -77,7 +77,7 @@
       <plugin>
       	<groupId>edu.uci.ics.hyracks</groupId>
       	<artifactId>hyracks-maven-plugin</artifactId>
-      	<version>0.0.1</version>
+      	<version>0.0.2</version>
         <configuration>
           <hyracksServerHome>${basedir}/../../../hyracks-server/target/hyracks-server-${project.version}-binary-assembly</hyracksServerHome>
           <hyracksCLIHome>${basedir}/../../../hyracks-cli/target/hyracks-cli-${project.version}-binary-assembly</hyracksCLIHome>
diff --git a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
index a1a5179..f594d7e 100644
--- a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
+++ b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
@@ -19,7 +19,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
 import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
@@ -112,7 +112,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job;
 
diff --git a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
index 17acc10..fe3cc03 100644
--- a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
+++ b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
@@ -20,7 +20,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
 import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
@@ -63,7 +63,7 @@
         public String host;
 
         @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
-        public int port = 1099;
+        public int port = 1098;
 
         @Option(name = "-app", usage = "Hyracks Application name", required = true)
         public String app;
@@ -95,7 +95,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits),
                 options.algo, options.htSize, options.sbSize, options.format);
diff --git a/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java b/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
index cffa0f4..e876f65 100644
--- a/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
+++ b/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
@@ -20,7 +20,7 @@
 import org.kohsuke.args4j.CmdLineParser;
 import org.kohsuke.args4j.Option;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
 import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
 import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
@@ -69,7 +69,7 @@
         public String host;
 
         @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)", required = false)
-        public int port = 1099;
+        public int port = 1098;
 
         @Option(name = "-app", usage = "Hyracks Application name", required = true)
         public String app;
@@ -118,7 +118,7 @@
         CmdLineParser parser = new CmdLineParser(options);
         parser.parseArgument(args);
 
-        IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+        IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
 
         JobSpecification job = createJob(parseFileSplits(options.inFileCustomerSplits),
                 parseFileSplits(options.inFileOrderSplits), parseFileSplits(options.outFileSplits),
diff --git a/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java b/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
index 9e76a3e..eadf679 100644
--- a/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
+++ b/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
@@ -5,7 +5,7 @@
 import java.util.Properties;
 import java.util.Set;
 
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
 import edu.uci.ics.hyracks.api.job.JobFlag;
 import edu.uci.ics.hyracks.api.job.JobId;
 import edu.uci.ics.hyracks.api.job.JobSpecification;
@@ -15,7 +15,7 @@
 
 public class HyracksClient {
 
-    private static HyracksRMIConnection connection;
+    private static HyracksConnection connection;
     private static final String jobProfilingKey = "jobProfilingKey";
     Set<String> systemLibs;
 
@@ -25,7 +25,7 @@
 
     private void initialize(Properties properties) throws Exception {
         String clusterController = (String) properties.get(ConfigurationConstants.clusterControllerHost);
-        connection = new HyracksRMIConnection(clusterController, 1099);
+        connection = new HyracksConnection(clusterController, 1098);
         systemLibs = new HashSet<String>();
         for (String systemLib : ConfigurationConstants.systemLibs) {
             String systemLibPath = properties.getProperty(systemLib);
diff --git a/hyracks-ipc/pom.xml b/hyracks-ipc/pom.xml
new file mode 100644
index 0000000..49c4323
--- /dev/null
+++ b/hyracks-ipc/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>hyracks-ipc</artifactId>
+  <parent>
+    <groupId>edu.uci.ics.hyracks</groupId>
+    <artifactId>hyracks</artifactId>
+    <version>0.2.0-SNAPSHOT</version>
+  </parent>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>2.0.2</version>
+        <configuration>
+          <source>1.6</source>
+          <target>1.6</target>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+  <dependencies>
+  <dependency>
+  	<groupId>junit</groupId>
+  	<artifactId>junit</artifactId>
+  	<version>4.8.1</version>
+  	<scope>test</scope>
+  </dependency>
+  </dependencies>
+</project>
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCHandle.java
similarity index 61%
copy from hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
copy to hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCHandle.java
index 2364d74..8a3630f 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCHandle.java
@@ -12,18 +12,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.api.util;
+package edu.uci.ics.hyracks.ipc.api;
 
-import java.io.Serializable;
+import java.net.InetSocketAddress;
 
-public final class Pair<F, S> implements Serializable {
-    private static final long serialVersionUID = 1L;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
 
-    public F first;
-    public S second;
+public interface IIPCHandle {
+    public InetSocketAddress getRemoteAddress();
 
-    public Pair(F first, S second) {
-        this.first = first;
-        this.second = second;
-    }
-}
+    public long send(long requestId, Object payload, Exception exception) throws IPCException;
+
+    public void setAttachment(Object attachment);
+
+    public Object getAttachment();
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCI.java
similarity index 77%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCI.java
index c728b0b..24ab943 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCI.java
@@ -12,10 +12,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.ipc.api;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
-
-    public R getResult();
+public interface IIPCI {
+    public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception);
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IResponseCallback.java
similarity index 79%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IResponseCallback.java
index c728b0b..7d25f88 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IResponseCallback.java
@@ -12,10 +12,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.ipc.api;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
-
-    public R getResult();
+public interface IResponseCallback {
+    public void callback(IIPCHandle handle, Object response, Exception exception);
 }
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/RPCInterface.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/RPCInterface.java
new file mode 100644
index 0000000..3340516
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/RPCInterface.java
@@ -0,0 +1,72 @@
+package edu.uci.ics.hyracks.ipc.api;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class RPCInterface implements IIPCI {
+    private final Map<Long, Request> reqMap;
+
+    public RPCInterface() {
+        reqMap = new HashMap<Long, RPCInterface.Request>();
+    }
+
+    public Object call(IIPCHandle handle, Object request) throws Exception {
+        Request req;
+        synchronized (this) {
+            req = new Request();
+            long mid = handle.send(-1, request, null);
+            reqMap.put(mid, req);
+        }
+        return req.getResponse();
+    }
+
+    @Override
+    public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+        Request req;
+        synchronized (this) {
+            req = reqMap.remove(rmid);
+        }
+        assert req != null;
+        if (exception != null) {
+            req.setException(exception);
+        } else {
+            req.setResult(payload);
+        }
+    }
+
+    private static class Request {
+        private boolean pending;
+
+        private Object result;
+
+        private Exception exception;
+
+        Request() {
+            pending = true;
+            result = null;
+            exception = null;
+        }
+
+        synchronized void setResult(Object result) {
+            this.pending = false;
+            this.result = result;
+            notifyAll();
+        }
+
+        synchronized void setException(Exception exception) {
+            this.pending = false;
+            this.exception = exception;
+            notifyAll();
+        }
+
+        synchronized Object getResponse() throws Exception {
+            while (pending) {
+                wait();
+            }
+            if (exception != null) {
+                throw exception;
+            }
+            return result;
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/exceptions/IPCException.java
similarity index 64%
copy from hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
copy to hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/exceptions/IPCException.java
index 2364d74..9ecf015 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/exceptions/IPCException.java
@@ -12,18 +12,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.api.util;
+package edu.uci.ics.hyracks.ipc.exceptions;
 
-import java.io.Serializable;
-
-public final class Pair<F, S> implements Serializable {
+public class IPCException extends Exception {
     private static final long serialVersionUID = 1L;
 
-    public F first;
-    public S second;
-
-    public Pair(F first, S second) {
-        this.first = first;
-        this.second = second;
+    public IPCException() {
+        super();
     }
-}
+
+    public IPCException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public IPCException(String message) {
+        super(message);
+    }
+
+    public IPCException(Throwable cause) {
+        super(cause);
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/HandleState.java
similarity index 80%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/HandleState.java
index c728b0b..47c3d4a 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/HandleState.java
@@ -12,10 +12,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.ipc.impl;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
-
-    public R getResult();
+enum HandleState {
+    INITIAL,
+    CONNECT_SENT,
+    CONNECT_RECEIVED,
+    CONNECTED,
+    CLOSED,
 }
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCConnectionManager.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCConnectionManager.java
new file mode 100644
index 0000000..eb9cbd3
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCConnectionManager.java
@@ -0,0 +1,282 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class IPCConnectionManager {
+    private static final Logger LOGGER = Logger.getLogger(IPCConnectionManager.class.getName());
+
+    private final IPCSystem system;
+
+    private final NetworkThread networkThread;
+
+    private final ServerSocketChannel serverSocketChannel;
+
+    private final Map<InetSocketAddress, IPCHandle> ipcHandleMap;
+
+    private final List<IPCHandle> pendingConnections;
+
+    private final List<IPCHandle> workingPendingConnections;
+
+    private final List<Message> sendList;
+
+    private final List<Message> workingSendList;
+
+    private final InetSocketAddress address;
+
+    private volatile boolean stopped;
+
+    IPCConnectionManager(IPCSystem system, InetSocketAddress socketAddress) throws IOException {
+        this.system = system;
+        this.networkThread = new NetworkThread();
+        this.serverSocketChannel = ServerSocketChannel.open();
+        serverSocketChannel.socket().setReuseAddress(true);
+        serverSocketChannel.configureBlocking(false);
+        ServerSocket socket = serverSocketChannel.socket();
+        socket.bind(socketAddress);
+        address = new InetSocketAddress(socket.getInetAddress(), socket.getLocalPort());
+        ipcHandleMap = new HashMap<InetSocketAddress, IPCHandle>();
+        pendingConnections = new ArrayList<IPCHandle>();
+        workingPendingConnections = new ArrayList<IPCHandle>();
+        sendList = new ArrayList<Message>();
+        workingSendList = new ArrayList<Message>();
+    }
+
+    InetSocketAddress getAddress() {
+        return address;
+    }
+
+    void start() {
+        stopped = false;
+        networkThread.start();
+    }
+
+    void stop() throws IOException {
+        stopped = true;
+        serverSocketChannel.close();
+    }
+
+    IPCHandle getIPCHandle(InetSocketAddress remoteAddress) throws IOException, InterruptedException {
+        IPCHandle handle;
+        synchronized (this) {
+            handle = ipcHandleMap.get(remoteAddress);
+            if (handle == null) {
+                handle = new IPCHandle(system, remoteAddress);
+                pendingConnections.add(handle);
+                networkThread.selector.wakeup();
+            }
+        }
+        handle.waitTillConnected();
+        return handle;
+    }
+
+    synchronized void registerHandle(IPCHandle handle) {
+        ipcHandleMap.put(handle.getRemoteAddress(), handle);
+    }
+
+    synchronized void write(Message msg) {
+        if (LOGGER.isLoggable(Level.FINE)) {
+            LOGGER.fine("Enqueued message: " + msg);
+        }
+        sendList.add(msg);
+        networkThread.selector.wakeup();
+    }
+
+    private synchronized void collectOutstandingWork() {
+        if (!pendingConnections.isEmpty()) {
+            workingPendingConnections.addAll(pendingConnections);
+            pendingConnections.clear();
+        }
+        if (!sendList.isEmpty()) {
+            workingSendList.addAll(sendList);
+            sendList.clear();
+        }
+    }
+
+    private Message createInitialReqMessage(IPCHandle handle) {
+        Message msg = new Message(handle);
+        msg.setMessageId(system.createMessageId());
+        msg.setRequestMessageId(-1);
+        msg.setFlag(Message.INITIAL_REQ);
+        msg.setPayload(address);
+        return msg;
+    }
+
+    private Message createInitialAckMessage(IPCHandle handle, Message req) {
+        Message msg = new Message(handle);
+        msg.setMessageId(system.createMessageId());
+        msg.setRequestMessageId(req.getMessageId());
+        msg.setFlag(Message.INITIAL_ACK);
+        msg.setPayload(null);
+        return msg;
+    }
+
+    void ack(IPCHandle handle, Message req) {
+        write(createInitialAckMessage(handle, req));
+    }
+
+    private class NetworkThread extends Thread {
+        private final Selector selector;
+
+        public NetworkThread() {
+            super("IPC Network Listener Thread");
+            setDaemon(true);
+            try {
+                selector = Selector.open();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void run() {
+            try {
+                serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT);
+            } catch (ClosedChannelException e) {
+                throw new RuntimeException(e);
+            }
+            while (!stopped) {
+                try {
+                    if (LOGGER.isLoggable(Level.FINE)) {
+                        LOGGER.fine("Starting Select");
+                    }
+                    int n = selector.select();
+                    collectOutstandingWork();
+                    if (!workingPendingConnections.isEmpty()) {
+                        for (IPCHandle handle : workingPendingConnections) {
+                            SocketChannel channel = SocketChannel.open();
+                            channel.configureBlocking(false);
+                            SelectionKey cKey = null;
+                            if (channel.connect(handle.getRemoteAddress())) {
+                                cKey = channel.register(selector, SelectionKey.OP_READ);
+                                handle.setState(HandleState.CONNECT_SENT);
+                                write(createInitialReqMessage(handle));
+                            } else {
+                                cKey = channel.register(selector, SelectionKey.OP_CONNECT);
+                            }
+                            handle.setKey(cKey);
+                            cKey.attach(handle);
+                        }
+                        workingPendingConnections.clear();
+                    }
+                    if (!workingSendList.isEmpty()) {
+                        for (Iterator<Message> i = workingSendList.iterator(); i.hasNext();) {
+                            Message msg = i.next();
+                            if (LOGGER.isLoggable(Level.FINE)) {
+                                LOGGER.fine("Processing send of message: " + msg);
+                            }
+                            IPCHandle handle = msg.getIPCHandle();
+                            if (handle.getState() == HandleState.CLOSED) {
+                                i.remove();
+                            } else if (!handle.full()) {
+                                while (true) {
+                                    ByteBuffer buffer = handle.getOutBuffer();
+                                    buffer.compact();
+                                    boolean success = msg.write(buffer);
+                                    buffer.flip();
+                                    if (success) {
+                                        i.remove();
+                                        SelectionKey key = handle.getKey();
+                                        key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+                                    } else {
+                                        if (buffer.position() == 0) {
+                                            handle.resizeOutBuffer();
+                                            continue;
+                                        }
+                                        handle.markFull();
+                                    }
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                    if (n > 0) {
+                        for (Iterator<SelectionKey> i = selector.selectedKeys().iterator(); i.hasNext();) {
+                            SelectionKey key = i.next();
+                            i.remove();
+                            SelectableChannel sc = key.channel();
+                            if (key.isReadable()) {
+                                SocketChannel channel = (SocketChannel) sc;
+                                IPCHandle handle = (IPCHandle) key.attachment();
+                                ByteBuffer readBuffer = handle.getInBuffer();
+                                int len = channel.read(readBuffer);
+                                if (len < 0) {
+                                    key.cancel();
+                                    channel.close();
+                                    handle.close();
+                                } else {
+                                    handle.processIncomingMessages();
+                                    if (!readBuffer.hasRemaining()) {
+                                        handle.resizeInBuffer();
+                                    }
+                                }
+                            } else if (key.isWritable()) {
+                                SocketChannel channel = (SocketChannel) sc;
+                                IPCHandle handle = (IPCHandle) key.attachment();
+                                ByteBuffer writeBuffer = handle.getOutBuffer();
+                                int len = channel.write(writeBuffer);
+                                if (len < 0) {
+                                    key.cancel();
+                                    channel.close();
+                                    handle.close();
+                                } else if (!writeBuffer.hasRemaining()) {
+                                    key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE);
+                                }
+                                handle.clearFull();
+                            } else if (key.isAcceptable()) {
+                                assert sc == serverSocketChannel;
+                                SocketChannel channel = serverSocketChannel.accept();
+                                channel.configureBlocking(false);
+                                IPCHandle handle = new IPCHandle(system, null);
+                                SelectionKey cKey = channel.register(selector, SelectionKey.OP_READ);
+                                handle.setKey(cKey);
+                                cKey.attach(handle);
+                                handle.setState(HandleState.CONNECT_RECEIVED);
+                            } else if (key.isConnectable()) {
+                                SocketChannel channel = (SocketChannel) sc;
+                                if (channel.finishConnect()) {
+                                    IPCHandle handle = (IPCHandle) key.attachment();
+                                    handle.setState(HandleState.CONNECT_SENT);
+                                    registerHandle(handle);
+                                    key.interestOps(SelectionKey.OP_READ);
+                                    write(createInitialReqMessage(handle));
+                                }
+                            }
+                        }
+                    }
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java
new file mode 100644
index 0000000..f3c05c6
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+
+final class IPCHandle implements IIPCHandle {
+    private static final int DEFAULT_BUFFER_SIZE = 1024 * 1024;
+
+    private final IPCSystem system;
+
+    private InetSocketAddress remoteAddress;
+
+    private HandleState state;
+
+    private SelectionKey key;
+
+    private Object attachment;
+
+    private ByteBuffer inBuffer;
+
+    private ByteBuffer outBuffer;
+
+    private boolean full;
+
+    IPCHandle(IPCSystem system, InetSocketAddress remoteAddress) {
+        this.system = system;
+        this.remoteAddress = remoteAddress;
+        inBuffer = ByteBuffer.allocate(DEFAULT_BUFFER_SIZE);
+        outBuffer = ByteBuffer.allocate(DEFAULT_BUFFER_SIZE);
+        outBuffer.flip();
+        state = HandleState.INITIAL;
+    }
+
+    @Override
+    public InetSocketAddress getRemoteAddress() {
+        return remoteAddress;
+    }
+
+    void setRemoteAddress(InetSocketAddress remoteAddress) {
+        this.remoteAddress = remoteAddress;
+    }
+
+    @Override
+    public synchronized long send(long requestId, Object req, Exception exception) throws IPCException {
+        if (state != HandleState.CONNECTED) {
+            throw new IPCException("Handle is not in Connected state");
+        }
+        Message msg = new Message(this);
+        long mid = system.createMessageId();
+        msg.setMessageId(mid);
+        msg.setRequestMessageId(requestId);
+        if (exception != null) {
+            msg.setFlag(Message.ERROR);
+            msg.setPayload(exception);
+        } else {
+            msg.setFlag(Message.NORMAL);
+            msg.setPayload(req);
+        }
+        system.getConnectionManager().write(msg);
+        return mid;
+    }
+
+    @Override
+    public void setAttachment(Object attachment) {
+        this.attachment = attachment;
+    }
+
+    @Override
+    public Object getAttachment() {
+        return attachment;
+    }
+
+    SelectionKey getKey() {
+        return key;
+    }
+
+    void setKey(SelectionKey key) {
+        this.key = key;
+    }
+
+    public synchronized boolean isConnected() {
+        return state == HandleState.CONNECTED;
+    }
+
+    synchronized HandleState getState() {
+        return state;
+    }
+
+    synchronized void setState(HandleState state) {
+        this.state = state;
+        notifyAll();
+    }
+
+    synchronized void waitTillConnected() throws InterruptedException {
+        while (!isConnected()) {
+            wait();
+        }
+    }
+
+    ByteBuffer getInBuffer() {
+        return inBuffer;
+    }
+
+    ByteBuffer getOutBuffer() {
+        return outBuffer;
+    }
+
+    synchronized void close() {
+        setState(HandleState.CLOSED);
+    }
+
+    synchronized void processIncomingMessages() {
+        inBuffer.flip();
+        while (Message.hasMessage(inBuffer)) {
+            Message message = new Message(this);
+            try {
+                message.read(inBuffer);
+            } catch (Exception e) {
+                message.setFlag(Message.ERROR);
+                message.setPayload(e);
+            }
+
+            if (state == HandleState.CONNECT_RECEIVED) {
+                remoteAddress = (InetSocketAddress) message.getPayload();
+                system.getConnectionManager().registerHandle(this);
+                setState(HandleState.CONNECTED);
+                system.getConnectionManager().ack(this, message);
+                continue;
+            } else if (state == HandleState.CONNECT_SENT) {
+                if (message.getFlag() == Message.INITIAL_ACK) {
+                    setState(HandleState.CONNECTED);
+                } else {
+                    throw new IllegalStateException();
+                }
+                continue;
+            }
+            system.deliverIncomingMessage(message);
+        }
+        inBuffer.compact();
+    }
+
+    void resizeInBuffer() {
+        inBuffer.flip();
+        ByteBuffer readBuffer = ByteBuffer.allocate(inBuffer.capacity() * 2);
+        readBuffer.put(inBuffer);
+        readBuffer.compact();
+        inBuffer = readBuffer;
+    }
+
+    void resizeOutBuffer() {
+        ByteBuffer writeBuffer = ByteBuffer.allocate(outBuffer.capacity() * 2);
+        writeBuffer.put(outBuffer);
+        writeBuffer.compact();
+        writeBuffer.flip();
+        outBuffer = writeBuffer;
+    }
+
+    void markFull() {
+        full = true;
+    }
+
+    void clearFull() {
+        full = false;
+    }
+
+    boolean full() {
+        return full;
+    }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCSystem.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCSystem.java
new file mode 100644
index 0000000..5f60c6d
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCSystem.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicLong;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+
+public class IPCSystem {
+    private final IPCConnectionManager cMgr;
+
+    private final IIPCI ipci;
+
+    private final AtomicLong midFactory;
+
+    public IPCSystem(InetSocketAddress socketAddress) throws IOException {
+        this(socketAddress, null);
+    }
+
+    public IPCSystem(InetSocketAddress socketAddress, IIPCI ipci) throws IOException {
+        cMgr = new IPCConnectionManager(this, socketAddress);
+        this.ipci = ipci;
+        midFactory = new AtomicLong();
+    }
+
+    public InetSocketAddress getSocketAddress() {
+        return cMgr.getAddress();
+    }
+
+    public void start() {
+        cMgr.start();
+    }
+
+    public IIPCHandle getHandle(InetSocketAddress remoteAddress) throws IPCException {
+        try {
+            return cMgr.getIPCHandle(remoteAddress);
+        } catch (IOException e) {
+            throw new IPCException(e);
+        } catch (InterruptedException e) {
+            throw new IPCException(e);
+        }
+    }
+
+    long createMessageId() {
+        return midFactory.incrementAndGet();
+    }
+
+    void deliverIncomingMessage(final Message message) {
+        long mid = message.getMessageId();
+        long rmid = message.getRequestMessageId();
+        Object payload = null;
+        Exception exception = null;
+        if (message.getFlag() == Message.ERROR) {
+            exception = (Exception) message.getPayload();
+        } else {
+            payload = message.getPayload();
+        }
+        ipci.deliverIncomingMessage(message.getIPCHandle(), mid, rmid, payload, exception);
+    }
+
+    IPCConnectionManager getConnectionManager() {
+        return cMgr;
+    }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/Message.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/Message.java
new file mode 100644
index 0000000..6bff56f
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/Message.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.nio.ByteBuffer;
+
+class Message {
+    private static final int MSG_SIZE_SIZE = 4;
+
+    private static final int HEADER_SIZE = 17;
+
+    static final byte INITIAL_REQ = 1;
+
+    static final byte INITIAL_ACK = 2;
+
+    static final byte ERROR = 3;
+
+    static final byte NORMAL = 0;
+
+    private IPCHandle ipcHandle;
+
+    private long messageId;
+
+    private long requestMessageId;
+
+    private byte flag;
+
+    private Object payload;
+
+    Message(IPCHandle ipcHandle) {
+        this.ipcHandle = ipcHandle;
+    }
+
+    IPCHandle getIPCHandle() {
+        return ipcHandle;
+    }
+
+    void setMessageId(long messageId) {
+        this.messageId = messageId;
+    }
+
+    long getMessageId() {
+        return messageId;
+    }
+
+    void setRequestMessageId(long requestMessageId) {
+        this.requestMessageId = requestMessageId;
+    }
+
+    long getRequestMessageId() {
+        return requestMessageId;
+    }
+
+    void setFlag(byte flag) {
+        this.flag = flag;
+    }
+
+    byte getFlag() {
+        return flag;
+    }
+
+    void setPayload(Object payload) {
+        this.payload = payload;
+    }
+
+    Object getPayload() {
+        return payload;
+    }
+
+    static boolean hasMessage(ByteBuffer buffer) {
+        if (buffer.remaining() < MSG_SIZE_SIZE) {
+            return false;
+        }
+        int msgSize = buffer.getInt(buffer.position());
+        return buffer.remaining() >= msgSize + MSG_SIZE_SIZE;
+    }
+
+    void read(ByteBuffer buffer) throws IOException, ClassNotFoundException {
+        assert hasMessage(buffer);
+        int msgSize = buffer.getInt();
+        messageId = buffer.getLong();
+        requestMessageId = buffer.getLong();
+        flag = buffer.get();
+        int finalPosition = buffer.position() + msgSize - HEADER_SIZE;
+        try {
+            ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(buffer.array(), buffer.position(),
+                    msgSize - HEADER_SIZE));
+            payload = ois.readObject();
+            ois.close();
+        } finally {
+            buffer.position(finalPosition);
+        }
+    }
+
+    boolean write(ByteBuffer buffer) throws IOException {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream oos = new ObjectOutputStream(baos);
+        oos.writeObject(payload);
+        oos.close();
+        byte[] bytes = baos.toByteArray();
+        if (buffer.remaining() >= MSG_SIZE_SIZE + HEADER_SIZE + bytes.length) {
+            buffer.putInt(HEADER_SIZE + bytes.length);
+            buffer.putLong(messageId);
+            buffer.putLong(requestMessageId);
+            buffer.put(flag);
+            buffer.put(bytes);
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return "MSG[" + messageId + ":" + requestMessageId + ":" + flag + ":" + payload + "]";
+    }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/test/java/edu/uci/ics/hyracks/ipc/tests/IPCTest.java b/hyracks-ipc/src/test/java/edu/uci/ics/hyracks/ipc/tests/IPCTest.java
new file mode 100644
index 0000000..dac93dd
--- /dev/null
+++ b/hyracks-ipc/src/test/java/edu/uci/ics/hyracks/ipc/tests/IPCTest.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.tests;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.api.RPCInterface;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
+
+public class IPCTest {
+    @Test
+    public void test() throws Exception {
+        IPCSystem server = createServerIPCSystem();
+        server.start();
+        InetSocketAddress serverAddr = server.getSocketAddress();
+
+        RPCInterface rpci = new RPCInterface();
+        IPCSystem client = createClientIPCSystem(rpci);
+        client.start();
+
+        IIPCHandle handle = client.getHandle(serverAddr);
+
+        for (int i = 0; i < 100; ++i) {
+            Assert.assertEquals(rpci.call(handle, Integer.valueOf(i)), Integer.valueOf(2 * i));
+        }
+
+        try {
+            rpci.call(handle, "Foo");
+            Assert.assertTrue(false);
+        } catch (Exception e) {
+            Assert.assertTrue(true);
+        }
+    }
+
+    private IPCSystem createServerIPCSystem() throws IOException {
+        final Executor executor = Executors.newCachedThreadPool();
+        IIPCI ipci = new IIPCI() {
+            @Override
+            public void deliverIncomingMessage(final IIPCHandle handle, final long mid, long rmid,
+                    final Object payload, Exception exception) {
+                executor.execute(new Runnable() {
+                    @Override
+                    public void run() {
+                        Object result = null;
+                        Exception exception = null;
+                        try {
+                            Integer i = (Integer) payload;
+                            result = i.intValue() * 2;
+                        } catch (Exception e) {
+                            exception = e;
+                        }
+                        try {
+                            handle.send(mid, result, exception);
+                        } catch (IPCException e) {
+                            e.printStackTrace();
+                        }
+                    }
+                });
+            }
+        };
+        return new IPCSystem(new InetSocketAddress("127.0.0.1", 0), ipci);
+    }
+
+    private IPCSystem createClientIPCSystem(RPCInterface rpci) throws IOException {
+        return new IPCSystem(new InetSocketAddress("127.0.0.1", 0), rpci);
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/pom.xml b/hyracks-net/pom.xml
new file mode 100644
index 0000000..12004f7
--- /dev/null
+++ b/hyracks-net/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>hyracks-net</artifactId>
+  <parent>
+    <groupId>edu.uci.ics.hyracks</groupId>
+    <artifactId>hyracks</artifactId>
+    <version>0.2.0-SNAPSHOT</version>
+  </parent>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>2.0.2</version>
+        <configuration>
+          <source>1.6</source>
+          <target>1.6</target>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+  <dependencies>
+  <dependency>
+  	<groupId>junit</groupId>
+  	<artifactId>junit</artifactId>
+  	<version>4.8.1</version>
+  	<scope>test</scope>
+  </dependency>
+  </dependencies>
+</project>
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
similarity index 80%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
index c728b0b..2f27bf0 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
@@ -12,10 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.net.buffers;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
+import java.nio.ByteBuffer;
 
-    public R getResult();
+public interface IBufferAcceptor {
+    public void accept(ByteBuffer buffer);
 }
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
similarity index 78%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
index c728b0b..c395ac9 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
@@ -12,10 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.net.buffers;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
+public interface ICloseableBufferAcceptor extends IBufferAcceptor {
+    public void close();
 
-    public R getResult();
+    public void error(int ecode);
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/exceptions/NetException.java
similarity index 65%
copy from hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
copy to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/exceptions/NetException.java
index 2364d74..ecd0373 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/exceptions/NetException.java
@@ -12,18 +12,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.api.util;
+package edu.uci.ics.hyracks.net.exceptions;
 
-import java.io.Serializable;
-
-public final class Pair<F, S> implements Serializable {
+public class NetException extends Exception {
     private static final long serialVersionUID = 1L;
 
-    public F first;
-    public S second;
-
-    public Pair(F first, S second) {
-        this.first = first;
-        this.second = second;
+    public NetException() {
     }
-}
+
+    public NetException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public NetException(String message) {
+        super(message);
+    }
+
+    public NetException(Throwable cause) {
+        super(cause);
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
new file mode 100644
index 0000000..99140db
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+
+public class ChannelControlBlock {
+    private final ChannelSet cSet;
+
+    private final int channelId;
+
+    private final ReadInterface ri;
+
+    private final WriteInterface wi;
+
+    private final AtomicBoolean localClose;
+
+    private final AtomicBoolean localCloseAck;
+
+    private final AtomicBoolean remoteClose;
+
+    ChannelControlBlock(ChannelSet cSet, int channelId) {
+        this.cSet = cSet;
+        this.channelId = channelId;
+        this.ri = new ReadInterface();
+        this.wi = new WriteInterface();
+        localClose = new AtomicBoolean();
+        localCloseAck = new AtomicBoolean();
+        remoteClose = new AtomicBoolean();
+    }
+
+    int getChannelId() {
+        return channelId;
+    }
+
+    public IChannelReadInterface getReadInterface() {
+        return ri;
+    }
+
+    public IChannelWriteInterface getWriteInterface() {
+        return wi;
+    }
+
+    private final class ReadInterface implements IChannelReadInterface {
+        private final Queue<ByteBuffer> riEmptyQueue;
+
+        private final IBufferAcceptor eba = new IBufferAcceptor() {
+            @Override
+            public void accept(ByteBuffer buffer) {
+                if (remoteClose.get()) {
+                    return;
+                }
+                int delta;
+                synchronized (ChannelControlBlock.this) {
+                    riEmptyQueue.add(buffer);
+                    delta = buffer.remaining();
+                }
+                credits.addAndGet(delta);
+                if (delta != 0) {
+                    cSet.markPendingCredits(channelId);
+                }
+            }
+        };
+
+        private ICloseableBufferAcceptor fba;
+
+        private final AtomicInteger credits;
+
+        private ByteBuffer currentReadBuffer;
+
+        ReadInterface() {
+            riEmptyQueue = new LinkedList<ByteBuffer>();
+            credits = new AtomicInteger();
+        }
+
+        @Override
+        public void setFullBufferAcceptor(ICloseableBufferAcceptor fullBufferAcceptor) {
+            fba = fullBufferAcceptor;
+        }
+
+        @Override
+        public IBufferAcceptor getEmptyBufferAcceptor() {
+            return eba;
+        }
+
+        int read(SocketChannel sc, int size) throws IOException, NetException {
+            while (true) {
+                if (size <= 0) {
+                    return size;
+                }
+                if (ri.currentReadBuffer == null) {
+                    ri.currentReadBuffer = ri.riEmptyQueue.poll();
+                    assert ri.currentReadBuffer != null;
+                }
+                int rSize = Math.min(size, ri.currentReadBuffer.remaining());
+                if (rSize > 0) {
+                    ri.currentReadBuffer.limit(ri.currentReadBuffer.position() + rSize);
+                    int len;
+                    try {
+                        len = sc.read(ri.currentReadBuffer);
+                        if (len < 0) {
+                            throw new NetException("Socket Closed");
+                        }
+                    } finally {
+                        ri.currentReadBuffer.limit(ri.currentReadBuffer.capacity());
+                    }
+                    size -= len;
+                    if (len < rSize) {
+                        return size;
+                    }
+                } else {
+                    return size;
+                }
+                if (ri.currentReadBuffer.remaining() <= 0) {
+                    flush();
+                }
+            }
+        }
+
+        void flush() {
+            if (currentReadBuffer != null) {
+                currentReadBuffer.flip();
+                fba.accept(ri.currentReadBuffer);
+                currentReadBuffer = null;
+            }
+        }
+    }
+
+    private final class WriteInterface implements IChannelWriteInterface {
+        private final Queue<ByteBuffer> wiFullQueue;
+
+        private int channelWriteEventCount;
+
+        private final ICloseableBufferAcceptor fba = new ICloseableBufferAcceptor() {
+            @Override
+            public void accept(ByteBuffer buffer) {
+                synchronized (ChannelControlBlock.this) {
+                    wiFullQueue.add(buffer);
+                    incrementLocalWriteEventCount();
+                }
+            }
+
+            @Override
+            public void close() {
+                synchronized (ChannelControlBlock.this) {
+                    if (eos) {
+                        return;
+                    }
+                    eos = true;
+                    incrementLocalWriteEventCount();
+                }
+            }
+
+            @Override
+            public void error(int ecode) {
+                synchronized (ChannelControlBlock.this) {
+                    WriteInterface.this.ecode = ecode;
+                    incrementLocalWriteEventCount();
+                }
+            }
+        };
+
+        private IBufferAcceptor eba;
+
+        private final AtomicInteger credits;
+
+        private boolean eos;
+
+        private boolean eosSent;
+
+        private int ecode;
+
+        private boolean ecodeSent;
+
+        private ByteBuffer currentWriteBuffer;
+
+        WriteInterface() {
+            wiFullQueue = new LinkedList<ByteBuffer>();
+            credits = new AtomicInteger();
+            eos = false;
+            eosSent = false;
+            ecode = -1;
+            ecodeSent = false;
+        }
+
+        @Override
+        public void setEmptyBufferAcceptor(IBufferAcceptor emptyBufferAcceptor) {
+            eba = emptyBufferAcceptor;
+        }
+
+        @Override
+        public ICloseableBufferAcceptor getFullBufferAcceptor() {
+            return fba;
+        }
+
+        void write(MultiplexedConnection.WriterState writerState) throws NetException {
+            if (currentWriteBuffer == null) {
+                currentWriteBuffer = wiFullQueue.poll();
+            }
+            if (currentWriteBuffer != null) {
+                int size = Math.min(currentWriteBuffer.remaining(), credits.get());
+                if (size > 0) {
+                    credits.addAndGet(-size);
+                    writerState.command.setChannelId(channelId);
+                    writerState.command.setCommandType(MuxDemuxCommand.CommandType.DATA);
+                    writerState.command.setData(size);
+                    writerState.reset(currentWriteBuffer, size, ChannelControlBlock.this);
+                }
+            } else if (ecode >= 0 && !ecodeSent) {
+                decrementLocalWriteEventCount();
+                writerState.command.setChannelId(channelId);
+                writerState.command.setCommandType(MuxDemuxCommand.CommandType.ERROR);
+                writerState.command.setData(ecode);
+                writerState.reset(null, 0, null);
+                ecodeSent = true;
+                localClose.set(true);
+            } else if (wi.eos && !wi.eosSent) {
+                decrementLocalWriteEventCount();
+                writerState.command.setChannelId(channelId);
+                writerState.command.setCommandType(MuxDemuxCommand.CommandType.CLOSE_CHANNEL);
+                writerState.command.setData(0);
+                writerState.reset(null, 0, null);
+                eosSent = true;
+                localClose.set(true);
+            }
+        }
+
+        void writeComplete() {
+            if (currentWriteBuffer.remaining() <= 0) {
+                currentWriteBuffer.clear();
+                eba.accept(currentWriteBuffer);
+                decrementLocalWriteEventCount();
+                currentWriteBuffer = null;
+            }
+        }
+
+        void incrementLocalWriteEventCount() {
+            ++channelWriteEventCount;
+            if (channelWriteEventCount == 1) {
+                cSet.markPendingWrite(channelId);
+            }
+        }
+
+        void decrementLocalWriteEventCount() {
+            --channelWriteEventCount;
+            if (channelWriteEventCount == 0) {
+                cSet.unmarkPendingWrite(channelId);
+            }
+        }
+    }
+
+    synchronized void write(MultiplexedConnection.WriterState writerState) throws NetException {
+        wi.write(writerState);
+    }
+
+    synchronized void writeComplete() {
+        wi.writeComplete();
+    }
+
+    synchronized int read(SocketChannel sc, int size) throws IOException, NetException {
+        return ri.read(sc, size);
+    }
+
+    void addReadCredits(int delta) {
+        ri.credits.addAndGet(delta);
+    }
+
+    int getAndResetReadCredits() {
+        return ri.credits.getAndSet(0);
+    }
+
+    void addWriteCredits(int delta) {
+        wi.credits.addAndGet(delta);
+    }
+
+    synchronized void reportRemoteEOS() {
+        ri.flush();
+        ri.fba.close();
+        remoteClose.set(true);
+    }
+
+    synchronized void reportLocalEOSAck() {
+        localCloseAck.set(true);
+    }
+
+    synchronized void reportRemoteError(int ecode) {
+        ri.flush();
+        ri.fba.error(ecode);
+        remoteClose.set(true);
+    }
+
+    boolean completelyClosed() {
+        return localCloseAck.get() && remoteClose.get();
+    }
+
+    @Override
+    public String toString() {
+        return "Channel:" + channelId + "[localClose: " + localClose + " localCloseAck: " + localCloseAck
+                + " remoteClose: " + remoteClose + " readCredits: " + ri.credits + " writeCredits: " + wi.credits + "]";
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java
new file mode 100644
index 0000000..9411d42
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java
@@ -0,0 +1,206 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+
+public class ChannelSet {
+    private static final Logger LOGGER = Logger.getLogger(ChannelSet.class.getName());
+
+    private static final int MAX_OPEN_CHANNELS = 1024;
+
+    private static final int INITIAL_SIZE = 16;
+
+    private final MultiplexedConnection mConn;
+
+    private ChannelControlBlock[] ccbArray;
+
+    private final BitSet allocationBitmap;
+
+    private final BitSet pendingChannelWriteBitmap;
+
+    private final BitSet pendingChannelCreditsBitmap;
+
+    private final BitSet pendingChannelSynBitmap;
+
+    private final BitSet pendingEOSAckBitmap;
+
+    private int openChannelCount;
+
+    private final IEventCounter pendingWriteEventsCounter;
+
+    ChannelSet(MultiplexedConnection mConn, IEventCounter pendingWriteEventsCounter) {
+        this.mConn = mConn;
+        ccbArray = new ChannelControlBlock[INITIAL_SIZE];
+        allocationBitmap = new BitSet();
+        pendingChannelWriteBitmap = new BitSet();
+        pendingChannelCreditsBitmap = new BitSet();
+        pendingChannelSynBitmap = new BitSet();
+        pendingEOSAckBitmap = new BitSet();
+        this.pendingWriteEventsCounter = pendingWriteEventsCounter;
+        openChannelCount = 0;
+    }
+
+    ChannelControlBlock allocateChannel() throws NetException {
+        synchronized (mConn) {
+            int idx = allocationBitmap.nextClearBit(0);
+            if (idx < 0 || idx == ccbArray.length) {
+                cleanupClosedChannels();
+                idx = allocationBitmap.nextClearBit(0);
+                if (idx < 0 || idx == ccbArray.length) {
+                    idx = ccbArray.length;
+                }
+            }
+            return createChannel(idx);
+        }
+    }
+
+    private void cleanupClosedChannels() {
+        for (int i = 0; i < ccbArray.length; ++i) {
+            ChannelControlBlock ccb = ccbArray[i];
+            if (ccb != null) {
+                if (ccb.completelyClosed()) {
+                    if (LOGGER.isLoggable(Level.FINE)) {
+                        LOGGER.fine("Cleaning free channel: " + ccb);
+                    }
+                    freeChannel(ccb);
+                }
+            }
+        }
+    }
+
+    ChannelControlBlock registerChannel(int channelId) throws NetException {
+        synchronized (mConn) {
+            return createChannel(channelId);
+        }
+    }
+
+    private void freeChannel(ChannelControlBlock channel) {
+        int idx = channel.getChannelId();
+        ccbArray[idx] = null;
+        allocationBitmap.clear(idx);
+        --openChannelCount;
+    }
+
+    ChannelControlBlock getCCB(int channelId) {
+        return ccbArray[channelId];
+    }
+
+    BitSet getPendingChannelWriteBitmap() {
+        return pendingChannelWriteBitmap;
+    }
+
+    BitSet getPendingChannelCreditsBitmap() {
+        return pendingChannelCreditsBitmap;
+    }
+
+    BitSet getPendingChannelSynBitmap() {
+        return pendingChannelSynBitmap;
+    }
+
+    BitSet getPendingEOSAckBitmap() {
+        return pendingEOSAckBitmap;
+    }
+
+    int getOpenChannelCount() {
+        return openChannelCount;
+    }
+
+    void initiateChannelSyn(int channelId) {
+        synchronized (mConn) {
+            assert !pendingChannelSynBitmap.get(channelId);
+            pendingChannelSynBitmap.set(channelId);
+            pendingWriteEventsCounter.increment();
+        }
+    }
+
+    void markPendingCredits(int channelId) {
+        synchronized (mConn) {
+            if (!pendingChannelCreditsBitmap.get(channelId)) {
+                pendingChannelCreditsBitmap.set(channelId);
+                pendingWriteEventsCounter.increment();
+            }
+        }
+    }
+
+    void unmarkPendingCredits(int channelId) {
+        synchronized (mConn) {
+            if (pendingChannelCreditsBitmap.get(channelId)) {
+                pendingChannelCreditsBitmap.clear(channelId);
+                pendingWriteEventsCounter.decrement();
+            }
+        }
+    }
+
+    void markPendingWrite(int channelId) {
+        synchronized (mConn) {
+            assert !pendingChannelWriteBitmap.get(channelId);
+            pendingChannelWriteBitmap.set(channelId);
+            pendingWriteEventsCounter.increment();
+        }
+    }
+
+    void unmarkPendingWrite(int channelId) {
+        synchronized (mConn) {
+            assert pendingChannelWriteBitmap.get(channelId);
+            pendingChannelWriteBitmap.clear(channelId);
+            pendingWriteEventsCounter.decrement();
+        }
+    }
+
+    void markEOSAck(int channelId) {
+        synchronized (mConn) {
+            assert !pendingEOSAckBitmap.get(channelId);
+            pendingEOSAckBitmap.set(channelId);
+            pendingWriteEventsCounter.increment();
+        }
+    }
+
+    private ChannelControlBlock createChannel(int idx) throws NetException {
+        if (idx >= ccbArray.length) {
+            expand(idx);
+        }
+        if (idx > MAX_OPEN_CHANNELS) {
+            throw new NetException("More than " + MAX_OPEN_CHANNELS + " opened concurrently");
+        }
+        if (ccbArray[idx] != null) {
+            assert ccbArray[idx].completelyClosed();
+            if (ccbArray[idx].completelyClosed()) {
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.fine("Cleaning free channel: " + ccbArray[idx]);
+                }
+                freeChannel(ccbArray[idx]);
+            }
+        }
+        assert idx < ccbArray.length;
+        assert !allocationBitmap.get(idx);
+        ChannelControlBlock channel = new ChannelControlBlock(this, idx);
+        ccbArray[idx] = channel;
+        allocationBitmap.set(idx);
+        ++openChannelCount;
+        return channel;
+    }
+
+    private void expand(int idx) {
+        while (idx >= ccbArray.length) {
+            ccbArray = Arrays.copyOf(ccbArray, ccbArray.length * 2);
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
similarity index 80%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
index c728b0b..0fc9b2a 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
@@ -12,10 +12,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
-
-    public R getResult();
+public interface IChannelOpenListener {
+    public void channelOpened(ChannelControlBlock channel);
 }
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
similarity index 64%
copy from hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
copy to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
index 2364d74..468a617 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
@@ -12,18 +12,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.api.util;
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
 
-import java.io.Serializable;
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
 
-public final class Pair<F, S> implements Serializable {
-    private static final long serialVersionUID = 1L;
+public interface IChannelReadInterface {
+    public void setFullBufferAcceptor(ICloseableBufferAcceptor fullBufferAcceptor);
 
-    public F first;
-    public S second;
-
-    public Pair(F first, S second) {
-        this.first = first;
-        this.second = second;
-    }
-}
+    public IBufferAcceptor getEmptyBufferAcceptor();
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
similarity index 64%
copy from hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
copy to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
index 2364d74..1e53d71 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
@@ -12,18 +12,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.api.util;
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
 
-import java.io.Serializable;
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
 
-public final class Pair<F, S> implements Serializable {
-    private static final long serialVersionUID = 1L;
+public interface IChannelWriteInterface {
+    public void setEmptyBufferAcceptor(IBufferAcceptor emptyBufferAcceptor);
 
-    public F first;
-    public S second;
-
-    public Pair(F first, S second) {
-        this.first = first;
-        this.second = second;
-    }
-}
+    public ICloseableBufferAcceptor getFullBufferAcceptor();
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IEventCounter.java
similarity index 81%
rename from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
rename to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IEventCounter.java
index c728b0b..148078c 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IEventCounter.java
@@ -12,10 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
+public interface IEventCounter {
+    public void increment();
 
-    public R getResult();
+    public void decrement();
 }
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java
new file mode 100644
index 0000000..238f2ee
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java
@@ -0,0 +1,350 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.util.BitSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+import edu.uci.ics.hyracks.net.protocols.tcp.ITCPConnectionEventListener;
+import edu.uci.ics.hyracks.net.protocols.tcp.TCPConnection;
+
+public class MultiplexedConnection implements ITCPConnectionEventListener {
+    private static final Logger LOGGER = Logger.getLogger(MultiplexedConnection.class.getName());
+
+    private static final int MAX_CHUNKS_READ_PER_CYCLE = 4;
+
+    private final MuxDemux muxDemux;
+
+    private final IEventCounter pendingWriteEventsCounter;
+
+    private final ChannelSet cSet;
+
+    private final ReaderState readerState;
+
+    private final WriterState writerState;
+
+    private TCPConnection tcpConnection;
+
+    private int lastChannelWritten;
+
+    public MultiplexedConnection(MuxDemux muxDemux) {
+        this.muxDemux = muxDemux;
+        pendingWriteEventsCounter = new IEventCounter() {
+            private int counter;
+
+            @Override
+            public synchronized void increment() {
+                ++counter;
+                if (counter == 1) {
+                    tcpConnection.enable(SelectionKey.OP_WRITE);
+                }
+            }
+
+            @Override
+            public synchronized void decrement() {
+                --counter;
+                if (counter == 0) {
+                    tcpConnection.disable(SelectionKey.OP_WRITE);
+                }
+                if (counter < 0) {
+                    throw new IllegalStateException();
+                }
+            }
+        };
+        cSet = new ChannelSet(this, pendingWriteEventsCounter);
+        readerState = new ReaderState();
+        writerState = new WriterState();
+        lastChannelWritten = -1;
+    }
+
+    synchronized void setTCPConnection(TCPConnection tcpConnection) {
+        this.tcpConnection = tcpConnection;
+        tcpConnection.enable(SelectionKey.OP_READ);
+        notifyAll();
+    }
+
+    synchronized void waitUntilConnected() throws InterruptedException {
+        while (tcpConnection == null) {
+            wait();
+        }
+    }
+
+    @Override
+    public void notifyIOReady(TCPConnection connection, boolean readable, boolean writable) throws IOException,
+            NetException {
+        if (readable) {
+            driveReaderStateMachine();
+        }
+        if (writable) {
+            driveWriterStateMachine();
+        }
+    }
+
+    public ChannelControlBlock openChannel() throws NetException, InterruptedException {
+        ChannelControlBlock channel = cSet.allocateChannel();
+        int channelId = channel.getChannelId();
+        cSet.initiateChannelSyn(channelId);
+        return channel;
+    }
+
+    class WriterState {
+        private final ByteBuffer writeBuffer;
+
+        final MuxDemuxCommand command;
+
+        private ByteBuffer pendingBuffer;
+
+        private int pendingWriteSize;
+
+        private ChannelControlBlock ccb;
+
+        public WriterState() {
+            writeBuffer = ByteBuffer.allocateDirect(MuxDemuxCommand.COMMAND_SIZE);
+            writeBuffer.flip();
+            command = new MuxDemuxCommand();
+            ccb = null;
+        }
+
+        boolean writePending() {
+            return writeBuffer.remaining() > 0 || (pendingBuffer != null && pendingWriteSize > 0);
+        }
+
+        void reset(ByteBuffer pendingBuffer, int pendingWriteSize, ChannelControlBlock ccb) {
+            writeBuffer.clear();
+            command.write(writeBuffer);
+            writeBuffer.flip();
+            this.pendingBuffer = pendingBuffer;
+            this.pendingWriteSize = pendingWriteSize;
+            this.ccb = ccb;
+        }
+
+        boolean performPendingWrite(SocketChannel sc) throws IOException {
+            int len = writeBuffer.remaining();
+            if (len > 0) {
+                int written = sc.write(writeBuffer);
+                muxDemux.getPerformanceCounters().addSignalingBytesWritten(written);
+                if (written < len) {
+                    return false;
+                }
+            }
+            if (pendingBuffer != null) {
+                if (pendingWriteSize > 0) {
+                    assert pendingWriteSize <= pendingBuffer.remaining();
+                    int oldLimit = pendingBuffer.limit();
+                    try {
+                        pendingBuffer.limit(pendingWriteSize + pendingBuffer.position());
+                        int written = sc.write(pendingBuffer);
+                        muxDemux.getPerformanceCounters().addPayloadBytesWritten(written);
+                        pendingWriteSize -= written;
+                    } finally {
+                        pendingBuffer.limit(oldLimit);
+                    }
+                }
+                if (pendingWriteSize > 0) {
+                    return false;
+                }
+                pendingBuffer = null;
+                pendingWriteSize = 0;
+            }
+            if (ccb != null) {
+                ccb.writeComplete();
+                ccb = null;
+            }
+            return true;
+        }
+    }
+
+    void driveWriterStateMachine() throws IOException, NetException {
+        SocketChannel sc = tcpConnection.getSocketChannel();
+        if (writerState.writePending()) {
+            if (!writerState.performPendingWrite(sc)) {
+                return;
+            }
+        }
+        int numCycles;
+
+        synchronized (MultiplexedConnection.this) {
+            numCycles = cSet.getOpenChannelCount();
+        }
+
+        for (int i = 0; i < numCycles; ++i) {
+            ChannelControlBlock writeCCB = null;
+            synchronized (MultiplexedConnection.this) {
+                BitSet pendingChannelSynBitmap = cSet.getPendingChannelSynBitmap();
+                for (int j = pendingChannelSynBitmap.nextSetBit(0); j >= 0; j = pendingChannelSynBitmap.nextSetBit(j)) {
+                    pendingChannelSynBitmap.clear(j);
+                    pendingWriteEventsCounter.decrement();
+                    writerState.command.setChannelId(j);
+                    writerState.command.setCommandType(MuxDemuxCommand.CommandType.OPEN_CHANNEL);
+                    writerState.command.setData(0);
+                    writerState.reset(null, 0, null);
+                    if (!writerState.performPendingWrite(sc)) {
+                        return;
+                    }
+                }
+                BitSet pendingChannelCreditsBitmap = cSet.getPendingChannelCreditsBitmap();
+                for (int j = pendingChannelCreditsBitmap.nextSetBit(0); j >= 0; j = pendingChannelCreditsBitmap
+                        .nextSetBit(j)) {
+                    pendingChannelCreditsBitmap.clear(j);
+                    pendingWriteEventsCounter.decrement();
+                    writerState.command.setChannelId(j);
+                    writerState.command.setCommandType(MuxDemuxCommand.CommandType.ADD_CREDITS);
+                    ChannelControlBlock ccb = cSet.getCCB(j);
+                    int credits = ccb.getAndResetReadCredits();
+                    writerState.command.setData(credits);
+                    writerState.reset(null, 0, null);
+                    if (!writerState.performPendingWrite(sc)) {
+                        return;
+                    }
+                }
+                BitSet pendingEOSAckBitmap = cSet.getPendingEOSAckBitmap();
+                for (int j = pendingEOSAckBitmap.nextSetBit(0); j >= 0; j = pendingEOSAckBitmap.nextSetBit(j)) {
+                    pendingEOSAckBitmap.clear(j);
+                    pendingWriteEventsCounter.decrement();
+                    writerState.command.setChannelId(j);
+                    writerState.command.setCommandType(MuxDemuxCommand.CommandType.CLOSE_CHANNEL_ACK);
+                    writerState.command.setData(0);
+                    writerState.reset(null, 0, null);
+                    if (!writerState.performPendingWrite(sc)) {
+                        return;
+                    }
+                }
+                BitSet pendingChannelWriteBitmap = cSet.getPendingChannelWriteBitmap();
+                lastChannelWritten = pendingChannelWriteBitmap.nextSetBit(lastChannelWritten + 1);
+                if (lastChannelWritten < 0) {
+                    lastChannelWritten = pendingChannelWriteBitmap.nextSetBit(0);
+                    if (lastChannelWritten < 0) {
+                        return;
+                    }
+                }
+                writeCCB = cSet.getCCB(lastChannelWritten);
+            }
+            writeCCB.write(writerState);
+            if (writerState.writePending()) {
+                if (!writerState.performPendingWrite(sc)) {
+                    return;
+                }
+            }
+        }
+    }
+
+    class ReaderState {
+        private final ByteBuffer readBuffer;
+
+        final MuxDemuxCommand command;
+
+        private int pendingReadSize;
+
+        private ChannelControlBlock ccb;
+
+        ReaderState() {
+            readBuffer = ByteBuffer.allocateDirect(MuxDemuxCommand.COMMAND_SIZE);
+            command = new MuxDemuxCommand();
+        }
+
+        void reset() {
+            readBuffer.clear();
+            pendingReadSize = 0;
+            ccb = null;
+        }
+
+        private ChannelControlBlock getCCBInCommand() {
+            synchronized (MultiplexedConnection.this) {
+                return cSet.getCCB(command.getChannelId());
+            }
+        }
+    }
+
+    void driveReaderStateMachine() throws IOException, NetException {
+        SocketChannel sc = tcpConnection.getSocketChannel();
+        int chunksRead = 0;
+        while (chunksRead < MAX_CHUNKS_READ_PER_CYCLE) {
+            if (readerState.readBuffer.remaining() > 0) {
+                int read = sc.read(readerState.readBuffer);
+                if (read < 0) {
+                    throw new NetException("Socket Closed");
+                }
+                muxDemux.getPerformanceCounters().addSignalingBytesRead(read);
+                if (readerState.readBuffer.remaining() > 0) {
+                    return;
+                }
+                readerState.readBuffer.flip();
+                readerState.command.read(readerState.readBuffer);
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.fine("Received command: " + readerState.command);
+                }
+                ChannelControlBlock ccb = null;
+                switch (readerState.command.getCommandType()) {
+                    case ADD_CREDITS: {
+                        ccb = readerState.getCCBInCommand();
+                        ccb.addWriteCredits(readerState.command.getData());
+                        break;
+                    }
+                    case CLOSE_CHANNEL: {
+                        ccb = readerState.getCCBInCommand();
+                        ccb.reportRemoteEOS();
+                        int channelId = ccb.getChannelId();
+                        cSet.markEOSAck(channelId);
+                        cSet.unmarkPendingCredits(channelId);
+                        break;
+                    }
+                    case CLOSE_CHANNEL_ACK: {
+                        ccb = readerState.getCCBInCommand();
+                        ccb.reportLocalEOSAck();
+                        break;
+                    }
+                    case DATA: {
+                        ccb = readerState.getCCBInCommand();
+                        readerState.pendingReadSize = readerState.command.getData();
+                        readerState.ccb = ccb;
+                        break;
+                    }
+                    case ERROR: {
+                        ccb = readerState.getCCBInCommand();
+                        ccb.reportRemoteError(readerState.command.getData());
+                        int channelId = ccb.getChannelId();
+                        cSet.markEOSAck(channelId);
+                        cSet.unmarkPendingCredits(channelId);
+                        break;
+                    }
+                    case OPEN_CHANNEL: {
+                        int channelId = readerState.command.getChannelId();
+                        ccb = cSet.registerChannel(channelId);
+                        muxDemux.getChannelOpenListener().channelOpened(ccb);
+                    }
+                }
+                if (LOGGER.isLoggable(Level.FINE)) {
+                    LOGGER.fine("Applied command: " + readerState.command + " on " + ccb);
+                }
+            }
+            if (readerState.pendingReadSize > 0) {
+                ++chunksRead;
+                int newPendingReadSize = readerState.ccb.read(sc, readerState.pendingReadSize);
+                muxDemux.getPerformanceCounters().addPayloadBytesRead(readerState.pendingReadSize - newPendingReadSize);
+                readerState.pendingReadSize = newPendingReadSize;
+                if (readerState.pendingReadSize > 0) {
+                    return;
+                }
+            }
+            readerState.reset();
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java
new file mode 100644
index 0000000..b041e2c
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.net.protocols.tcp.ITCPConnectionListener;
+import edu.uci.ics.hyracks.net.protocols.tcp.TCPConnection;
+import edu.uci.ics.hyracks.net.protocols.tcp.TCPEndpoint;
+
+public class MuxDemux {
+    private final InetSocketAddress localAddress;
+
+    private final IChannelOpenListener channelOpenListener;
+
+    private final Map<InetSocketAddress, MultiplexedConnection> connectionMap;
+
+    private final TCPEndpoint tcpEndpoint;
+
+    private final PerformanceCounters perfCounters;
+
+    public MuxDemux(InetSocketAddress localAddress, IChannelOpenListener listener, int nThreads) {
+        this.localAddress = localAddress;
+        this.channelOpenListener = listener;
+        connectionMap = new HashMap<InetSocketAddress, MultiplexedConnection>();
+        this.tcpEndpoint = new TCPEndpoint(new ITCPConnectionListener() {
+            @Override
+            public void connectionEstablished(TCPConnection connection) {
+                MultiplexedConnection mConn;
+                synchronized (MuxDemux.this) {
+                    mConn = connectionMap.get(connection.getRemoteAddress());
+                }
+                assert mConn != null;
+                mConn.setTCPConnection(connection);
+                connection.setEventListener(mConn);
+                connection.setAttachment(mConn);
+            }
+
+            @Override
+            public void acceptedConnection(TCPConnection connection) {
+                MultiplexedConnection mConn = new MultiplexedConnection(MuxDemux.this);
+                mConn.setTCPConnection(connection);
+                connection.setEventListener(mConn);
+                connection.setAttachment(mConn);
+            }
+        }, nThreads);
+        perfCounters = new PerformanceCounters();
+    }
+
+    public void start() throws IOException {
+        tcpEndpoint.start(localAddress);
+    }
+
+    public MultiplexedConnection connect(InetSocketAddress remoteAddress) throws InterruptedException {
+        MultiplexedConnection mConn = null;
+        synchronized (this) {
+            mConn = connectionMap.get(remoteAddress);
+            if (mConn == null) {
+                mConn = new MultiplexedConnection(this);
+                connectionMap.put(remoteAddress, mConn);
+                tcpEndpoint.initiateConnection(remoteAddress);
+            }
+        }
+        mConn.waitUntilConnected();
+        return mConn;
+    }
+
+    IChannelOpenListener getChannelOpenListener() {
+        return channelOpenListener;
+    }
+
+    public InetSocketAddress getLocalAddress() {
+        return tcpEndpoint.getLocalAddress();
+    }
+
+    public PerformanceCounters getPerformanceCounters() {
+        return perfCounters;
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java
new file mode 100644
index 0000000..9124aad
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java
@@ -0,0 +1,75 @@
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+
+class MuxDemuxCommand {
+    static final int MAX_CHANNEL_ID = 0x3ff;
+
+    static final int COMMAND_SIZE = 4;
+
+    static final int MAX_DATA_VALUE = 0x7ffff;
+
+    enum CommandType {
+        OPEN_CHANNEL,
+        CLOSE_CHANNEL,
+        CLOSE_CHANNEL_ACK,
+        ERROR,
+        ADD_CREDITS,
+        DATA,
+    }
+
+    private int channelId;
+
+    private CommandType type;
+
+    private int data;
+
+    public int getChannelId() {
+        return channelId;
+    }
+
+    public void setChannelId(int channelId) throws NetException {
+        if (channelId > MAX_CHANNEL_ID) {
+            throw new NetException("channelId " + channelId + " exceeds " + MAX_CHANNEL_ID);
+        }
+        this.channelId = channelId;
+    }
+
+    public CommandType getCommandType() {
+        return type;
+    }
+
+    public void setCommandType(CommandType type) {
+        this.type = type;
+    }
+
+    public int getData() {
+        return data;
+    }
+
+    public void setData(int data) throws NetException {
+        if (channelId > MAX_DATA_VALUE) {
+            throw new NetException("data " + data + " exceeds " + MAX_DATA_VALUE);
+        }
+        this.data = data;
+    }
+
+    public void write(ByteBuffer buffer) {
+        int cmd = (channelId << 22) | (type.ordinal() << 19) | (data & 0x7ffff);
+        buffer.putInt(cmd);
+    }
+
+    public void read(ByteBuffer buffer) {
+        int cmd = buffer.getInt();
+        channelId = (cmd >> 22) & 0x3ff;
+        type = CommandType.values()[(cmd >> 19) & 0x7];
+        data = cmd & 0x7ffff;
+    }
+
+    @Override
+    public String toString() {
+        return channelId + ":" + type + ":" + data;
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/PerformanceCounters.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/PerformanceCounters.java
new file mode 100644
index 0000000..a203f06
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/PerformanceCounters.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class PerformanceCounters {
+    private final AtomicLong payloadBytesRead;
+
+    private final AtomicLong payloadBytesWritten;
+
+    private final AtomicLong signalingBytesRead;
+
+    private final AtomicLong signalingBytesWritten;
+
+    public PerformanceCounters() {
+        payloadBytesRead = new AtomicLong();
+        payloadBytesWritten = new AtomicLong();
+        signalingBytesRead = new AtomicLong();
+        signalingBytesWritten = new AtomicLong();
+    }
+
+    public void addPayloadBytesRead(long delta) {
+        payloadBytesRead.addAndGet(delta);
+    }
+
+    public long getPayloadBytesRead() {
+        return payloadBytesRead.get();
+    }
+
+    public void addPayloadBytesWritten(long delta) {
+        payloadBytesWritten.addAndGet(delta);
+    }
+
+    public long getPayloadBytesWritten() {
+        return payloadBytesWritten.get();
+    }
+
+    public void addSignalingBytesRead(long delta) {
+        signalingBytesRead.addAndGet(delta);
+    }
+
+    public long getSignalingBytesRead() {
+        return signalingBytesRead.get();
+    }
+
+    public void addSignalingBytesWritten(long delta) {
+        signalingBytesWritten.addAndGet(delta);
+    }
+
+    public long getSignalingBytesWritten() {
+        return signalingBytesWritten.get();
+    }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
similarity index 66%
rename from hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
rename to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
index 2364d74..607bf31 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
@@ -12,18 +12,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.api.util;
+package edu.uci.ics.hyracks.net.protocols.tcp;
 
-import java.io.Serializable;
+import java.io.IOException;
 
-public final class Pair<F, S> implements Serializable {
-    private static final long serialVersionUID = 1L;
+import edu.uci.ics.hyracks.net.exceptions.NetException;
 
-    public F first;
-    public S second;
-
-    public Pair(F first, S second) {
-        this.first = first;
-        this.second = second;
-    }
-}
+public interface ITCPConnectionEventListener {
+    public void notifyIOReady(TCPConnection connection, boolean readable, boolean writable) throws IOException,
+            NetException;
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionListener.java
similarity index 74%
copy from hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
copy to hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionListener.java
index c728b0b..cdaabf4 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionListener.java
@@ -12,10 +12,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package edu.uci.ics.hyracks.control.cc.remote;
+package edu.uci.ics.hyracks.net.protocols.tcp;
 
-public interface Accumulator<T, R> {
-    public void accumulate(T o);
+public interface ITCPConnectionListener {
+    public void acceptedConnection(TCPConnection connection);
 
-    public R getResult();
+    public void connectionEstablished(TCPConnection connection);
 }
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPConnection.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPConnection.java
new file mode 100644
index 0000000..210508b
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPConnection.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.tcp;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.SocketChannel;
+
+public class TCPConnection {
+    private final TCPEndpoint endpoint;
+
+    private final SocketChannel channel;
+
+    private final SelectionKey key;
+
+    private final Selector selector;
+
+    private ITCPConnectionEventListener eventListener;
+
+    private Object attachment;
+
+    public TCPConnection(TCPEndpoint endpoint, SocketChannel channel, SelectionKey key, Selector selector) {
+        this.endpoint = endpoint;
+        this.channel = channel;
+        this.key = key;
+        this.selector = selector;
+    }
+
+    public TCPEndpoint getEndpoint() {
+        return endpoint;
+    }
+
+    public SocketChannel getSocketChannel() {
+        return channel;
+    }
+
+    public InetSocketAddress getLocalAddress() {
+        return (InetSocketAddress) channel.socket().getLocalSocketAddress();
+    }
+
+    public InetSocketAddress getRemoteAddress() {
+        return (InetSocketAddress) channel.socket().getRemoteSocketAddress();
+    }
+
+    public void enable(int ops) {
+        key.interestOps(key.interestOps() | ops);
+        selector.wakeup();
+    }
+
+    public void disable(int ops) {
+        key.interestOps(key.interestOps() & ~(ops));
+        selector.wakeup();
+    }
+
+    public ITCPConnectionEventListener getEventListener() {
+        return eventListener;
+    }
+
+    public void setEventListener(ITCPConnectionEventListener eventListener) {
+        this.eventListener = eventListener;
+    }
+
+    public Object getAttachment() {
+        return attachment;
+    }
+
+    public void setAttachment(Object attachment) {
+        this.attachment = attachment;
+    }
+
+    public void close() {
+        key.cancel();
+        try {
+            channel.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java
new file mode 100644
index 0000000..37641cb
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.tcp;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class TCPEndpoint {
+    private final ITCPConnectionListener connectionListener;
+
+    private final int nThreads;
+
+    private ServerSocketChannel serverSocketChannel;
+
+    private InetSocketAddress localAddress;
+
+    private IOThread[] ioThreads;
+
+    private int nextThread;
+
+    public TCPEndpoint(ITCPConnectionListener connectionListener, int nThreads) {
+        this.connectionListener = connectionListener;
+        this.nThreads = nThreads;
+    }
+
+    public void start(InetSocketAddress localAddress) throws IOException {
+        serverSocketChannel = ServerSocketChannel.open();
+        ServerSocket serverSocket = serverSocketChannel.socket();
+        serverSocket.bind(localAddress);
+        this.localAddress = (InetSocketAddress) serverSocket.getLocalSocketAddress();
+        ioThreads = new IOThread[nThreads];
+        for (int i = 0; i < ioThreads.length; ++i) {
+            ioThreads[i] = new IOThread();
+        }
+        ioThreads[0].registerServerSocket(serverSocketChannel);
+        for (int i = 0; i < ioThreads.length; ++i) {
+            ioThreads[i].start();
+        }
+    }
+
+    private synchronized int getNextThread() {
+        int result = nextThread;
+        nextThread = (nextThread + 1) % nThreads;
+        return result;
+    }
+
+    public void initiateConnection(InetSocketAddress remoteAddress) {
+        int targetThread = getNextThread();
+        ioThreads[targetThread].initiateConnection(remoteAddress);
+    }
+
+    private void addIncomingConnection(SocketChannel channel) {
+        int targetThread = getNextThread();
+        ioThreads[targetThread].addIncomingConnection(channel);
+    }
+
+    public InetSocketAddress getLocalAddress() {
+        return localAddress;
+    }
+
+    private class IOThread extends Thread {
+        private final List<InetSocketAddress>[] pendingConnections;
+
+        private final List<SocketChannel>[] incomingConnections;
+
+        private int writerIndex;
+
+        private int readerIndex;
+
+        private Selector selector;
+
+        public IOThread() throws IOException {
+            super("TCPEndpoint IO Thread");
+            setPriority(MAX_PRIORITY);
+            this.pendingConnections = new List[] { new ArrayList<InetSocketAddress>(),
+                    new ArrayList<InetSocketAddress>() };
+            this.incomingConnections = new List[] { new ArrayList<SocketChannel>(), new ArrayList<SocketChannel>() };
+            writerIndex = 0;
+            readerIndex = 1;
+            selector = Selector.open();
+        }
+
+        @Override
+        public void run() {
+            while (true) {
+                try {
+                    int n = selector.select();
+                    swapReadersAndWriters();
+                    if (!pendingConnections[readerIndex].isEmpty()) {
+                        for (InetSocketAddress address : pendingConnections[readerIndex]) {
+                            SocketChannel channel = SocketChannel.open();
+                            channel.configureBlocking(false);
+                            if (!channel.connect(address)) {
+                                channel.register(selector, SelectionKey.OP_CONNECT);
+                            } else {
+                                SelectionKey key = channel.register(selector, 0);
+                                createConnection(key, channel);
+                            }
+                        }
+                        pendingConnections[readerIndex].clear();
+                    }
+                    if (!incomingConnections[readerIndex].isEmpty()) {
+                        for (SocketChannel channel : incomingConnections[readerIndex]) {
+                            channel.configureBlocking(false);
+                            SelectionKey sKey = channel.register(selector, 0);
+                            TCPConnection connection = new TCPConnection(TCPEndpoint.this, channel, sKey, selector);
+                            sKey.attach(connection);
+                            synchronized (connectionListener) {
+                                connectionListener.acceptedConnection(connection);
+                            }
+                        }
+                        incomingConnections[readerIndex].clear();
+                    }
+                    if (n > 0) {
+                        Iterator<SelectionKey> i = selector.selectedKeys().iterator();
+                        while (i.hasNext()) {
+                            SelectionKey key = i.next();
+                            i.remove();
+                            SelectableChannel sc = key.channel();
+                            boolean readable = key.isReadable();
+                            boolean writable = key.isWritable();
+
+                            if (readable || writable) {
+                                TCPConnection connection = (TCPConnection) key.attachment();
+                                connection.getEventListener().notifyIOReady(connection, readable, writable);
+                            }
+                            if (key.isAcceptable()) {
+                                assert sc == serverSocketChannel;
+                                SocketChannel channel = serverSocketChannel.accept();
+                                addIncomingConnection(channel);
+                            } else if (key.isConnectable()) {
+                                SocketChannel channel = (SocketChannel) sc;
+                                if (channel.finishConnect()) {
+                                    createConnection(key, channel);
+                                }
+                            }
+                        }
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+
+        private void createConnection(SelectionKey key, SocketChannel channel) {
+            TCPConnection connection = new TCPConnection(TCPEndpoint.this, channel, key, selector);
+            key.attach(connection);
+            key.interestOps(0);
+            connectionListener.connectionEstablished(connection);
+        }
+
+        synchronized void initiateConnection(InetSocketAddress remoteAddress) {
+            pendingConnections[writerIndex].add(remoteAddress);
+            selector.wakeup();
+        }
+
+        synchronized void addIncomingConnection(SocketChannel channel) {
+            incomingConnections[writerIndex].add(channel);
+            selector.wakeup();
+        }
+
+        void registerServerSocket(ServerSocketChannel serverSocketChannel) throws IOException {
+            serverSocketChannel.configureBlocking(false);
+            serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT);
+        }
+
+        private synchronized void swapReadersAndWriters() {
+            int temp = readerIndex;
+            readerIndex = writerIndex;
+            writerIndex = temp;
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java b/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java
new file mode 100644
index 0000000..31bc2df
--- /dev/null
+++ b/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java
@@ -0,0 +1,213 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.tests;
+
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.IChannelOpenListener;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MultiplexedConnection;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MuxDemux;
+
+public class NetTest {
+    @Test
+    public void test() throws Exception {
+        AtomicBoolean failFlag = new AtomicBoolean();
+
+        MuxDemux md1 = createMuxDemux("md1", failFlag);
+        md1.start();
+        MuxDemux md2 = createMuxDemux("md2", failFlag);
+        md2.start();
+        InetSocketAddress md2Address = md2.getLocalAddress();
+
+        MultiplexedConnection md1md2 = md1.connect(md2Address);
+
+        Thread t1 = createThread(md1md2, 1);
+        Thread t2 = createThread(md1md2, -1);
+        t1.start();
+        t2.start();
+
+        t1.join();
+        t2.join();
+
+        Assert.assertFalse("Failure flag was set to true", failFlag.get());
+    }
+
+    private Thread createThread(final MultiplexedConnection md1md2, final int factor) {
+        return new Thread() {
+            @Override
+            public void run() {
+                try {
+                    ChannelControlBlock md1md2c1 = md1md2.openChannel();
+
+                    final Semaphore sem = new Semaphore(1);
+                    sem.acquire();
+                    md1md2c1.getWriteInterface().setEmptyBufferAcceptor(new IBufferAcceptor() {
+                        @Override
+                        public void accept(ByteBuffer buffer) {
+                        }
+                    });
+
+                    md1md2c1.getReadInterface().setFullBufferAcceptor(new ICloseableBufferAcceptor() {
+                        @Override
+                        public void accept(ByteBuffer buffer) {
+                        }
+
+                        @Override
+                        public void error(int ecode) {
+                        }
+
+                        @Override
+                        public void close() {
+                            sem.release();
+                        }
+                    });
+
+                    ICloseableBufferAcceptor fba = md1md2c1.getWriteInterface().getFullBufferAcceptor();
+                    for (int i = 0; i < 10000; ++i) {
+                        ByteBuffer buffer = ByteBuffer.allocate(1024);
+                        for (int j = 0; j < 256; ++j) {
+                            buffer.putInt(factor * (i + j));
+                        }
+                        buffer.flip();
+                        fba.accept(buffer);
+                    }
+                    fba.close();
+                    sem.acquire();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+
+    }
+
+    private MuxDemux createMuxDemux(final String label, final AtomicBoolean failFlag) {
+        IChannelOpenListener md1OpenListener = new IChannelOpenListener() {
+            @Override
+            public void channelOpened(final ChannelControlBlock channel) {
+                final ChannelIO cio = new ChannelIO(label, channel);
+                channel.getReadInterface().setFullBufferAcceptor(cio.rifba);
+                channel.getWriteInterface().setEmptyBufferAcceptor(cio.wieba);
+
+                final IBufferAcceptor rieba = channel.getReadInterface().getEmptyBufferAcceptor();
+                for (int i = 0; i < 50; ++i) {
+                    rieba.accept(ByteBuffer.allocate(1024));
+                }
+                new Thread() {
+                    private int prevTotal = 0;
+
+                    @Override
+                    public void run() {
+                        while (true) {
+                            ByteBuffer fbuf = null;
+                            synchronized (channel) {
+                                while (!cio.eos && cio.ecode == 0 && cio.rifq.isEmpty()) {
+                                    try {
+                                        channel.wait();
+                                    } catch (InterruptedException e) {
+                                        e.printStackTrace();
+                                    }
+                                }
+                                if (!cio.rifq.isEmpty()) {
+                                    fbuf = cio.rifq.poll();
+                                } else if (cio.ecode != 0) {
+                                    throw new RuntimeException("Error: " + cio.ecode);
+                                } else if (cio.eos) {
+                                    channel.getWriteInterface().getFullBufferAcceptor().close();
+                                    return;
+                                }
+                            }
+                            int counter = 0;
+                            while (fbuf.remaining() > 0) {
+                                counter += fbuf.getInt();
+                            }
+                            if (prevTotal != 0) {
+                                if (Math.abs(counter - prevTotal) != 256) {
+                                    failFlag.set(true);
+                                }
+                            }
+                            prevTotal = counter;
+                            fbuf.compact();
+                            rieba.accept(fbuf);
+                        }
+                    }
+                }.start();
+            }
+        };
+        return new MuxDemux(new InetSocketAddress("127.0.0.1", 0), md1OpenListener, 1);
+    }
+
+    private class ChannelIO {
+        private ChannelControlBlock channel;
+
+        private Queue<ByteBuffer> rifq;
+
+        private Queue<ByteBuffer> wieq;
+
+        private boolean eos;
+
+        private int ecode;
+
+        private ICloseableBufferAcceptor rifba;
+
+        private IBufferAcceptor wieba;
+
+        public ChannelIO(final String label, ChannelControlBlock channel) {
+            this.channel = channel;
+            this.rifq = new LinkedList<ByteBuffer>();
+            this.wieq = new LinkedList<ByteBuffer>();
+
+            rifba = new ICloseableBufferAcceptor() {
+                @Override
+                public void accept(ByteBuffer buffer) {
+                    rifq.add(buffer);
+                    ChannelIO.this.channel.notifyAll();
+                }
+
+                @Override
+                public void error(int ecode) {
+                    ChannelIO.this.ecode = ecode;
+                    ChannelIO.this.channel.notifyAll();
+                }
+
+                @Override
+                public void close() {
+                    eos = true;
+                    ChannelIO.this.channel.notifyAll();
+                }
+            };
+
+            wieba = new IBufferAcceptor() {
+                @Override
+                public void accept(ByteBuffer buffer) {
+                    wieq.add(buffer);
+                    ChannelIO.this.channel.notifyAll();
+                }
+            };
+        }
+    }
+}
\ No newline at end of file
diff --git a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
index 8ed7bb7..4faa619 100644
--- a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
+++ b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
@@ -27,8 +27,11 @@
         @Option(name = "-n", required = false, usage = "Number of node controllers (default: 2)")
         public int n = 2;
 
-        @Option(name = "-cc-port", required = false, usage = "CC Port (default: 1099)")
-        public int ccPort = 1099;
+        @Option(name = "-cc-client-net-port", required = false, usage = "CC Port (default: 1098)")
+        public int ccClientNetPort = 1098;
+
+        @Option(name = "-cc-cluster-net-port", required = false, usage = "CC Port (default: 1099)")
+        public int ccClusterNetPort = 1099;
 
         @Option(name = "-cc-http-port", required = false, usage = "CC Port (default: 19001)")
         public int ccHttpPort = 19001;
@@ -46,7 +49,10 @@
         }
 
         CCConfig ccConfig = new CCConfig();
-        ccConfig.port = options.ccPort;
+        ccConfig.clusterNetIpAddress = "127.0.0.1";
+        ccConfig.clusterNetPort = options.ccClusterNetPort;
+        ccConfig.clientNetIpAddress = "127.0.0.1";
+        ccConfig.clientNetPort = options.ccClientNetPort;
         ccConfig.httpPort = options.ccHttpPort;
         HyracksCCProcess ccp = new HyracksCCProcess(ccConfig);
         ccp.start();
@@ -56,7 +62,9 @@
         HyracksNCProcess ncps[] = new HyracksNCProcess[options.n];
         for (int i = 0; i < options.n; ++i) {
             NCConfig ncConfig = new NCConfig();
-            ncConfig.ccHost = "localhost";
+            ncConfig.ccHost = "127.0.0.1";
+            ncConfig.ccPort = options.ccClusterNetPort;
+            ncConfig.clusterNetIPAddress = "127.0.0.1";
             ncConfig.nodeId = "nc" + i;
             ncConfig.dataIPAddress = "127.0.0.1";
             ncps[i] = new HyracksNCProcess(ncConfig);
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
index 0af27d6..d00bea6 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
@@ -59,7 +59,7 @@
     public void open() throws HyracksDataException {
         accessor = new FrameTupleAccessor(ctx.getFrameSize(), inputRecDesc);
         writeBuffer = ctx.allocateFrame();
-        builder = new ArrayTupleBuilder(outputRecDesc.getFields().length);
+        builder = new ArrayTupleBuilder(outputRecDesc.getFieldCount());
         builderDos = builder.getDataOutput();
         appender = new FrameTupleAppender(ctx.getFrameSize());
         appender.reset(writeBuffer, true);
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
index 47e3246..b24d416 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
@@ -72,7 +72,7 @@
 
     protected RecordDescriptor queryTokenRecDesc = new RecordDescriptor(
             new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
-    protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFields().length);
+    protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFieldCount());
     protected DataOutput queryTokenDos = queryTokenBuilder.getDataOutput();
     protected FrameTupleAppender queryTokenAppender;
     protected ByteBuffer queryTokenFrame;
diff --git a/pom.xml b/pom.xml
index bdf6031..3b713a5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
             <forkMode>pertest</forkMode>
-            <argLine>-enableassertions -Djava.util.logging.config.file=${user.home}/logging.properties</argLine>
+            <argLine>-enableassertions -Djava.util.logging.config.file=${user.home}/logging.properties -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=n</argLine>
         </configuration>
       </plugin>
     </plugins>
@@ -79,12 +79,14 @@
   </pluginRepositories>
 
   <modules>
+    <module>hyracks-ipc</module>
     <module>hyracks-api</module>
     <module>hyracks-dataflow-common</module>
     <module>hyracks-dataflow-std</module>
     <module>hyracks-dataflow-hadoop</module>
     <module>hyracks-control-common</module>
     <module>hyracks-control-cc</module>
+    <module>hyracks-net</module>
     <module>hyracks-control-nc</module>
     <module>hyracks-data</module>
     <module>hyracks-cli</module>