diff --git a/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/graph/Vertex.java b/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/graph/Vertex.java
index a93d744..c2a82d8 100644
--- a/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/graph/Vertex.java
+++ b/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/graph/Vertex.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,6 +25,10 @@
 import java.util.Map;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.WritableUtils;
@@ -40,7 +44,7 @@
 /**
  * User applications should all inherit {@link Vertex}, and implement their own
  * *compute* method.
- * 
+ *
  * @param <I>
  *            Vertex identifier type
  * @param <V>
@@ -68,11 +72,17 @@
     /** List of incoming messages from the previous superstep */
     private final List<M> msgList = new ArrayList<M>();
     /** a delegate for hyracks stuff */
-    private VertexDelegate<I, V, E, M> delegate = new VertexDelegate<I, V, E, M>(this);
+    private final VertexDelegate<I, V, E, M> delegate = new VertexDelegate<I, V, E, M>(this);
     /** this vertex is updated or not */
     private boolean updated = false;
     /** has outgoing messages */
     private boolean hasMessage = false;
+    /** whether the vertex has spilled to HDFS */
+    private boolean spilled = false;
+    /** the spilled path */
+    private String pathStr;
+    /** file system handle */
+    private FileSystem dfs;
     /** created new vertex */
     private boolean createdNewLiveVertex = false;
     /** terminate the partition */
@@ -81,9 +91,9 @@
     /**
      * use object pool for re-using objects
      */
-    private List<Edge<I, E>> edgePool = new ArrayList<Edge<I, E>>();
-    private List<M> msgPool = new ArrayList<M>();
-    private List<V> valuePool = new ArrayList<V>();
+    private final List<Edge<I, E>> edgePool = new ArrayList<Edge<I, E>>();
+    private final List<M> msgPool = new ArrayList<M>();
+    private final List<V> valuePool = new ArrayList<V>();
     private int usedEdge = 0;
     private int usedMessage = 0;
     private int usedValue = 0;
@@ -101,7 +111,7 @@
      * 3. In each partition, the vertex Java object is reused
      * for all the vertice to be processed in the same partition. (The model
      * is the same as the key-value objects in hadoop map tasks.)
-     * 
+     *
      * @param msgIterator
      *            an iterator of incoming messages
      */
@@ -109,7 +119,7 @@
 
     /**
      * Add an edge for the vertex.
-     * 
+     *
      * @param targetVertexId
      * @param edgeValue
      * @return successful or not
@@ -125,7 +135,7 @@
 
     /**
      * Initialize a new vertex
-     * 
+     *
      * @param vertexId
      * @param vertexValue
      * @param edges
@@ -157,11 +167,12 @@
         usedMessage = 0;
         usedValue = 0;
         updated = false;
+        spilled = false;
     }
 
     /**
      * Set the vertex id
-     * 
+     *
      * @param vertexId
      */
     public final void setVertexId(I vertexId) {
@@ -171,7 +182,7 @@
 
     /**
      * Get the vertex id
-     * 
+     *
      * @return vertex id
      */
     public final I getVertexId() {
@@ -180,7 +191,7 @@
 
     /**
      * Get the vertex value
-     * 
+     *
      * @return the vertex value
      */
     public final V getVertexValue() {
@@ -189,7 +200,7 @@
 
     /**
      * Set the vertex value
-     * 
+     *
      * @param vertexValue
      */
     public final void setVertexValue(V vertexValue) {
@@ -199,7 +210,7 @@
 
     /***
      * Send a message to a specific vertex
-     * 
+     *
      * @param id
      *            the receiver vertex id
      * @param msg
@@ -215,7 +226,7 @@
 
     /**
      * Send a message to all direct outgoing neighbors
-     * 
+     *
      * @param msg
      *            the message
      */
@@ -241,7 +252,7 @@
     /**
      * Vote to halt. Once all vertex vote to halt and no more messages, a
      * Pregelix job will terminate.
-     * 
+     *
      * @param update
      *            whether or not to save the vertex value
      */
@@ -261,7 +272,7 @@
 
     /**
      * Activate a halted vertex such that it is alive again.
-     * 
+     *
      * @param update
      *            whether or not to save the vertex value
      */
@@ -283,6 +294,35 @@
         if (vertexId == null) {
             vertexId = BspUtils.<I> createVertexIndex(getContext().getConfiguration());
         }
+
+        //read the boolean state
+        byte booleanState = in.readByte();
+        halt = (booleanState & 1) == 1 ? true : false;
+        spilled = (booleanState & 2) == 2 ? true : false;
+
+        if (!spilled) {
+            //read the vertex data
+            readVertexData(in);
+        } else {
+            //read file hosted vertex
+            if (dfs == null) {
+                dfs = FileSystem.get(getContext().getConfiguration());
+            }
+            pathStr = in.readUTF();
+            Path path = new Path(pathStr);
+            FSDataInputStream dataInput = dfs.open(path);
+            readVertexData(dataInput);
+            dataInput.close();
+        }
+    }
+
+    /**
+     * Read B-tree stored vertex
+     *
+     * @param in
+     * @throws IOException
+     */
+    private void readVertexData(DataInput in) throws IOException {
         vertexId.readFields(in);
         delegate.setVertexId(vertexId);
         boolean hasVertexValue = in.readBoolean();
@@ -307,7 +347,6 @@
             msg.readFields(in);
             msgList.add(msg);
         }
-        halt = in.readBoolean();
         updated = false;
         hasMessage = false;
         createdNewLiveVertex = false;
@@ -315,6 +354,38 @@
 
     @Override
     public void write(DataOutput out) throws IOException {
+        //write boolean states
+        int haltBit = halt ? 1 : 0;
+        int spilledBit = spilled ? 2 : 0;
+        byte booleanStates = (byte) (haltBit | spilledBit);
+        out.writeByte(booleanStates);
+        if (!spilled) {
+            // write B-tree stored vertex data
+            writeVertexData(out);
+        } else {
+            //write file-hosted vertex
+            if (dfs == null) {
+                dfs = FileSystem.get(getContext().getConfiguration());
+            }
+            Path path = new Path(pathStr);
+            if (dfs.exists(path)) {
+                dfs.delete(path, true);
+            }
+            FSDataOutputStream dataOutput = dfs.create(path, true);
+            writeVertexData(dataOutput);
+            dataOutput.flush();
+            dataOutput.close();
+            // write the btree content
+            pathStr = path.toUri().toString();
+            out.writeUTF(pathStr);
+        }
+    }
+
+    /**
+     * @param out
+     * @throws IOException
+     */
+    private void writeVertexData(DataOutput out) throws IOException {
         vertexId.write(out);
         out.writeBoolean(vertexValue != null);
         if (vertexValue != null) {
@@ -328,12 +399,11 @@
         for (M msg : msgList) {
             msg.write(out);
         }
-        out.writeBoolean(halt);
     }
 
     /**
      * Get the list of incoming messages
-     * 
+     *
      * @return the list of messages
      */
     public List<M> getMsgList() {
@@ -342,7 +412,7 @@
 
     /**
      * Get outgoing edge list
-     * 
+     *
      * @return a list of outgoing edges
      */
     public List<Edge<I, E>> getEdges() {
@@ -350,8 +420,12 @@
     }
 
     @Override
-    @SuppressWarnings("unchecked")
     public String toString() {
+        return getStringRepresetation();
+    }
+
+    @SuppressWarnings("unchecked")
+    public String getStringRepresetation() {
         Collections.sort(destEdgeList);
         StringBuffer edgeBuffer = new StringBuffer();
         edgeBuffer.append("(");
@@ -364,7 +438,7 @@
 
     /**
      * Get the number of outgoing edges
-     * 
+     *
      * @return the number of outging edges
      */
     public int getNumOutEdges() {
@@ -373,7 +447,7 @@
 
     /**
      * Pregelix internal use only
-     * 
+     *
      * @param writers
      */
     public void setOutputWriters(List<IFrameWriter> writers) {
@@ -382,7 +456,7 @@
 
     /**
      * Pregelix internal use only
-     * 
+     *
      * @param writers
      */
     public void setOutputAppenders(List<FrameTupleAppender> appenders) {
@@ -391,7 +465,7 @@
 
     /**
      * Pregelix internal use only
-     * 
+     *
      * @param writers
      */
     public void setOutputTupleBuilders(List<ArrayTupleBuilder> tbs) {
@@ -400,7 +474,7 @@
 
     /**
      * Pregelix internal use only
-     * 
+     *
      * @param writers
      */
     public void finishCompute() throws IOException {
@@ -438,6 +512,24 @@
     }
 
     /**
+     * set the HDFS spilled bit and path
+     *
+     * @param spilledHDFS
+     */
+    public void setSpilled(String path) {
+        this.spilled = true;
+        this.pathStr = path;
+    }
+
+    /**
+     * set the spilled bit to false and the path to null
+     */
+    public void setUnSpilled() {
+        this.spilled = false;
+        this.pathStr = null;
+    }
+
+    /**
      * Allocate a new edge from the edge pool
      */
     private Edge<I, E> allocateEdge() {
@@ -471,7 +563,7 @@
 
     /**
      * Add an outgoing edge into the vertex
-     * 
+     *
      * @param edge
      *            the edge to be added
      * @return true if the edge list changed as a result of this call
@@ -484,7 +576,7 @@
 
     /**
      * remove an outgoing edge in the graph
-     * 
+     *
      * @param edge
      *            the edge to be removed
      * @return true if the edge is in the edge list of the vertex
@@ -496,7 +588,7 @@
 
     /**
      * Add a new vertex into the graph
-     * 
+     *
      * @param vertexId
      *            the vertex id
      * @param vertex
@@ -509,7 +601,7 @@
 
     /**
      * Delete a vertex from id
-     * 
+     *
      * @param vertexId
      *            the vertex id
      */
@@ -519,7 +611,7 @@
 
     /**
      * Allocate a vertex value from the object pool
-     * 
+     *
      * @return a vertex value instance
      */
     private V allocateValue() {
@@ -537,7 +629,7 @@
 
     /**
      * Get the current global superstep number
-     * 
+     *
      * @return the current superstep number
      */
     public final long getSuperstep() {
@@ -546,7 +638,7 @@
 
     /**
      * Get the number of vertexes in the graph
-     * 
+     *
      * @return the number of vertexes in the graph
      */
     public final long getNumVertices() {
@@ -555,7 +647,7 @@
 
     /**
      * Get the number of edges from this graph
-     * 
+     *
      * @return the number of edges in the graph
      */
     public final long getNumEdges() {
@@ -593,7 +685,7 @@
     public void configure(Configuration conf) {
 
     }
-    
+
     /**
      * called *once* per partition at the end of each iteration,
      * before calls to compute() or close()
@@ -635,7 +727,7 @@
     /**
      * Terminate the Pregelix job.
      * This will take effect only when the current iteration completed.
-     * 
+     *
      * @throws Exception
      */
     protected void terminateJob() throws Exception {
@@ -652,7 +744,7 @@
 
     /**
      * Set the vertex context
-     * 
+     *
      * @param ctx
      */
     public void setVertexContext(VertexContext ctx) {
@@ -661,7 +753,7 @@
 
     /***
      * Get the vertex context
-     * 
+     *
      * @return the vertex context
      */
     public VertexContext getVertexContext() {
diff --git a/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/BspUtils.java b/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/BspUtils.java
index 1198a3e..92a6320 100644
--- a/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/BspUtils.java
+++ b/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/BspUtils.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,6 +15,7 @@
 
 package edu.uci.ics.pregelix.api.util;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -56,7 +57,7 @@
 
     /**
      * Get the user's subclassed {@link VertexInputFormat}.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex input format class
@@ -70,7 +71,7 @@
 
     /**
      * Create a user vertex input format class
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex input format class
@@ -89,7 +90,7 @@
 
     /**
      * Get the user's subclassed {@link VertexOutputFormat}.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex output format class
@@ -103,7 +104,7 @@
 
     /**
      * Create a user vertex output format class
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex output format class
@@ -121,7 +122,7 @@
 
     /**
      * Get the user's subclassed {@link MessageCombiner}.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex combiner class
@@ -135,7 +136,7 @@
 
     /**
      * Get the user's subclassed {@link GlobalAggregator}.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex combiner class
@@ -170,7 +171,7 @@
 
     /**
      * Create a user vertex combiner class
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex combiner class
@@ -188,7 +189,7 @@
 
     /**
      * Create a user-defined normalized key computer class
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user-defined normalized key computer
@@ -204,7 +205,7 @@
 
     /**
      * Create a global aggregator object
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex combiner class
@@ -226,7 +227,7 @@
 
     /**
      * Get global aggregator class names
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return An array of Global aggregator names
@@ -245,7 +246,7 @@
 
     /**
      * Get global aggregator class names
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return An array of Global aggregator names
@@ -264,7 +265,7 @@
 
     /**
      * Get the user's subclassed Vertex.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex class
@@ -277,7 +278,7 @@
 
     /**
      * Create a user vertex
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex
@@ -297,7 +298,7 @@
 
     /**
      * Get the user's subclassed vertex index class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex index class
@@ -309,7 +310,7 @@
 
     /**
      * Create a user vertex index
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex index
@@ -328,7 +329,7 @@
 
     /**
      * Get the user's subclassed vertex value class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex value class
@@ -340,7 +341,7 @@
 
     /**
      * Create a user vertex value
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex value
@@ -358,7 +359,7 @@
 
     /**
      * Get the user's subclassed edge value class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex edge value class
@@ -370,7 +371,7 @@
 
     /**
      * Create a user edge value
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user edge value
@@ -388,7 +389,7 @@
 
     /**
      * Get the user's subclassed vertex message value class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's vertex message value class
@@ -400,7 +401,7 @@
 
     /**
      * Get the user's subclassed global aggregator's partial aggregate value class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's global aggregate value class
@@ -412,7 +413,7 @@
 
     /**
      * Get the user's subclassed combiner's partial combine value class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's global aggregate value class
@@ -424,7 +425,7 @@
 
     /**
      * Get the user's subclassed normalized key computer class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's normalized key computer class
@@ -437,7 +438,7 @@
 
     /**
      * Get the user's subclassed normalized key computer class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return User's global aggregate value class
@@ -449,7 +450,7 @@
 
     /**
      * Create a user vertex message value
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user vertex message value
@@ -467,7 +468,7 @@
 
     /**
      * Create a user partial aggregate value
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user aggregate value
@@ -485,7 +486,7 @@
 
     /**
      * Create the list of user partial aggregate values
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user partial aggregate values
@@ -508,7 +509,7 @@
 
     /**
      * Create a user partial combine value
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user aggregate value
@@ -532,7 +533,7 @@
 
     /**
      * Create a user aggregate value
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user aggregate value
@@ -550,7 +551,7 @@
 
     /**
      * Create the list of user aggregate values
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user aggregate value
@@ -573,7 +574,7 @@
 
     /**
      * Create a user aggregate value
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user aggregate value
@@ -592,7 +593,7 @@
 
     /**
      * Create a checkpoint hook
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user aggregate value
@@ -610,7 +611,7 @@
 
     /**
      * Create a hook that indicates an iteration is complete
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return Instantiated user aggregate value
@@ -628,7 +629,7 @@
 
     /**
      * Get the user's subclassed vertex partitioner class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return The user defined vertex partitioner class
@@ -640,7 +641,7 @@
 
     /**
      * Get the user's subclassed checkpoint hook class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return The user defined vertex checkpoint hook class
@@ -652,7 +653,7 @@
 
     /**
      * Get the user's subclassed iteration complete reporter hook class.
-     * 
+     *
      * @param conf
      *            Configuration to check
      * @return The user defined vertex iteration complete reporter class
@@ -666,7 +667,7 @@
 
     /**
      * Get the job configuration parameter whether the vertex states will increase dynamically
-     * 
+     *
      * @param conf
      *            the job configuration
      * @return the boolean setting of the parameter, by default it is false
@@ -677,7 +678,7 @@
 
     /**
      * Get the specified frame size
-     * 
+     *
      * @param conf
      *            the job configuration
      * @return the specified frame size; -1 if it is not set by users
@@ -688,7 +689,7 @@
 
     /**
      * Should the job use LSM or B-tree to store vertices
-     * 
+     *
      * @param conf
      * @return
      */
@@ -698,52 +699,52 @@
 
     /***
      * Get the spilling dir name for global aggregates
-     * 
+     *
      * @param conf
      * @param superStep
      * @return the spilling dir name
      */
     public static String getGlobalAggregateSpillingDirName(Configuration conf, long superStep) {
-        return "/tmp/pregelix/agg/" + conf.get(PregelixJob.JOB_ID) + "/" + superStep;
+        return TMP_DIR + conf.get(PregelixJob.JOB_ID) + "/agg/" + superStep;
     }
 
     /**
      * Get the path for vertex checkpointing
-     * 
+     *
      * @param conf
      * @param lastSuperStep
      * @return the path for vertex checkpointing
      */
     public static String getVertexCheckpointPath(Configuration conf, long lastSuperStep) {
-        return "/tmp/ckpoint/" + BspUtils.getJobId(conf) + "/vertex/" + lastSuperStep;
+        return TMP_DIR + BspUtils.getJobId(conf) + "/ckpoint/vertex/" + lastSuperStep;
     }
 
     /**
      * Get the path for message checkpointing
-     * 
+     *
      * @param conf
      * @param lastSuperStep
      * @return the path for message checkpointing
      */
     public static String getMessageCheckpointPath(Configuration conf, long lastSuperStep) {
-        String path = "/tmp/ckpoint/" + BspUtils.getJobId(conf) + "/message/" + lastSuperStep;
+        String path = TMP_DIR + BspUtils.getJobId(conf) + "/ckpoint/message/" + lastSuperStep;
         return path;
     }
 
     /**
      * Get the path for message checkpointing
-     * 
+     *
      * @param conf
      * @param lastSuperStep
      * @return the path for message checkpointing
      */
     public static String getSecondaryIndexCheckpointPath(Configuration conf, long lastSuperStep) {
-        return "/tmp/ckpoint/" + BspUtils.getJobId(conf) + "/secondaryindex/" + lastSuperStep;
+        return TMP_DIR + BspUtils.getJobId(conf) + "/ckpoint/secondaryindex/" + lastSuperStep;
     }
 
     /***
      * Get the recovery count
-     * 
+     *
      * @return recovery count
      */
     public static int getRecoveryCount(Configuration conf) {
@@ -752,7 +753,7 @@
 
     /***
      * Get enable dynamic optimization
-     * 
+     *
      * @param conf
      *            Configuration
      * @return true if enabled; otherwise false
@@ -763,7 +764,7 @@
 
     /***
      * Get the user-set checkpoint interval
-     * 
+     *
      * @param conf
      * @return the checkpoint interval
      */
@@ -773,7 +774,7 @@
 
     /**
      * Get the grouping algorithm
-     * 
+     *
      * @param conf
      * @return true-sort; false-hash
      */
@@ -783,7 +784,7 @@
 
     /**
      * Get the memory limit for the grouping algorithm (hash only)
-     * 
+     *
      * @param conf
      * @return the memory limit for hash-based grouping
      */
@@ -793,7 +794,7 @@
 
     /**
      * Get the memory limit for the sort algorithm
-     * 
+     *
      * @param conf
      * @return the memory limit for sorting
      */
@@ -803,7 +804,7 @@
 
     /**
      * Get the desired number of workers
-     * 
+     *
      * @param conf
      * @return the number of workers
      */
@@ -813,7 +814,7 @@
 
     /**
      * Get whether the combiner key can be skipped when calling a user-defined combine function
-     * 
+     *
      * @param conf
      * @return true to skip; false otherwise
      */
@@ -823,7 +824,7 @@
 
     /**
      * Get whether a merge connector is used
-     * 
+     *
      * @param conf
      * @return true -merge; false-no merge
      */
@@ -833,7 +834,7 @@
 
     /**
      * return the maximum iteration number
-     * 
+     *
      * @param conf
      * @return the maximum iteration number
      */
@@ -845,7 +846,7 @@
             throws HyracksDataException {
         try {
             FileSystem dfs = FileSystem.get(conf);
-            String pathStr = TMP_DIR + jobId + "agg";
+            String pathStr = TMP_DIR + jobId + File.separator + "global-agg";
             Path path = new Path(pathStr);
             FSDataInputStream input = dfs.open(path);
             int numOfAggs = createFinalAggregateValues(conf).size();
@@ -868,7 +869,7 @@
 
     public static HashMap<String, Writable> readAllGlobalAggregateValues(Configuration conf, String jobId)
             throws HyracksDataException {
-        String pathStr = TMP_DIR + jobId + "agg";
+        String pathStr = TMP_DIR + jobId + File.separator + "global-agg";
         Path path = new Path(pathStr);
         List<Writable> aggValues = createFinalAggregateValues(conf);
         HashMap<String, Writable> finalAggs = new HashMap<>();
@@ -895,7 +896,7 @@
     }
 
     static Counters readCounters(int superstep, Configuration conf, String jobId) throws HyracksDataException {
-        String pathStr = TMP_DIR + jobId + BspUtils.COUNTERS_VALUE_ON_ITERATION + superstep;
+        String pathStr = TMP_DIR + jobId + File.separator + BspUtils.COUNTERS_VALUE_ON_ITERATION + superstep;
         Path path = new Path(pathStr);
         Counters savedCounters = new Counters();
         try {
@@ -911,7 +912,7 @@
 
     static void writeCounters(Counters toWrite, int superstep, Configuration conf, String jobId)
             throws HyracksDataException {
-        String pathStr = TMP_DIR + jobId + BspUtils.COUNTERS_VALUE_ON_ITERATION + superstep;
+        String pathStr = TMP_DIR + jobId + File.separator + BspUtils.COUNTERS_VALUE_ON_ITERATION + superstep;
         Path path = new Path(pathStr);
         try {
             FileSystem dfs = FileSystem.get(conf);
@@ -924,7 +925,7 @@
     }
 
     static int readCountersLastIteration(Configuration conf, String jobId) throws HyracksDataException {
-        String pathStr = TMP_DIR + jobId + BspUtils.COUNTERS_LAST_ITERATION_COMPLETED;
+        String pathStr = TMP_DIR + jobId + File.separator + BspUtils.COUNTERS_LAST_ITERATION_COMPLETED;
         Path path = new Path(pathStr);
         IntWritable lastIter = new IntWritable();
         try {
@@ -939,7 +940,7 @@
     }
 
     static void writeCountersLastIteration(int superstep, Configuration conf, String jobId) throws HyracksDataException {
-        String pathStr = TMP_DIR + jobId + BspUtils.COUNTERS_LAST_ITERATION_COMPLETED;
+        String pathStr = TMP_DIR + jobId + File.separator + BspUtils.COUNTERS_LAST_ITERATION_COMPLETED;
         Path path = new Path(pathStr);
         try {
             FileSystem dfs = FileSystem.get(conf);
diff --git a/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/JobStateUtils.java b/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/JobStateUtils.java
index 4a98167..cdcf040 100644
--- a/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/JobStateUtils.java
+++ b/pregelix/pregelix-api/src/main/java/edu/uci/ics/pregelix/api/util/JobStateUtils.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,6 +14,7 @@
  */
 package edu.uci.ics.pregelix.api.util;
 
+import java.io.File;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
@@ -33,7 +34,7 @@
     public static void writeForceTerminationState(Configuration conf, String jobId) throws HyracksDataException {
         try {
             FileSystem dfs = FileSystem.get(conf);
-            String pathStr = TMP_DIR + jobId + "fterm";
+            String pathStr = TMP_DIR + jobId + File.separator + "fterm";
             Path path = new Path(pathStr);
             if (!dfs.exists(path)) {
                 FSDataOutputStream output = dfs.create(path, true);
@@ -49,7 +50,7 @@
     public static boolean readForceTerminationState(Configuration conf, String jobId) throws HyracksDataException {
         try {
             FileSystem dfs = FileSystem.get(conf);
-            String pathStr = TMP_DIR + jobId + "fterm";
+            String pathStr = TMP_DIR + jobId + File.separator + "fterm";
             Path path = new Path(pathStr);
             if (dfs.exists(path)) {
                 return true;
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java
index 3d5f5cd..3057ba0 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/driver/Driver.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -75,9 +75,9 @@
             Counters.DISK_READ, Counters.DISK_WRITE, Counters.NETWORK_IO_READ, Counters.NETWORK_IO_WRITE };
     private static final Log LOG = LogFactory.getLog(Driver.class);
     private IHyracksClientConnection hcc;
-    private Class exampleClass;
+    private final Class exampleClass;
     private boolean profiling = false;
-    private StringBuffer counterBuffer = new StringBuffer();
+    private final StringBuffer counterBuffer = new StringBuffer();
 
     public Driver(Class exampleClass) {
         this.exampleClass = exampleClass;
@@ -148,6 +148,7 @@
                         boolean compatible = i == 0 ? false : compatible(lastJob, currentJob);
                         /** load the data */
                         if (!failed) {
+                            IterationUtils.makeTempDirectory(currentJob.getConfiguration());
                             if (i == 0) {
                                 jobGen.reset(currentJob);
                                 loadData(currentJob, jobGen, deploymentId);
@@ -175,9 +176,6 @@
                     /** finish the jobs */
                     finishJobs(jobGen, deploymentId);
 
-                    /** clear checkpoints if any */
-                    jobGen.clearCheckpoints();
-
                     /** clear state */
                     runClearState(deploymentId, jobGen, true);
 
@@ -203,6 +201,15 @@
             counterContext.stop();
         } catch (Exception e) {
             throw new HyracksException(e);
+        } finally {
+            /** clear temporary directories */
+            try {
+                for (PregelixJob job : jobs) {
+                    IterationUtils.removeTempDirectory(job.getConfiguration());
+                }
+            } catch (Exception e) {
+                throw new HyracksException(e);
+            }
         }
     }
 
@@ -220,8 +227,8 @@
         return lastVertexIdClass.equals(currentVertexIdClass)
                 && lastVertexValueClass.equals(currentVertexValueClass)
                 && lastEdgeValueClass.equals(currentEdegeValueClass)
-                && (currentInputPaths.length == 0 || (currentInputPaths.length == 1 && lastOutputPath
-                        .equals(currentInputPaths[0])));
+                && (currentInputPaths.length == 0 || currentInputPaths.length == 1
+                        && lastOutputPath.equals(currentInputPaths[0]));
     }
 
     private JobGen selectJobGen(Plan planChoice, PregelixJob currentJob, IOptimizer optimizer) {
@@ -266,9 +273,11 @@
         URLClassLoader classLoader = (URLClassLoader) exampleClass.getClassLoader();
         List<File> jars = new ArrayList<File>();
         URL[] urls = classLoader.getURLs();
-        for (URL url : urls)
-            if (url.toString().endsWith(".jar"))
+        for (URL url : urls) {
+            if (url.toString().endsWith(".jar")) {
                 jars.add(new File(url.getPath()));
+            }
+        }
         DeploymentId deploymentId = installApplication(jars);
         return deploymentId;
     }
@@ -332,7 +341,7 @@
             terminate = IterationUtils.readTerminationState(job.getConfiguration(), jobGen.getJobId())
                     || IterationUtils.readForceTerminationState(job.getConfiguration(), jobGen.getJobId())
                     || i >= BspUtils.getMaxIteration(job.getConfiguration());
-            if (ckpHook.checkpoint(i) || (ckpInterval > 0 && i % ckpInterval == 0)) {
+            if (ckpHook.checkpoint(i) || ckpInterval > 0 && i % ckpInterval == 0) {
                 runCheckpoint(deploymentId, jobGen, i);
                 snapshotJobIndex.set(currentJobIndex);
                 snapshotSuperstep.set(i);
@@ -480,12 +489,13 @@
 }
 
 class FileFilter implements FilenameFilter {
-    private String ext;
+    private final String ext;
 
     public FileFilter(String ext) {
         this.ext = "." + ext;
     }
 
+    @Override
     public boolean accept(File dir, String name) {
         return name.endsWith(ext);
     }
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
index c1fb82c..adc495e 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/jobgen/JobGen.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -137,7 +137,7 @@
 
 public abstract class JobGen implements IJobGen {
     private static final Logger LOGGER = Logger.getLogger(JobGen.class.getName());
-    protected static final int BF_HINT=100000;
+    protected static final int BF_HINT = 100000;
     protected static final int MB = 1048576;
     protected static final float DEFAULT_BTREE_FILL_FACTOR = 1.00f;
     protected static final int tableSize = 1575767;
@@ -149,7 +149,7 @@
     protected IStorageManagerInterface storageManagerInterface = StorageManagerInterface.INSTANCE;
     protected String jobId = UUID.randomUUID().toString();;
     protected int frameSize = ClusterConfig.getFrameSize();
-    protected int maxFrameNumber = (int) (((long) 64 * MB) / frameSize);
+    protected int maxFrameNumber = (int) ((long) 64 * MB / frameSize);
     protected IOptimizer optimizer;
 
     private static final Map<String, String> MERGE_POLICY_PROPERTIES;
@@ -257,19 +257,21 @@
 
     @Override
     public JobSpecification generateJob(int iteration) throws HyracksException {
-        if (iteration <= 0)
+        if (iteration <= 0) {
             throw new IllegalStateException("iteration number cannot be less than 1");
-        if (iteration == 1)
+        }
+        if (iteration == 1) {
             return generateFirstIteration(iteration);
-        else
+        } else {
             return generateNonFirstIteration(iteration);
+        }
     }
 
     @SuppressWarnings({ "rawtypes", "unchecked" })
     public JobSpecification scanSortPrintGraph(String nodeName, String path) throws HyracksException {
         Class<? extends WritableComparable<?>> vertexIdClass = BspUtils.getVertexIndexClass(conf);
         Class<? extends Writable> vertexClass = BspUtils.getVertexClass(conf);
-        int maxFrameLimit = (int) (((long) 512 * MB) / frameSize);
+        int maxFrameLimit = (int) ((long) 512 * MB / frameSize);
         JobSpecification spec = new JobSpecification(frameSize);
         IFileSplitProvider fileSplitProvider = getFileSplitProvider(jobId, PRIMARY_INDEX);
 
@@ -365,7 +367,8 @@
         typeTraits[1] = new TypeTraits(false);
         BTreeSearchOperatorDescriptor scanner = new BTreeSearchOperatorDescriptor(spec, recordDescriptor,
                 storageManagerInterface, lcManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, null,
-                null, null, true, true, getIndexDataflowHelperFactory(), false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+                null, null, true, true, getIndexDataflowHelperFactory(), false, false, null,
+                NoOpOperationCallbackFactory.INSTANCE);
         setLocationConstraint(spec, scanner);
 
         /**
@@ -436,12 +439,6 @@
         return spec;
     }
 
-    public void clearCheckpoints() throws IOException {
-        FileSystem dfs = FileSystem.get(conf);
-        // clear the checkpoint directory
-        dfs.delete(new Path("/tmp/ckpoint/" + jobId), true);
-    }
-
     @Override
     public JobSpecification[] generateLoadingCheckpoint(int lastCheckpointedIteration) throws HyracksException {
         try {
@@ -473,6 +470,7 @@
     /***
      * generate a "clear state" job
      */
+    @Override
     public JobSpecification generateClearState(boolean allStates) throws HyracksException {
         JobSpecification spec = new JobSpecification();
         ClearStateOperatorDescriptor clearState = new ClearStateOperatorDescriptor(spec, jobId, allStates);
@@ -483,7 +481,7 @@
 
     /***
      * drop the sindex
-     * 
+     *
      * @return JobSpecification
      * @throws HyracksException
      */
@@ -540,8 +538,9 @@
         try {
             splits = inputFormat.getSplits(job, fileSplitProvider.getFileSplits().length);
             LOGGER.info("number of splits: " + splits.size());
-            for (InputSplit split : splits)
+            for (InputSplit split : splits) {
                 LOGGER.info(split.toString());
+            }
         } catch (Exception e) {
             throw new HyracksDataException(e);
         }
@@ -628,7 +627,8 @@
 
         BTreeSearchOperatorDescriptor scanner = new BTreeSearchOperatorDescriptor(spec, recordDescriptor,
                 storageManagerInterface, lcManagerProvider, fileSplitProvider, typeTraits, comparatorFactories, null,
-                null, null, true, true, getIndexDataflowHelperFactory(), false, false, null, NoOpOperationCallbackFactory.INSTANCE);
+                null, null, true, true, getIndexDataflowHelperFactory(), false, false, null,
+                NoOpOperationCallbackFactory.INSTANCE);
         setLocationConstraint(spec, scanner);
 
         ExternalSortOperatorDescriptor sort = null;
@@ -747,8 +747,9 @@
                     job.getConfiguration());
             splits = inputFormat.getSplits(tmpJob);
             LOGGER.info("number of splits: " + splits.size());
-            for (InputSplit split : splits)
+            for (InputSplit split : splits) {
                 LOGGER.info(split.toString());
+            }
         } catch (Exception e) {
             throw new HyracksDataException(e);
         }
@@ -808,7 +809,7 @@
 
     /**
      * Switch the plan to a desired one
-     * 
+     *
      * @param iteration
      *            , the latest completed iteration number
      * @param plan
@@ -827,7 +828,7 @@
 
     /**
      * Build a jobspec to bulkload the live vertex btree
-     * 
+     *
      * @param iteration
      * @return the job specification
      * @throws HyracksException
@@ -892,7 +893,7 @@
 
     /**
      * set the location constraint for operators
-     * 
+     *
      * @param spec
      * @param operator
      */
@@ -902,7 +903,7 @@
 
     /**
      * get the file split provider
-     * 
+     *
      * @param jobId
      * @param indexName
      * @return the IFileSplitProvider instance
@@ -920,7 +921,7 @@
 
     /**
      * Generate the pipeline for local grouping
-     * 
+     *
      * @param spec
      *            the JobSpecification
      * @param sortOrHash
@@ -981,7 +982,7 @@
             }
         } else {
             int frameLimit = BspUtils.getGroupingMemoryLimit(conf);
-            int hashTableSize = Math.round(((float) frameLimit / 1000f) * tableSize);
+            int hashTableSize = Math.round(frameLimit / 1000f * tableSize);
             /**
              * construct local group-by operator
              */
diff --git a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/util/PregelixHyracksIntegrationUtil.java b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/util/PregelixHyracksIntegrationUtil.java
index c584971..fd0d6f9 100644
--- a/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/util/PregelixHyracksIntegrationUtil.java
+++ b/pregelix/pregelix-core/src/main/java/edu/uci/ics/pregelix/core/util/PregelixHyracksIntegrationUtil.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -62,7 +62,7 @@
         ccConfig.defaultMaxJobAttempts = 0;
         ccConfig.jobHistorySize = 1;
         ccConfig.profileDumpPeriod = -1;
-        ccConfig.heartbeatPeriod = 50;
+        ccConfig.heartbeatPeriod = 500;
         ccConfig.maxHeartbeatLapsePeriods = 20;
 
         // cluster controller
diff --git a/pregelix/pregelix-dataflow-std/src/main/java/edu/uci/ics/pregelix/dataflow/std/util/TupleDeserializer.java b/pregelix/pregelix-dataflow-std/src/main/java/edu/uci/ics/pregelix/dataflow/std/util/TupleDeserializer.java
index dd1a64e..458d842 100644
--- a/pregelix/pregelix-dataflow-std/src/main/java/edu/uci/ics/pregelix/dataflow/std/util/TupleDeserializer.java
+++ b/pregelix/pregelix-dataflow-std/src/main/java/edu/uci/ics/pregelix/dataflow/std/util/TupleDeserializer.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -26,10 +26,10 @@
 
 public class TupleDeserializer {
     private static String ERROR_MSG = "Out-of-bound read in your Writable implementations of types for vertex id, vertex value, edge value or message --- check your readFields and write implmenetation";
-    private Object[] record;
-    private RecordDescriptor recordDescriptor;
-    private ResetableByteArrayInputStream bbis;
-    private DataInputStream di;
+    private final Object[] record;
+    private final RecordDescriptor recordDescriptor;
+    private final ResetableByteArrayInputStream bbis;
+    private final DataInputStream di;
 
     public TupleDeserializer(RecordDescriptor recordDescriptor) {
         this.recordDescriptor = recordDescriptor;
@@ -116,10 +116,9 @@
             if (nullLeft) {
                 byte[] rightData = right.getFieldData(1);
                 int rightFieldOffset = right.getFieldStart(1);
-                int rightLen = right.getFieldLength(1);
                 /** skip a halted and no message vertex without deserializing it */
-                if (rightData[rightFieldOffset + rightLen - 1] == 1) {
-                    // halt flag is the last byte of any vertex
+                if ((rightData[rightFieldOffset] & 1) == 1) {
+                    // halt flag is the last bit of the first byte of any vertex value
                     record[0] = null;
                     record[1] = null;
                     return record;
diff --git a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/VertexFileScanOperatorDescriptor.java b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/VertexFileScanOperatorDescriptor.java
index e444975..36a93e5 100644
--- a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/VertexFileScanOperatorDescriptor.java
+++ b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/VertexFileScanOperatorDescriptor.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,6 +15,7 @@
 package edu.uci.ics.pregelix.dataflow;
 
 import java.io.DataOutput;
+import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.ByteBuffer;
@@ -77,18 +78,24 @@
         this.recordDescriptors[0] = rd;
     }
 
+    @Override
     public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
             IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
             throws HyracksDataException {
         final List<FileSplit> splits = splitsFactory.getSplits();
 
         return new AbstractUnaryOutputSourceOperatorNodePushable() {
-            private ContextFactory ctxFactory = new ContextFactory();
+            private final ContextFactory ctxFactory = new ContextFactory();
+            private String jobId;
 
             @Override
             public void initialize() throws HyracksDataException {
                 try {
                     Configuration conf = confFactory.createConfiguration(ctx);
+
+                    //get the info for spilling vertices to HDFS
+                    jobId = BspUtils.getJobId(conf);
+
                     writer.open();
                     for (int i = 0; i < scheduledLocations.length; i++) {
                         if (scheduledLocations[i].equals(ctx.getJobletContext().getApplicationContext().getNodeId())) {
@@ -113,7 +120,7 @@
 
             /**
              * Load the vertices
-             * 
+             *
              * @parameter IHyracks ctx
              * @throws IOException
              * @throws IllegalAccessException
@@ -125,8 +132,10 @@
             private void loadVertices(final IHyracksTaskContext ctx, Configuration conf, int splitId)
                     throws IOException, ClassNotFoundException, InterruptedException, InstantiationException,
                     IllegalAccessException, NoSuchFieldException, InvocationTargetException {
+                int treeVertexSizeLimit = IterationUtils.getVFrameSize(ctx) / 2;
+                int dataflowPageSize = ctx.getFrameSize();
                 ByteBuffer frame = ctx.allocateFrame();
-                FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+                FrameTupleAppender appender = new FrameTupleAppender(dataflowPageSize);
                 appender.reset(frame, true);
 
                 VertexInputFormat vertexInputFormat = BspUtils.createVertexInputFormat(conf);
@@ -136,7 +145,7 @@
 
                 VertexReader vertexReader = vertexInputFormat.createVertexReader(split, mapperContext);
                 vertexReader.initialize(split, mapperContext);
-                Vertex readerVertex = (Vertex) BspUtils.createVertex(mapperContext.getConfiguration());
+                Vertex readerVertex = BspUtils.createVertex(mapperContext.getConfiguration());
                 ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldSize);
                 DataOutput dos = tb.getDataOutput();
 
@@ -146,7 +155,7 @@
                 /**
                  * empty vertex value
                  */
-                Writable emptyVertexValue = (Writable) BspUtils.createVertexValue(conf);
+                Writable emptyVertexValue = BspUtils.createVertexValue(conf);
 
                 while (vertexReader.nextVertex()) {
                     readerVertex = vertexReader.getCurrentVertex();
@@ -165,13 +174,29 @@
                     readerVertex.write(dos);
                     tb.addFieldEndOffset();
 
+                    if (tb.getSize() >= treeVertexSizeLimit || tb.getSize() > dataflowPageSize) {
+                        //if (tb.getSize() < dataflowPageSize) {
+                        //spill vertex to HDFS if it cannot fit into a tree storage page
+                        String pathStr = BspUtils.TMP_DIR + jobId + File.separator + vertexId;
+                        readerVertex.setSpilled(pathStr);
+                        tb.reset();
+                        vertexId.write(dos);
+                        tb.addFieldEndOffset();
+                        //vertex content will be spilled to HDFS
+                        readerVertex.write(dos);
+                        tb.addFieldEndOffset();
+                        readerVertex.setUnSpilled();
+                    }
                     if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
-                        if (appender.getTupleCount() <= 0)
+                        if (appender.getTupleCount() <= 0) {
                             throw new IllegalStateException("zero tuples in a frame!");
+                        }
                         FrameUtils.flushFrame(frame, writer);
                         appender.reset(frame, true);
                         if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
-                            throw new IllegalStateException();
+                            //this place should never be reached, otherwise it is a bug
+                            throw new IllegalStateException(
+                                    "An overflow vertex content should not be flushed into bulkload dataflow.");
                         }
                     }
                 }
diff --git a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java
index 0f10b4d..0c11b60 100644
--- a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java
+++ b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/context/RuntimeContext.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -59,14 +59,17 @@
     private final IFileMapManager fileMapManager;
     private final IOManager ioManager;
     private final Map<String, PJobContext> activeJobs = new ConcurrentHashMap<String, PJobContext>();
+    private final int vFrameSize;
 
     private final ThreadFactory threadFactory = new ThreadFactory() {
+        @Override
         public Thread newThread(Runnable r) {
             return new Thread(r);
         }
     };
 
     public RuntimeContext(INCApplicationContext appCtx, int vFrameSize) {
+        this.vFrameSize = vFrameSize;
         int pageSize = vFrameSize;
         long memSize = Runtime.getRuntime().maxMemory();
         long bufferSize = memSize / 4;
@@ -168,6 +171,10 @@
         return activeJob == null ? 0 : activeJob.getVertexContext().getSuperstep();
     }
 
+    public int getVFrameSize() {
+        return vFrameSize;
+    }
+
     public void setJobContext(String jobId, TaskAttemptContext tCtx) {
         PJobContext activeJob = getOrCreateActiveJob(jobId);
         activeJob.getVertexContext().setContext(tCtx);
@@ -211,4 +218,4 @@
         return ioManager.createWorkspaceFile(prefix);
     }
 
-}
\ No newline at end of file
+}
diff --git a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/util/IterationUtils.java b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/util/IterationUtils.java
index 4ca67e6..f85c05a 100644
--- a/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/util/IterationUtils.java
+++ b/pregelix/pregelix-dataflow/src/main/java/edu/uci/ics/pregelix/dataflow/util/IterationUtils.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -14,6 +14,7 @@
  */
 package edu.uci.ics.pregelix.dataflow.util;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
@@ -45,7 +46,7 @@
 
     /**
      * Get the input files' byte size
-     * 
+     *
      * @param job
      */
     public static long getInputFileSize(PregelixJob job) {
@@ -122,6 +123,12 @@
         return context.getSuperstep(pregelixJobId);
     }
 
+    public static int getVFrameSize(IHyracksTaskContext ctx) {
+        INCApplicationContext appContext = ctx.getJobletContext().getApplicationContext();
+        RuntimeContext context = (RuntimeContext) appContext.getApplicationObject();
+        return context.getVFrameSize();
+    }
+
     public static void setJobContext(String pregelixJobId, IHyracksTaskContext ctx, TaskAttemptContext tCtx) {
         INCApplicationContext appContext = ctx.getJobletContext().getApplicationContext();
         RuntimeContext context = (RuntimeContext) appContext.getApplicationObject();
@@ -146,7 +153,7 @@
             throws HyracksDataException {
         try {
             FileSystem dfs = FileSystem.get(conf);
-            String pathStr = IterationUtils.TMP_DIR + pregelixJobId;
+            String pathStr = IterationUtils.TMP_DIR + pregelixJobId + File.separator + "terminate";
             Path path = new Path(pathStr);
             FSDataOutputStream output = dfs.create(path, true);
             output.writeBoolean(terminate);
@@ -161,7 +168,7 @@
             List<Writable> aggs) throws HyracksDataException {
         try {
             FileSystem dfs = FileSystem.get(conf);
-            String pathStr = IterationUtils.TMP_DIR + pregelixJobId + "agg";
+            String pathStr = IterationUtils.TMP_DIR + pregelixJobId + File.separator + "global-agg";
             Path path = new Path(pathStr);
             FSDataOutputStream output;
             output = dfs.create(path, true);
@@ -181,7 +188,7 @@
     public static boolean readTerminationState(Configuration conf, String pregelixJobId) throws HyracksDataException {
         try {
             FileSystem dfs = FileSystem.get(conf);
-            String pathStr = IterationUtils.TMP_DIR + pregelixJobId;
+            String pathStr = IterationUtils.TMP_DIR + pregelixJobId + File.separator + "terminate";
             Path path = new Path(pathStr);
             FSDataInputStream input = dfs.open(path);
             boolean terminate = input.readBoolean();
@@ -210,4 +217,24 @@
         return BspUtils.readAllGlobalAggregateValues(conf, jobId);
     }
 
+    public static void makeTempDirectory(Configuration conf) throws IOException {
+        FileSystem dfs = FileSystem.get(conf);
+        String jobId = BspUtils.getJobId(conf);
+        String pathStr = TMP_DIR + jobId;
+        Path path = new Path(pathStr);
+        if (dfs.exists(path)) {
+            dfs.mkdirs(path);
+        }
+    }
+
+    public static void removeTempDirectory(Configuration conf) throws IOException {
+        FileSystem dfs = FileSystem.get(conf);
+        String jobId = BspUtils.getJobId(conf);
+        String pathStr = TMP_DIR + jobId;
+        Path path = new Path(pathStr);
+        if (dfs.exists(path)) {
+            dfs.deleteOnExit(path);
+        }
+    }
+
 }
diff --git a/pregelix/pregelix-example/data/skew/data.txt b/pregelix/pregelix-example/data/skew/data.txt
new file mode 100644
index 0000000..9de6b8d
--- /dev/null
+++ b/pregelix/pregelix-example/data/skew/data.txt
@@ -0,0 +1,23 @@
+0 1
+1 1 2
+2 1 2 3
+3 1 2 3 4
+4 1 2 3 4 5
+5 1 2 3 4 5 6
+6 1 2 3 4 5 6 7
+7 1 2 3 4 5 6 7 8
+8 1 2 3 4 5 6 7 8 9
+9 1 2 3 4 5 6 7 8 9 10
+10 11
+11 11 12
+12 11 12 13
+13 11 12 13 14
+14 11 12 13 14 15
+15 11 12 13 14 15 16
+16 11 12 13 14 15 16 17
+17 11 12 13 14 15 16 17 18
+18 11 12 13 14 15 16 17 18 19
+19 0 11 12 13 14 15 16 17 18 19
+412454 96 2487 2575 2982 5420 5770 6522 6697 6949 7200 8315 9287 9972 10331 10762 10926 11684 11781 13357 14913 15080 15618 16284 17935 20317 22313 22513 23611 23819 25092 25165 26197 26257 27259 27288 27448 28000 28076 28356 28576 28616 28637 29468 31491 31566 32224 33788 34254 35152 35207 35874 36215 36241 37512 37898 38085 38995 39138 39459 39892 39969 41136 41702 41717 43378 43840 43998 44352 44424 44791 44950 45042 45484 45708 45983 46143 46292 46846 47146 47478 47721 49524 50080 50146 50694 51342 51450 52108 52215 52695 53876 54570 54907 55449 55485 55887 56677 56756 57471 59964 60097 60872 61547 61568 61796 62292 63076 63359 63915 64051 65958 67429 68578 69316 71618 71771 72091 72260 72498 72662 72978 73182 74205 74469 74642 74949 75511 76440 77452 77744 78268 79515 79590 79832 80030 81446 82117 84207 84764 84920 84927 85204 86102 86726 86837 89053 89219 89655 90866 91140 91396 91868 93801 94038 94079 94846 95338 95424 95504 96575 96614 96822 96978 97233 97379 97696 98421 98580 98982 99080 99655 99788 101147 101869 102203 102446 103077 103328 103338 103980 104109 104423 104937 105172 106707 106769 107395 107914 108223 109193 109434 110107 110998 111032 111401 111537 111817 111972 111999 112010 112171 112451 112501 113467 113474 113972 114395 115110 115129 115603 115652 116126 116648 117130 118439 118564 118833 118969 119113 119119 119243 119388 119884 121229 121507 121638 121880 122046 122088 122133 122584 123127 123362 123396 123470 123530 123809 123959 124254 125458 125499 125607 125676 126217 126290 126558 126580 126582 127015 127302 127319 127553 127833 128152 128262 128407 128807 128811 129125 129893 129895 130143 130160 130174 130902 130912 131337 131398 131692 132035 132057 132476 133140 133282 133399 133492 133909 134280 134664 135564 136050 136825 137058 137482 137898 140143 140593 141374 142638 143585 144376 144429 145339 145510 145658 145805 146461 147160 148611 149659 149912 151114 151975 152001 152002 153489 153609 153700 154517 155003 155068 155369 155496 155507 155875 157164 157789 158153 158249 158546 158620 159282 159367 159790 161406 161730 163437 164061 164860 165341 166981 167176 167258 167742 171230 172256 172855 172860 172983 173667 174239 174378 174689 174864 175771 175870 178070 178112 179150 180147 180275 180883 181523 181586 182103 184541 184577 184834 184976 185127 185461 185970 186866 186938 187433 187477 188162 188170 189113 189224 189632 189645 191033 191270 194670 195291 196506 196899 197910 198453 198477 198640 200613 201199 201360 201401 201499 201621 203522 203596 203925 204004 205529 206834 206932 207974 208238 208799 208824 210234 211208 211279 211458 212115 212281 212872 213039 213180 214728 215198 215549 215773 215873 216078 216110 216395 217238 219282 219980 221261 221279 221425 221741 222068 222096 223806 224614 224645 226022 226184 226684 226956 227190 227590 228816 230112 232219 233295 233569 233574 234449 234780 235190 235499 235817 236991 237765 237990 238311 238671 239138 239144 239176 239984 240076 240579 240753 241764 241910 242053 243086 244561 244647 244816 245154 245205 245413 245693 246133 246605 247067 247560 247992 248251 248957 249788 249799 249946 249989 250517 250823 251982 252530 252558 252575 253558 254110 255529 255707 255822 255913 256442 256507 256574 256702 256742 257456 257583 257659 257750 258145 258366 258425 258512 258557 259068 259192 259399 260152 261266 261535 261676 261806 262309 262507 262995 263604 263676 264028 264123 265486 265827 265903 266443 267043 267275 267432 267565 268149 268252 268494 269131 269143 269279 269322 269706 270196 270430 270807 271143 271326 271651 272434 273067 274539 274702 275218 275291 277329 277941 278433 278798 280853 281353 281950 282099 283684 283785 283875 284669 284777 285229 285257 286264 287579 287718 288579 290403 291133 291918 292561 293279 293671 294600 294824 294912 295421 296056 296131 296848 297135 297192 297247 299283 301322 302639 304391 305797 306046 307437 308382 309363 312443 312888 313128 313251 313914 314448 314739 315782 317734 317952 317959 318180 318603 321661 321808 322132 322197 323008 323489 323518 324351 324779 324810 324817 324924 325129 325487 325682 326889 327161 327718 329263 331118 332971 333003 333404 333944 334012 334014 335370 335378 335655 335878 337522 337822 338361 338710 339195 339453 343287 345074 346723 347127 347962 348322 348427 348436 348488 348744 348752 349864 350298 351176 351308 352520 353531 353619 353776 354377 356378 356480 356568 356601 356659 357146 357360 357410 358289 358351 358503 359227 359761 361120 362360 362370 363170 363272 363969 364873 365286 366044 366413 366539 367054 367125 367523 367761 368510 368986 369607 369941 370489 370702 370715 372083 372409 372805 372886 373118 373916 373949 375315 376512 376729 376761 376794 376879 377664 377843 378553 378684 379037 379070 379256 379300 379487 379622 380356 380694 381523 381667 382105 383002 383930 384275 384378 384594 384749 385301 385455 385459 385658 385792 386722 386755 387357 388323 388470 388542 388607 388625 388984 389600 389648 390593 390757 391858 391951 392270 392666 392912 393429 393884 393895 393991 394120 394140 394433 394686 394720 395062 395203 395274 395491 395882 396031 396454 396746 396776 396974 397057 397211 397450 397732 397782 397900 397954 398218 398234 398348 398530 398601 399034 399190 399476 399620 399730 399826 400285 400447 400491 400560 401088 401209 401495 401918 402044 402639 402933 403611 404097 404231 404370 404377 404762 405021 405844 406410 406717 407325 408170 409310 410166 410285 411497 412076 412789 413522 414276 414293 414417 414738 414881 415700 415859 415950 416341 416580 417851 419696 420824 421847 422602 424639 424838 424911 425423 426669 426938 427290 427905 428375 428876 429666 432846 432851 432905 433963 434075 435238 435603 435986 437869 440529 440571 440601 441581 442033 443074 444053 444348 444779 446890 446914 448635 449931 450041 451145 451371 452642 453598 454316 454821 455746 455839 456799 457795 457800 457864 458077 459139 459298 459818 460412 460905 461547 461693 462002 462678 462755 464151 464672 468082 468104 468769 469148 469594 470235 470578 471058 472168 475386 475456 475557 476865 479787 481273 482052 482296 482853 483230 483840 484077 484713 485945 488467 489484 489917 491955 492130 492141 492638 493008 493030 494832 495063 495110 495708 495810 496211 496527 497103 498162 498944 500109 500894 501016 501812 502027 502098 502441 502607 503092 504384 504404 504808 505744 505893 506008 506229 506244 506381 508085 508656 508966 508989 509105 509437 509583 510130 510381 510678 512681 513824 513967 514107 514186 515764 516190 516308 516791 519265 520119 520723 521071 522051 522060 522283 522681 523712 524819 525013 525071 525265 525800 525900 526281 526411 526657 527434 527472 528561 529649 530286 530537 530634 530782 532218 532333 532500 532746 532856 533283 533848 533900 534198 534405 534485 534555 534866 535507 535583 536021 536421 536907 537192 537233 537248 537380 537924 539021 539236 539304 539881 540601 543402 544391 545287 546439 546596 546838 548055 548875 549391 549841 550558 550722 550841 552079 553009 553172 553245 554481 555768 556986 557749 557859 558280 559373 559651 559935 560009 560182 560297 560331 561550 562101 562645 562725 563165 564061 565043 565204 565729 566604 567258 567352 567360 567388 568154 570497 571573 571884 571938 572041 572177 572822 573247 573575 574103 575049 575415 575504 576066 576625 577815 579231 580416 582510 584307 584324 585499 586845 587883 588716 588752 588768 588911 590513 590976 592733 593710 594992 595621 596845 598821 599231 601003 602497 605144 608194 608329 608698 609011 610111 610892 611868 612098 612105 612745 615853 616488 616890 619747 620845 621591 622070 622285 624480 624499 624581 625054 625082 626141 629285 629288 630494 630674 630780 631029 631412 632409 633016 633427 633655 634329 634421 634513 635932 636258 636596 636917 637102 637240 637575 638549 638811 639130 639409 639563 642035 642873 644791 645200 645264 645372 645580 646121 646149 646475 646786 647041 647072 647370 647683 648029 648252 648809 649402 649895 649937 650533 651080 651728 652154 653191 653904 654188 655278 656702 657332 657505 657553 658244 658284 658679 658941 659800 659887 660041 660487 661335 661817 662714 663114 663162 663630 664048 664134 665216 665333 665403 665615 667202 667586 668010 668065 668452 668454 668540 668542 668671 668724 668870 668986 669050 669363 669627 669964 669968 669997 670283 670394 670653 670871 670876 670956 671469 671840 672107 672224 672335 672425 673005 673036 673586 673707 673731 673925 674193 674686 674803 675060 675178 675192 675199 675203 675451 675550 675691 675999 676009 676022 676234 676712 677514 678274 679072 679817 679834 680077 680480 681180 681242 681339 681361 681401 681772 681897 681955 682258 682627 683327 683934 683995 683997 684203 685067 686069 687147 687523 687561 688309 688317 689742 690964 691000 694517 694729 694730 696923 704811 704891 706537 706620 707452 707671 709061 709819 710358 710824 711409 714975 716304 716391 716799 718120 718267 719745 719934 719990 720646 721325 721791 721873 722415 722529 722975 723509 724845 724977 725704 726240 726244 726280 726747 727475 727900 729659 730059 730943 731349 731427 731526 733194 733620 733827 734064 734152 734536 734765 735097 735953 736313 736829 737084 737157 737349 738298 738323 738641 738965 739323 740049 742397 742864 743088 743618 744207 745209 745857 747325 747345 748857 749181 750648 750766 751222 751463 752558 752615 752935 755473 755693 757269 758125 759002 759490 759640 760266 760787 761117 761168 761463 761838 762216 762835 763566 763892 765464 766616 767810 768979 769010 769190 771222 771334 772726 773300 773302 773975 774696 776439 776661 776679 780025 780055 780857 781239 781518 782176 782447 784631 785199 785545 786099 789280 789581 789610 790155 790182 790369 791079 792356 794991 795142 795659 795750 796824 797678 798132 798368 798617 799701 799828 800892 801821 802090 802197 802409 802464 802666 802743 803039 803571 803724 803971 804979 804980 805789 806264 806558 806802 807031 807032 807280 807604 807764 808339 808728 809062 809576 809726 809849 810010 810097 810111 810264 810354 810660 810877 811469 811526 811750 811957 812450 813003 813063 813362 813677 813915 814580 814873 814986 815104 815523 816571 816598 816599 816761 816784 816880 817224 817251 817778 818301 818382 818533 819917 819938 820101 821047 821340 821399 822469 822520 823076 823707 824162 824536 830113 830817 831807 832278 833496 833525 833999 834523 836835 838073 838616 839028 839969 841459 842292 842394 842515 842795 843524 843536 843761 843763 843767 844449 845425 845457 845951 846072 846584 847084 847756 850162 850276 851452 852245 852377 854141 855125 856336 856749 856866 857364 857691 857725 859549 860392 860912 861476 862055 862420 863132 863255 863704 863753 864080 864380 864962 866563 867951 868113 868944 871542 871841 872261 873141 873176 873181 873336 873426 873693 874675 874717 875065 875666 876030 876383 876853 876973 877376 877922 878062 878148 878675 880171 880771 882024 882370 883991 884407 884534 885191 885317 887280 888237 888618 889277 889995 890522 890739 891047 891177 891566 891703 892371 892500 893359 893689 893938 894038 894414 895123 895576 895680 896311 896759 897137 898066 898743 899395 899898 899967 900229 900904 901506 901583 901738 902563 903187 904180 904962 905827 906401 907214 907288 907327 907401 907412 907484 907702 908618 910280 910384 911492 911603 912628 912896 913537 913803 913914 914461 915470 915506 916584 916669 917134 917345 917488 918447 919479 920515 920630 921745 922292 922331 923997 924123 924451 924567 924611 924620 924953 925114 925347 925367 926489 926745 926890 927126 928180 928216 928601 929646 930166 930236 930285 931480 932794 933721 935352 935950 936087 936263 936847 937894 938179 938515 940344 941404 941618 941748 942021 942890 943413 943538 943540 943710 943782 944153 944304 944766 945042 945366 945436 945897 945995 946310 946535 946555 946659 946732 946941 947023 947067 947201 947223 947437 947594 949087 949209 949406 949829 949833 950124 950133 950802 950997 951093 951149 951977 952003 952486 952720 952887 953467 953675 953799 954512 954878 955009 955021 955664 955703 956164 956373 956394 956677 957314 957834 958708 958796 959291 959316 959986 960297 960379 960525 961056 961119 961224 961796 961906 962885 963770 965951 966302 966656 966982 967046 967336 967421 967719 969312 969381 970039 972363 974554 974619 975096 976572 976581 977234 977293 978100 979116 981708 981873 982675 983168 984721 985668 985924 987036 987306 987375 987982 989809 990112 990930 990979 991154 991411 991866 992388 992771 993048 994479 994618 995326 995603 996286 997151 997825 998521 999528 1000038 1000420 1000462 1000641 1000729 1000842 1002464 1002469 1003781 1004416 1005513 1005832 1006279 1006688 1006701 1008659 1009140 1009453 1010277 1010571 1011347 1011437 1011453 1011701 1011794 1012174 1013718 1013975 1014243 1015548 1016436 1016743 1017547 1018114 1018393 1019173 1019194 1021722 1022143 1022866 1023086 1024531 1024648 1024988 1025357 1026611 1027175 1027581 1028124 1028139 1028843 1029325 1030314 1032536 1032928 1032979 1033913 1033945 1034428 1034836 1037673 1037712 1037952 1038313 1039478 1039504 1040073 1040307 1043293 1043811 1046364 1046379 1046692 1048898 1049023 1051396 1051457 1051628 1051729 1052834 1052979 1053383 1053505 1054123 1054753 1055649 1057694 1058300 1058332 1058379 1058461 1058567 1058627 1059277 1059811 1060086 1060859 1061689 1062019 1062606 1066694 1066723 1066742 1066918 1068264 1069852 1070098 1070808 1072141 1072424 1072937 1073303 1073351 1073575 1073599 1074330 1074463 1076687 1076859 1077393 1077604 1077627 1078367 1079438 1079835 1080148 1081051 1081267 1081293 1081801 1082539 1083097 1083195 1083481 1086274 1086646 1087050 1087448 1088120 1088125 1088134 1088395 1088459 1088497 1089581 1089707 1089804 1090308 1090465 1090560 1090845 1090935 1091076 1091265 1091289 1091440 1091556 1091666 1091908 1092385 1093372 1093705 1093862 1094039 1094088 1094568 1094683 1094704 1094764 1095006 1095746 1096394 1097377 1097488 1097805 1097946 1098168 1099215 1099357 1099436 1099893 1100145 1100621 1100716 1102001 1102032 1102413 1102794 1102918 1103014 1103758 1104251 1104675 1104940 1106271 1106588 1107130 1107547 1107772 1108578 1109308 1109626 1110123 1110306 1110342 1110600 1112793 1113465 1114253 1114629 1118989 1119121 1119179 1120202 1120554 1121403 1121557 1121719 1121819 1123527 1123658 1124986 1125226 1125837 1125903 1125982 1126693 1127056 1128079 1128786 1130121 1130275 1130364 1130489 1130548 1131249 1131265 1131929 1132560 1132575 1134241 1135291 1135563 1135929 1136118 1139696 1140068 1140104 1140167 1140168 1140654 1140924 1141249 1143970 1144497 1145275 1145450 1145616 1145848 1146979 1147809 1148325 1148497 1148555 1148753 1148777 1148970 1149119 1149833 1149953 1150851 1151087 1151504 1152039 1152928 1153823 1154094 1154185 1155552 1156134 1156379 1156404 1157021 1157053 1157549 1158830 1159376 1159478 1159607 1159841 1160270 1160769 1161376 1161830 1162015 1162912 1163304 1163939 1163962 1165209 1166788 1167063 1167186 1167857 1169822 1170004 1170156 1171125 1171424 1171946 1173426 1173731 1175562 1175587 1176627 1177689 1178303 1178619 1179004 1179962 1181088 1181395 1181724 1182147 1182258 1182703 1182762 1183369 1183389 1183628 1183936 1183990 1184834 1186859 1186921 1187443 1187809 1188316 1189746 1190758 1191087 1191177 1192462 1192483 1193066 1193308 1193485 1194316 1195112 1196360 1196368 1196448 1198596 1199262 1199701 1200404 1200509 1200516 1200537 1200688 1200840 1201306 1202447 1203754 1203932 1204462 1204575 1204760 1205069 1205211 1205368 1205478 1205619 1206354 1206486 1206992 1207220 1207889 1208812 1208858 1209090 1210571 1211549 1211883 1213961 1214490 1214551 1216784 1217359 1217829 1217983 1218080 1218807 1219072 1219563 1219618 1221400 1221878 1222401 1222532 1223127 1223428 1223674 1224223 1224420 1224549 1224707 1224941 1225434 1225764 1225971 1226015 1226287 1226330 1226603 1227222 1227683 1227964 1228669 1228716 1228952 1229471 1229559 1229683 1229716 1229926 1230139 1230486 1230552 1230589 1230696 1231564 1231610 1231628 1231671 1232204 1232250 1232472 1232789 1233269 1233351 1233523 1233987 1234103 1234412 1234812 1235450 1235682 1235829 1235884 1236891 1237003 1237708 1237814 1238012 1238031 1238203 1238770 1238810 1239384 1239918 1240112 1240766 1242035 1244203 1244710 1245773 1245835 1246225 1246930 1247043 1248413 1248483 1249087 1249583 1249875 1250113 1250582 1250679 1252683 1252733 1252890 1253500 1254149 1254168 1254277 1254817 1254911 1255056 1256136 1258506 1258917 1259170 1259302 1259896 1259962 1260393 1260446 1260980 1261674 1261739 1262001 1262211 1262483 1263768 1265045 1265129 1266332 1266361 1268348 1268474 1268566 1268610 1269136 1270110 1270947 1271125 1271656 1271873 1272138 1272154 1272324 1272815 1273244 1273915 1274891 1275858 1276316 1276534 1277007 1278046 1278544 1278647 1278984 1279061 1280391 1280450 1282030 1282362 1284130 1285925 1286071 1286228 1286238 1286362 1286548 1287847 1289413 1289968 1290275 1290970 1291446 1292396 1293276 1293625 1294180 1294201 1294698 1294764 1295261 1297430 1298502 1298842 1299788 1300084 1301853 1301865 1302846 1302890 1303382 1304390 1305185 1307884 1308846 1308978 1311586 1312178 1312334 1313425 1313871 1314087 1314997 1315215 1316071 1316318 1317188 1317299 1318892 1319726 1320046 1320210 1320470 1320535 1320937 1322227 1322381 1322388 1323788 1324022 1324147 1324541 1324944 1324950 1325276 1325639 1325654 1325730 1325783 1326040 1327926 1328147 1329170 1329227 1330115 1330573 1330606 1330946 1331734 1332863 1333540 1334210 1334293 1334669 1334730 1334997 1335512 1335874 1337799 1338120 1338563 1339325 1340737 1342088 1342302 1342583 1344065 1344117 1344655 1344764 1345646 1345891 1346916 1347399 1348054 1348116 1349494 1350651 1350717 1351027 1352037 1352076 1352426 1352659 1353107 1353244 1353543 1354172 1354191 1355188 1355628 1355685 1355810 1356111 1356426 1356428 1357345 1357560 1357797 1358237 1358728 1358958 1358988 1359250 1359469 1360018 1360137 1360823 1360962 1361542 1361786 1361904 1362729 1362795 1362958 1363037 1363213 1363253 1363514 1363616 1363878 1363936 1364122 1364291 1364884 1365392 1365881 1365885 1366191 1366403 1366406 1366555 1366605 1367404 1367506 1367930 1368793 1369016 1369336 1369533 1369544 1369592 1370059 1371057 1371188 1371921 1371998 1372260 1372682 1372881 1373156 1373973 1374425 1374440 1375126 1375561 1376246 1376483 1376520 1377025 1377146 1377228 1377694 1379363 1379702 1379786 1381745 1381870 1383749 1383863 1384133 1384943 1384965 1385122 1387188 1387194 1387856 1388110 1389153 1389652 1390023 1391083 1391441 1393069 1393231 1393351 1393932 1394647 1396815 1397135 1397357 1397580 1399125 1399203 1399561 1399855 1400166 1400237 1400973 1401042 1402207 1402700 1403557 1404354 1404642 1404658 1405066 1405228 1405693 1405841 1406986 1407070 1407206 1408301 1408526 1408927 1410152 1411331 1412474 1413181 1414264 1415419 1415759 1417472 1417745 1418631 1418781 1419007 1419684 1420096 1421095 1421207 1422145 1422893 1423650 1424229 1425631 1425915 1425989 1426499 1426905 1428836 1429235 1429464 1430228 1430418 1430504 1431294 1433481 1433577 1433582 1435187 1435815 1436029 1436245 1436802 1437056 1438722 1438808 1438829 1440046 1440189 1440584 1440637 1441505 1442320 1443323 1443736 1443864 1446174 1446879 1448225 1448767 1448899 1449395 1449414 1450647 1451614 1452143 1453260 1453378 1453757 1454099 1454368 1454715 1454814 1454876 1456198 1456293 1456650 1456715 1459015 1459538 1460287 1461581 1461674 1461750 1462495 1462590 1462616 1463066 1463293 1463481 1464374 1464468 1464837 1466895 1467119 1467524 1468109 1468396 1469076 1469999 1470448 1470482 1470535 1471880 1472878 1473467 1474813 1475015 1475293 1475302 1475459 1475647 1475650 1475880 1476063 1477218 1478042 1479047 1479765 1480673 1482100 1482250 1482570 1483057 1483407 1483663 1483859 1484089 1484809 1485290 1485779 1486599 1487439 1488854 1489164 1490201 1490530 1491319 1491811 1492032 1492381 1492550 1492615 1492833 1492976 1493165 1493450 1493527 1493743 1494150 1494348 1495120 1495157 1495170 1495847 1495938 1496167 1496266 1496418 1496501 1496576 1496604 1496666 1497015 1497815 1498000 1498236 1498377 1498499 1498650 1498681 1499181 1499316 1499557 1499760 1499775 1499994 1500017 1500167 1500358 1500578 1500750 1501206 1501261 1501741 1501999 1502346 1502775 1502946 1503057 1503265 1503561 1503698 1503898 1504278 1504323 1504587 1504892 1505213 1505335 1505514 1505523 1505567 1505577 1505653 1505789 1505821 1506396 1506743 1507203 1507688 1507852 1508154 1508929 1508933 1509621 1509751 1510043 1510149 1510373 1511045 1511609 1511651 1512117 1512409 1513020 1514088 1514437 1514984 1515151 1515476 1515642 1515682 1516141 1516209 1516227 1516484 1516567 1516802 1516932 1518181 1519096 1519345 1519849 1521065 1521569 1523029 1523157 1523641 1523672 1523757 1524023 1526720 1528493 1529530 1530042 1530748 1531676 1532492 1533105 1535213 1535940 1536052 1536423 1537033 1537172 1537655 1538133 1538403 1538517 1538674 1538858 1539232 1540219 1540398 1540629 1540666 1540854 1541127 1541420 1543753 1546980 1547236 1547951 1550192 1551446 1553782 1554442 1556693 1556925 1557171 1558003 1558353 1559433 1561391 1562333 1562468 1562473 1562947 1564138 1565954 1566831 1566837 1567947 1568678 1569026 1569793 1570382 1570721 1570778 1572313 1572317 1572639 1572870 1576050 1577060 1577283 1577458 1578319 1578357 1578469 1580259 1581355 1582207 1583025 1583156 1583459 1583722 1584568 1588149 1588620 1588962 1588967 1590188 1590297 1591048 1591135 1591183 1592424 1592817 1593037 1593688 1594226 1594686 1595012 1595329 1597279 1597908 1598251 1598731 1599150 1599247 1599257 1600561 1601016 1601081 1601495 1601508 1601521 1601586 1601825 1602539 1603167 1603668 1604067 1604998 1605643 1605927 1607518 1607607 1608303 1608545 1609306 1609464 1609759 1609806 1609991 1610839 1611043 1611242 1611518 1611901 1612169 1612291 1613691 1614508 1614532 1614966 1615760 1616004 1616019 1616167 1616232 1616401 1616741 1616954 1617167 1617274 1617571 1618518 1619036 1619537 1621190 1621208 1621244 1621580 1621912 1622096 1622657 1623299 1623526 1624065 1624954 1625791 1625845 1625935 1626049 1626246 1626279 1626977 1627262 1628530 1629076 1629098 1629371 1629820 1629929 1630049 1630503 1630519 1630984 1631870 1632866 1633252 1633373 1633617 1634041 1634166 1634423 1634646 1635188 1635320 1635780 1635829 1636357 1636624 1636931 1637225 1637273 1637558 1637942 1637966 1638187 1638456 1639474 1639730 1639788 1639839 1639891 1640003 1640449 1641207 1641238 1641251 1641308 1641518 1642312 1642475 1642547 1642720 1642881 1642921 1643063 1643132 1643185 1644400 1644521 1644798 1645138 1645338 1645549 1645630 1646617 1647282 1647287 1647383 1647843 1649026 1649029 1649789 1650560 1650656 1650973 1652326 1653144 1653462 1654915 1655404 1655481 1656223 1656665 1657862 1658623 1658703 1659539 1660042 1661429 1661480 1661659 1661705 1662322 1664225 1665696 1665863 1666025 1666259 1668392 1669638 1670069 1671069 1671113 1671946 1672043 1672272 1672866 1673145 1673546 1675306 1675629 1677005 1678343 1678943 1679363 1680876 1681177 1681955 1682059 1682074 1682888 1685273 1686866 1687232 1689258 1690254 1691534 1691785 1692335 1693180 1693259 1694065 1696155 1697222 1698084 1698259 1698851 1698881 1699055 1699626 1702080 1702174 1702961 1703431 1704955 1705576 1705853 1706240 1706377 1706571 1706958 1708387 1708528 1708658 1710570 1710666 1711644 1712633 1714051 1714118 1714515 1714569 1714801 1716042 1716945 1717624 1718457 1718464 1718760 1718829 1719165 1719594 1720033 1721206 1721400 1723681 1723695 1724624 1725050 1725169 1726151 1726783 1727011 1727682 1728233 1728287 1728925 1728981 1729201 1729491 1729954 1730708 1731215 1731332 1731463 1732269 1733199 1733612 1735534 1735578 1735897 1736662 1737087 1737941 1738038 1738065 1739078 1739596 1741239 1741383 1743416 1743619 1744016 1744184 1744483 1745196 1745507 1745550 1746435 1746439 1746482 1747588 1747839 1747853 1748931 1748961 1750912 1751958 1753368 1755227 1755862 1756222 1756547 1757049 1758322 1760873 1761362 1761373 1761991 1762427 1762967 1763510 1763727 1764268 1765109 1765816 1766269 1766530 1766730 1766879 1766973 1768036 1768141 1770001 1770147 1770281 1770411 1770592 1770829 1771347 1772331 1772438 1773445 1773585 1773814 1774322 1774345 1774402 1774696 1774826 1775613 1777801 1777924 1777985 1778022 1778328 1778444 1778498 1778663 1780267 1780294 1780500 1780755 1781097 1781278 1782411 1782755 1782786 1783266 1783599 1783667 1784040 1784095 1784320 1784941 1784947 1785135 1785454 1785694 1785750 1786617 1786725 1786791 1786891 1786924 1787009 1787368 1787590 1787620 1788122 1788203 1788691 1788720 1788851 1789260 1789478 1789671 1789714 1790646 1790663 1790958 1791069 1791211 1794288 1794492 1794636 1795374 1796644 1796866 1799169 1799616 1799700 1800043 1800278 1802465 1803031 1805184 1805708 1807162 1808005 1808615 1809200 1809332 1810015 1810181 1810263 1811302 1814217 1814480 1814889 1814908 1815247 1816878 1816888 1819845 1820548 1822134 1822549 1822570 1822818 1823220 1823519 1823934 1825244 1825321 1825644 1826233 1826904 1827249 1827631 1828992 1829696 1829819 1829901 1829928 1830531 1830546 1830817 1831993 1832141 1832263 1832288 1832494 1832981 1833719 1833955 1835180 1835224 1837162 1839141 1839230 1839284 1839416 1839534 1840384 1840395 1840461 1841185 1841268 1841988 1842969 1842976 1843284 1843537 1843858 1844990 1845009 1845693 1846222 1846336 1846408 1848587 1849288 1850597 1850965 1851763 1853481 1853531 1854841 1855291 1855681 1857995 1859271 1859456 1860332 1861452 1862328 1862422 1862749 1862839 1863719 1864702 1865849 1866431 1866968 1867334 1867843 1867888 1868676 1869015 1869356 1869699 1870321 1870516 1870524 1870702 1870841 1871209 1871441 1872409 1872898 1873541 1875767 1876116 1877041 1877398 1877519 1877548 1879061 1879544 1881432 1881621 1881636 1881723 1882145 1882298 1882520 1882590 1882945 1883798 1884107 1884316 1884484 1884614 1884823 1885275 1885752 1885931 1886168 1886272 1886397 1886564 1886579 1886823 1886869 1887301 1887530 1887705 1888299 1888370 1889582 1889589 1890281 1890307 1891362 1891481 1891511 1891532 1891966 1894670 1894874 1895075 1896236 1896364 1896439 1897107 1898725 1898817 1898993 1899394 1899725 1900008 1900120 1901214 1901665 1901782 1903530 1904048 1904356 1904662 1908164 1908294 1908367 1908577 1908974 1908989 1909532 1909646 1909827 1909958 1910360 1910482 1910753 1910806 1911030 1911088 1911608 1912261 1912333 1912418 1912476 1912623 1912958 1912978 1913007 1913190 1913300 1913494 1913805 1913963 1914048 1914151 1914712 1914795 1915102 1915552 1916961 1917476 1917860 1918069 1918387 1918561 1918819 1919590 1920454 1920546 1921032 1921484 1921863 1922023 1922276 1923206 1923291 1924267 1924959 1925232 1925809 1928598 1929395 1932488 1933454 1934259 1934587 1935379 1936035 1936577 1936608 1936738 1937545 1938809 1939492 1939885 1941597 1942924 1943540 1943909 1945145 1945961 1947201 1947464 1947911 1948412 1951888 1953875 1954061 1955370 1955438 1955650 1956322 1956652 1957306 1957414 1958663 1960090 1960922 1960926 1961531 1961748 1962927 1963772 1966716 1967150 1967893 1968284 1969277 1969320 1969709 1970353 1971058 1971406 1971746 1972422 1972710 1973190 1973245 1974690 1975107 1975892 1977496 1979000 1979352 1980616 1982070 1982130 1984346 1985484 1988036 1988070 1988100 1988775 1989255 1990352 1991145 1991811 1991964 1992077 1992834 1993201 1993354 1993493 1994039 1994991 1996109 1996198 1996407 1996622 1999900 2000492 2002133 2002753 2003484 2003708 2004213 2004334 2004758 2006156 2008433 2008657 2009031 2009573 2009694 2010074 2010118 2010234 2011395 2011707 2012130 2013849 2013921 2014651 2016411 2016420 2016449 2017105 2018686 2019683 2020149 2020288 2021798 2022026 2023766 2024232 2024260 2025030 2025056 2025413 2025831 2026268 2026574 2027017 2027419 2027565 2027987 2028032 2028053 2028445 2028771 2028924 2029592 2029742 2029803 2029810 2029958 2030718 2031040 2031486 2031735 2033110 2033531 2033935 2034600 2034608 2034673 2035847 2036076 2036331 2036399 2036718 2036875 2037224 2037495 2037549 2037769 2037824 2037965 2038391 2038474 2039274 2039471 2039660 2039825 2040335 2040534 2040943 2041151 2041415 2042018 2042345 2042914 2043241 2043471 2043647 2044184 2044547 2044613 2044857 2045426 2045848 2046472 2046494 2047263 2047372 2047557 2047935 2048067 2048381 2048400 2048744 2048809 2049531 2049581 2050024 2050246 2050767 2051115 2051392 2051394 2051632 2051904 2052442 2052956 2053224 2053534 2053551 2054236 2054264 2054317 2054467 2054858 2055239 2055404 2055409 2055456 2056729 2056951 2057310 2057673 2057799 2057840 2057972 2058155 2058454 2058510 2058769 2058916 2059278 2059635 2059696 2059747 2060266 2060328 2060445 2060651 2060990 2061620 2061683 2061723 2061730 2062233 2062509 2062513 2062684 2062796 2063184 2063363 2063579 2064007 2064260 2064348 2065782 2065807 2066109 2066152 2066283 2067752 2067774 2068894 2069438 2069511 2070087 2071063 2072550 2073970 2074449 2074568 2075593 2075813 2076621 2077107 2077108 2078616 2079837 2080241 2080655 2081614 2083085 2083300 2084112 2084361 2084514 2084614 2084631 2086305 2087408 2087722 2087997 2088845 2089065 2089183 2089402 2089445 2089652 2090497 2091364 2092001 2094411 2094641 2095068 2096560 2097941 2098023 2099138 2099441 2099868 2100667 2101005 2101530 2101545 2102265 2102357 2102822 2103123 2103665 2103902 2104607 2105358 2106260 2106417 2107666 2108206 2109361 2109758 2110107 2110334 2111358 2111948 2112423 2112772 2113013 2113523 2113658 2113914 2115447 2115483 2115716 2116010 2116316 2116593 2117321 2118177 2118801 2119191 2120317 2120493 2120909 2122962 2123675 2124957 2125196 2125465 2125766 2126681 2127049 2127144 2127306 2127651 2127897 2127922 2128229 2128729 2129331 2129953 2130587 2130756 2131690 2132110 2132358 2132826 2133849 2134028 2134211 2134272 2136868 2137165 2139648 2140529 2142983 2143617 2143993 2145978 2147603 2148273 2148319 2148808 2148955 2149585 2152508 2152873 2153377 2153647 2153649 2154455 2154810 2154931 2156319 2156679 2157007 2158279 2158616 2159022 2159303 2159313 2159857 2159891 2159895 2160200 2160756 2161317 2162398 2162652 2162866 2163127 2163189 2163814 2164218 2164308 2165338 2165345 2165515 2166209 2166924 2167982 2168196 2170211 2170716 2171042 2171174 2171588 2171831 2172051 2172516 2172912 2173175 2173254 2173914 2174125 2174340 2174450 2174567 2176374 2176854 2177172 2177225 2177547 2177860 2177873 2178359 2178537 2178873 2179285 2180400 2181107 2181335 2182244 2182251 2182499 2183318 2183622 2183635 2183660 2184257 2184337 2184547 2184554 2184591 2184883 2185138 2185261 2185693 2185971 2186110 2186380 2186383 2186421 2187100 2187226 2187365 2188447 2188685 2188759 2188962 2188970 2188975 2189138 2189346 2189354 2189541 2189650 2189778 2190084 2190112 2191415 2191422 2192183 2192202 2192379 2192801 2193244 2193756 2193777 2194006 2194074 2194372 2194714 2194900 2195488 2195520 2196210 2197158 2197862
+21 19 20
+22 12 21 1
\ No newline at end of file
diff --git a/pregelix/pregelix-example/src/main/java/edu/uci/ics/pregelix/example/PageRankVertex.java b/pregelix/pregelix-example/src/main/java/edu/uci/ics/pregelix/example/PageRankVertex.java
index bc4adc6..8ba4472 100644
--- a/pregelix/pregelix-example/src/main/java/edu/uci/ics/pregelix/example/PageRankVertex.java
+++ b/pregelix/pregelix-example/src/main/java/edu/uci/ics/pregelix/example/PageRankVertex.java
@@ -3,9 +3,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * you may obtain a copy of the License from
- * 
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -54,8 +54,8 @@
 public class PageRankVertex extends Vertex<VLongWritable, DoubleWritable, FloatWritable, DoubleWritable> {
 
     public static final String ITERATIONS = "HyracksPageRankVertex.iteration";
-    private DoubleWritable outputValue = new DoubleWritable();
-    private DoubleWritable tmpVertexValue = new DoubleWritable();
+    private final DoubleWritable outputValue = new DoubleWritable();
+    private final DoubleWritable tmpVertexValue = new DoubleWritable();
     private int maxIteration = -1;
 
     /**
@@ -63,7 +63,7 @@
      */
     public static class SimpleSumCombiner extends MessageCombiner<VLongWritable, DoubleWritable, DoubleWritable> {
         private double sum = 0.0;
-        private DoubleWritable agg = new DoubleWritable();
+        private final DoubleWritable agg = new DoubleWritable();
         private MsgList<DoubleWritable> msgList;
 
         @SuppressWarnings({ "rawtypes", "unchecked" })
@@ -114,7 +114,7 @@
             return agg;
         }
     }
-    
+
     @Override
     public void configure(Configuration conf){
         maxIteration = conf.getInt(ITERATIONS, 10);
@@ -131,7 +131,7 @@
             while (msgIterator.hasNext()) {
                 sum += msgIterator.next().get();
             }
-            tmpVertexValue.set((0.15 / getNumVertices()) + 0.85 * sum);
+            tmpVertexValue.set(0.15 / getNumVertices() + 0.85 * sum);
             setVertexValue(tmpVertexValue);
         }
 
@@ -151,7 +151,7 @@
             GeneratedVertexReader<VLongWritable, DoubleWritable, FloatWritable, DoubleWritable> {
         /** Class logger */
         private static final Logger LOG = Logger.getLogger(SimulatedPageRankVertexReader.class.getName());
-        private Map<VLongWritable, FloatWritable> edges = Maps.newHashMap();
+        private final Map<VLongWritable, FloatWritable> edges = Maps.newHashMap();
 
         public SimulatedPageRankVertexReader() {
             super();
@@ -168,7 +168,7 @@
             Vertex<VLongWritable, DoubleWritable, FloatWritable, DoubleWritable> vertex = BspUtils
                     .createVertex(configuration);
 
-            VLongWritable vertexId = new VLongWritable((inputSplit.getSplitIndex() * totalRecords) + recordsRead);
+            VLongWritable vertexId = new VLongWritable(inputSplit.getSplitIndex() * totalRecords + recordsRead);
             DoubleWritable vertexValue = new DoubleWritable(vertexId.get() * 10d);
             long destVertexId = (vertexId.get() + 1) % (inputSplit.getNumSplits() * totalRecords);
             float edgeValue = vertexId.get() * 100f;
diff --git a/pregelix/pregelix-example/src/test/java/edu/uci/ics/pregelix/example/SkewTest.java b/pregelix/pregelix-example/src/test/java/edu/uci/ics/pregelix/example/SkewTest.java
new file mode 100644
index 0000000..4024ec6
--- /dev/null
+++ b/pregelix/pregelix-example/src/test/java/edu/uci/ics/pregelix/example/SkewTest.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.pregelix.example;
+
+import java.io.File;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.junit.Test;
+
+import edu.uci.ics.pregelix.api.job.PregelixJob;
+import edu.uci.ics.pregelix.api.util.ConservativeCheckpointHook;
+import edu.uci.ics.pregelix.core.driver.Driver;
+import edu.uci.ics.pregelix.core.util.PregelixHyracksIntegrationUtil;
+import edu.uci.ics.pregelix.example.PageRankVertex.SimplePageRankVertexOutputFormat;
+import edu.uci.ics.pregelix.example.data.VLongNormalizedKeyComputer;
+import edu.uci.ics.pregelix.example.inputformat.TextPageRankInputFormat;
+import edu.uci.ics.pregelix.example.util.TestCluster;
+import edu.uci.ics.pregelix.example.util.TestUtils;
+
+/**
+ * This test case tests big vertex, using PageRankVertex.
+ *
+ * @author yingyib
+ */
+public class SkewTest {
+    private static String INPUTPATH = "data/skew";
+    private static String OUTPUTPAH = "actual/result";
+    private static String EXPECTEDPATH = "src/test/resources/expected/skew";
+
+    @Test
+    public void test() throws Exception {
+        TestCluster testCluster = new TestCluster();
+        try {
+            PregelixJob job = new PregelixJob(PageRankVertex.class.getName());
+            job.setVertexClass(PageRankVertex.class);
+            job.setVertexInputFormatClass(TextPageRankInputFormat.class);
+            job.setVertexOutputFormatClass(SimplePageRankVertexOutputFormat.class);
+            job.setMessageCombinerClass(PageRankVertex.SimpleSumCombiner.class);
+            job.setNoramlizedKeyComputerClass(VLongNormalizedKeyComputer.class);
+            FileInputFormat.setInputPaths(job, INPUTPATH);
+            FileOutputFormat.setOutputPath(job, new Path(OUTPUTPAH));
+            job.getConfiguration().setLong(PregelixJob.NUM_VERTICE, 21);
+            job.setCheckpointHook(ConservativeCheckpointHook.class);
+            job.setFixedVertexValueSize(true);
+            job.setFrameSize(16384);
+
+            testCluster.setUp();
+            Driver driver = new Driver(PageRankVertex.class);
+            driver.runJob(job, "127.0.0.1", PregelixHyracksIntegrationUtil.TEST_HYRACKS_CC_CLIENT_PORT);
+            TestUtils.compareWithResultDir(new File(EXPECTEDPATH), new File(OUTPUTPAH));
+        } catch (Exception e) {
+            throw e;
+        } finally {
+            testCluster.tearDown();
+        }
+    }
+}
diff --git a/pregelix/pregelix-example/src/test/resources/cluster/cluster.properties b/pregelix/pregelix-example/src/test/resources/cluster/cluster.properties
index baa8a24..c75e65c 100644
--- a/pregelix/pregelix-example/src/test/resources/cluster/cluster.properties
+++ b/pregelix/pregelix-example/src/test/resources/cluster/cluster.properties
@@ -42,6 +42,9 @@
 #The frame size of the internal dataflow engine
 FRAME_SIZE=65536
 
+#The frame size of the internal vertex storage
+VFRAME_SIZE=65536
+
 #CC JAVA_OPTS
 CCJAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,address=7001,server=y,suspend=n -Xmx3g -Djava.util.logging.config.file=logging.properties"
 # Yourkit option: -agentpath:/grid/0/dev/vborkar/tools/yjp-10.0.4/bin/linux-x86-64/libyjpagent.so=port=20001"
diff --git a/pregelix/pregelix-example/src/test/resources/expected/skew/part-0 b/pregelix/pregelix-example/src/test/resources/expected/skew/part-0
new file mode 100755
index 0000000..c2c0cc0
--- /dev/null
+++ b/pregelix/pregelix-example/src/test/resources/expected/skew/part-0
@@ -0,0 +1,6 @@
+1	0.1538057869748262
+5	0.021540394083972653
+9	0.008851770565007962
+13	0.08308297649373161
+17	0.012504121497877515
+21	0.009166666666666667
diff --git a/pregelix/pregelix-example/src/test/resources/expected/skew/part-1 b/pregelix/pregelix-example/src/test/resources/expected/skew/part-1
new file mode 100755
index 0000000..5a792e7
--- /dev/null
+++ b/pregelix/pregelix-example/src/test/resources/expected/skew/part-1
@@ -0,0 +1,6 @@
+2	0.14475932277139475
+6	0.015003751200304748
+10	0.0078953714534803
+14	0.040205598595470717
+18	0.010592752407987198
+22	0.007142857142857143
diff --git a/pregelix/pregelix-example/src/test/resources/expected/skew/part-2 b/pregelix/pregelix-example/src/test/resources/expected/skew/part-2
new file mode 100755
index 0000000..c21220f
--- /dev/null
+++ b/pregelix/pregelix-example/src/test/resources/expected/skew/part-2
@@ -0,0 +1,5 @@
+3	0.07941505736441216
+7	0.011947009713572225
+11	0.1561028584551254
+15	0.022550744820502584
+19	0.013158324980956872
diff --git a/pregelix/pregelix-example/src/test/resources/expected/skew/part-3 b/pregelix/pregelix-example/src/test/resources/expected/skew/part-3
new file mode 100755
index 0000000..39a8af1
--- /dev/null
+++ b/pregelix/pregelix-example/src/test/resources/expected/skew/part-3
@@ -0,0 +1,6 @@
+0	0.008261482871440377
+4	0.038415704685818675
+8	0.010122315837328184
+12	0.15141534420136182
+16	0.015706612612729605
+412454	0.007142857142857143