diff --git a/asterix-app/data/hdfs/external-indexing-test.rc b/asterix-app/data/hdfs/external-indexing-test.rc
new file mode 100644
index 0000000..4801881
--- /dev/null
+++ b/asterix-app/data/hdfs/external-indexing-test.rc
Binary files differ
diff --git a/asterix-app/data/hdfs/external-indexing-test.seq b/asterix-app/data/hdfs/external-indexing-test.seq
new file mode 100644
index 0000000..0146c83
--- /dev/null
+++ b/asterix-app/data/hdfs/external-indexing-test.seq
Binary files differ
diff --git a/asterix-app/data/hdfs/external-indexing-test.txt b/asterix-app/data/hdfs/external-indexing-test.txt
new file mode 100644
index 0000000..555272e
--- /dev/null
+++ b/asterix-app/data/hdfs/external-indexing-test.txt
@@ -0,0 +1,11 @@
+1|Steve|50
+2|John|23
+3|Samuel|22
+4|Mary|29
+5|William|75
+6|Sarah|16
+7|Noel|33
+8|Carlos|40
+9|Joseph|45
+10|David|22
+11|Nadine|10
diff --git a/asterix-app/data/hdfs/spatialData.json b/asterix-app/data/hdfs/spatialData.json
new file mode 100644
index 0000000..9c78064
--- /dev/null
+++ b/asterix-app/data/hdfs/spatialData.json
@@ -0,0 +1,21 @@
+{"id": 1, "point": point("4.1,7.0"), "kwds": "sign ahead", "line1": line("4.0,7.0 9.0,7.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.2,7.0"), "circle" : circle("1.0,1.0 10.0")}
+{"id": 2, "point": point("40.2152,-75.0449"), "kwds": "factory hosedan", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0")}
+{"id": 3, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("3.0,0.0 0.0,4.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("5.5,1.0 10.0")}
+{"id": 4, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0 2.0,1.0 1.0,0.0"), "poly2": polygon("2.0,1.0 2.0,2.0 3.0,2.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("77.0,4.0 30.0")}
+{"id": 5, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("100.0,100.0 100.0,400.0 300.0,400.0 300.0,100.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("88.0,1.0 10.0")}
+{"id": 6, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 1.0,7.0"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("3.1,1.0 2.9,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,1.0 10.0")}
+{"id": 7, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 4.0,7.1"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("3.0,1.0 3.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("3.0,6.0 5.0,7.0"), "circle" : circle("13.0,75.0 1.0")}
+{"id": 8, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("-3.0,-3.0 -1.0,-3.0 -3.0,-5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("76.0,87.0 50.0")}
+{"id": 9, "point": point("5.0,1.0"), "kwds": "sign ahead", "line1": line("5.0,1.0 5.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("11.0,14.0 15.0")}
+{"id": 10, "point": point("2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("6.01,1.0 6.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,76.0 17.0")}
+{"id": 11, "point": point("4.9,0.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("4.9,0.1 4.9,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("22.0,35.0 144.0")}
+{"id": 12, "point": point("6.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("4.0,1.0 4.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,23.0 12.0")}
+{"id": 13, "point": point("5.0,5.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("6.0,1.0 6.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("30.0,11.0 11.0")}
+{"id": 14, "point": point("5.1,5.1"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 5.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,66.0 17.0")}
+{"id": 15, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.1,1.0 5.1,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("12.0,87.0 10.0")}
+{"id": 16, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,35.0 10.0")}
+{"id": 17, "point": point("4.1,7.0"), "kwds": "sign ahead", "line1": line("4.0,7.0 9.0,7.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("0.0,6.0 0.0,0.0 3.0,0.0 4.0,1.0 6.0,1.0 8.0,0.0 12.0,0.0 13.0,2.0 8.0,2.0 8.0,4.0 11.0,4.0 11.0,6.0 6.0,6.0 4.0,3.0 2.0,6.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,51.0 10.0")}
+{"id": 18, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 7.0,1.0 7.0,4.0 6.0,2.0 5.0,4.0"), "poly2": polygon("6.0,3.0 7.0,5.0 6.0,7.0 5.0,5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("43.0,45.0 12.0")}
+{"id": 19, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 7.0,1.0 7.0,4.0 6.0,2.0 5.0,4.0"), "poly2": polygon("6.0,1.0 7.0,5.0 6.0,7.0 5.0,5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("65.0,2.0 13.0")}
+{"id": 20, "point": point("4.0,3.0"), "kwds": "sign ahead", "line1": line("20.0,20.0 30.0,40.0"), "line2": line("5.0,8.0 0.0,1.0"), "poly1": polygon("4.0,1.0 4.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("50.0,10.0 50.0,40.0 70.0,40.0 70.0,10.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,23.0 12.0")}
+{"id": 21, "point": point("0.0,5.0"), "kwds": "sign ahead", "line1": line("0.0,5.0 0.0,40.0"), "line2": line("5.0,8.0 0.0,1.0"), "poly1": polygon("5.1,5.1 14.0,14.0 22.0,14.0 22.0,10.0"), "poly2": polygon("50.0,10.0 50.0,40.0 70.0,40.0 70.0,10.0"), "rec": rectangle("0.0,0.0 5.1,5.1"), "circle" : circle("1.0,23.0 12.0")}
\ No newline at end of file
diff --git a/asterix-app/data/hdfs/tw_for_indexleftouterjoin.adm b/asterix-app/data/hdfs/tw_for_indexleftouterjoin.adm
new file mode 100644
index 0000000..405dd08
--- /dev/null
+++ b/asterix-app/data/hdfs/tw_for_indexleftouterjoin.adm
@@ -0,0 +1,250 @@
+{ "tweetid": 1i64, "user": { "screen-name": "WardLoewentsein@340", "lang": "en", "friends-count": 11, "statuses-count": 388, "name": "Ward Loewentsein", "followers-count": 129 }, "sender-location": point("42.83,72.44"), "send-time": datetime("2009-10-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile the speed is mind-blowing:)", "countA": 1, "countB": 26 }
+{ "tweetid": 2i64, "user": { "screen-name": "KyleGraham_120", "lang": "en", "friends-count": 55, "statuses-count": 231, "name": "Kyle Graham", "followers-count": 42 }, "sender-location": point("34.81,72.44"), "send-time": datetime("2011-09-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " hate samsung the 3G is horrible", "countA": 2, "countB": 131 }
+{ "tweetid": 3i64, "user": { "screen-name": "TateGarneys@542", "lang": "en", "friends-count": 74, "statuses-count": 370, "name": "Tate Garneys", "followers-count": 111 }, "sender-location": point("24.54,82.66"), "send-time": datetime("2009-12-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " hate iphone the shortcut-menu is bad", "countA": 3, "countB": 187 }
+{ "tweetid": 4i64, "user": { "screen-name": "BuckFields@708", "lang": "en", "friends-count": 20, "statuses-count": 469, "name": "Buck Fields", "followers-count": 181 }, "sender-location": point("38.14,68.1"), "send-time": datetime("2008-10-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " dislike samsung the speed is OMG", "countA": 4, "countB": 52 }
+{ "tweetid": 5i64, "user": { "screen-name": "NoreenBaldwin_373", "lang": "en", "friends-count": 89, "statuses-count": 144, "name": "Noreen Baldwin", "followers-count": 187 }, "sender-location": point("35.4,68.89"), "send-time": datetime("2008-10-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " hate motorola its 3G is OMG:(", "countA": 5, "countB": 35 }
+{ "tweetid": 6i64, "user": { "screen-name": "IselaHatcher_237", "lang": "en", "friends-count": 85, "statuses-count": 333, "name": "Isela Hatcher", "followers-count": 148 }, "sender-location": point("42.75,78.5"), "send-time": datetime("2011-10-15T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " hate sprint the wireless is terrible:(", "countA": 6, "countB": 61 }
+{ "tweetid": 7i64, "user": { "screen-name": "NicolaJolce$660", "lang": "en", "friends-count": 45, "statuses-count": 420, "name": "Nicola Jolce", "followers-count": 12 }, "sender-location": point("48.16,71.59"), "send-time": datetime("2005-11-23T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-command" }}, "message-text": " like motorola its voice-command is amazing", "countA": 7, "countB": 47 }
+{ "tweetid": 8i64, "user": { "screen-name": "MorganKeppel_176", "lang": "en", "friends-count": 74, "statuses-count": 190, "name": "Morgan Keppel", "followers-count": 2 }, "sender-location": point("36.17,72.56"), "send-time": datetime("2011-12-02T10:10:00.000Z"), "referred-topics": {{ "verizon", "3G" }}, "message-text": " hate verizon the 3G is OMG:(", "countA": 8, "countB": 98 }
+{ "tweetid": 9i64, "user": { "screen-name": "GerardMcdonald$43", "lang": "en", "friends-count": 72, "statuses-count": 151, "name": "Gerard Mcdonald", "followers-count": 96 }, "sender-location": point("38.02,70.38"), "send-time": datetime("2005-10-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " love sprint its voice-clarity is amazing", "countA": 9, "countB": 69 }
+{ "tweetid": 10i64, "user": { "screen-name": "WynonnaButler_286", "lang": "en", "friends-count": 30, "statuses-count": 375, "name": "Wynonna Butler", "followers-count": 78 }, "sender-location": point("38.71,90.05"), "send-time": datetime("2008-09-21T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " love motorola its wireless is good:)", "countA": 10, "countB": 75 }
+{ "tweetid": 11i64, "user": { "screen-name": "BrodyKing@977", "lang": "en", "friends-count": 3, "statuses-count": 62, "name": "Brody King", "followers-count": 106 }, "sender-location": point("32.26,73.48"), "send-time": datetime("2007-05-20T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " can't stand sprint the shortcut-menu is OMG", "countA": 11, "countB": 28 }
+{ "tweetid": 12i64, "user": { "screen-name": "ValentineSchofield@448", "lang": "en", "friends-count": 16, "statuses-count": 260, "name": "Valentine Schofield", "followers-count": 136 }, "sender-location": point("24.99,70.66"), "send-time": datetime("2009-07-26T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " can't stand sprint its shortcut-menu is terrible", "countA": 12, "countB": 159 }
+{ "tweetid": 13i64, "user": { "screen-name": "MaryroseBennett#483", "lang": "en", "friends-count": 99, "statuses-count": 496, "name": "Maryrose Bennett", "followers-count": 57 }, "sender-location": point("38.94,93.31"), "send-time": datetime("2005-01-06T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " love at&t its platform is awesome", "countA": 13, "countB": 13 }
+{ "tweetid": 14i64, "user": { "screen-name": "GarlandAlliman$490", "lang": "en", "friends-count": 23, "statuses-count": 146, "name": "Garland Alliman", "followers-count": 51 }, "sender-location": point("29.96,75.0"), "send-time": datetime("2009-09-26T10:10:00.000Z"), "referred-topics": {{ "motorola", "platform" }}, "message-text": " dislike motorola the platform is bad:(", "countA": 14, "countB": 74 }
+{ "tweetid": 15i64, "user": { "screen-name": "AnnabelPirl$171", "lang": "en", "friends-count": 43, "statuses-count": 402, "name": "Annabel Pirl", "followers-count": 137 }, "sender-location": point("27.22,86.32"), "send-time": datetime("2008-09-08T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " like at&t the network is amazing:)", "countA": 15, "countB": 19 }
+{ "tweetid": 16i64, "user": { "screen-name": "JarvisPickering@42", "lang": "en", "friends-count": 98, "statuses-count": 498, "name": "Jarvis Pickering", "followers-count": 20 }, "sender-location": point("30.54,81.6"), "send-time": datetime("2005-11-05T10:10:00.000Z"), "referred-topics": {{ "iphone", "wireless" }}, "message-text": " love iphone the wireless is awesome", "countA": 16, "countB": 27 }
+{ "tweetid": 17i64, "user": { "screen-name": "LillyHoffhants@595", "lang": "en", "friends-count": 35, "statuses-count": 391, "name": "Lilly Hoffhants", "followers-count": 129 }, "sender-location": point("47.96,95.21"), "send-time": datetime("2007-02-14T10:10:00.000Z"), "referred-topics": {{ "verizon", "customer-service" }}, "message-text": " like verizon its customer-service is amazing", "countA": 17, "countB": 55 }
+{ "tweetid": 18i64, "user": { "screen-name": "AllanPolson_455", "lang": "en", "friends-count": 36, "statuses-count": 227, "name": "Allan Polson", "followers-count": 113 }, "sender-location": point("46.39,73.85"), "send-time": datetime("2009-12-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " hate iphone the platform is bad", "countA": 18, "countB": 199 }
+{ "tweetid": 19i64, "user": { "screen-name": "DonnieWentzel#857", "lang": "en", "friends-count": 76, "statuses-count": 23, "name": "Donnie Wentzel", "followers-count": 78 }, "sender-location": point("37.21,95.76"), "send-time": datetime("2012-04-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "reachability" }}, "message-text": " like iphone the reachability is mind-blowing", "countA": 19, "countB": 68 }
+{ "tweetid": 20i64, "user": { "screen-name": "TraversFaast@428", "lang": "en", "friends-count": 42, "statuses-count": 70, "name": "Travers Faast", "followers-count": 116 }, "sender-location": point("26.54,74.71"), "send-time": datetime("2010-06-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone the shortcut-menu is mind-blowing", "countA": 20, "countB": 18 }
+{ "tweetid": 21i64, "user": { "screen-name": "KameronSandford#555", "lang": "en", "friends-count": 22, "statuses-count": 104, "name": "Kameron Sandford", "followers-count": 17 }, "sender-location": point("25.8,88.76"), "send-time": datetime("2007-11-20T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " can't stand at&t the network is bad:(", "countA": 21, "countB": 64 }
+{ "tweetid": 22i64, "user": { "screen-name": "TiannaArmitage_372", "lang": "en", "friends-count": 32, "statuses-count": 307, "name": "Tianna Armitage", "followers-count": 126 }, "sender-location": point("45.79,80.75"), "send-time": datetime("2005-04-14T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola its reachability is terrible:(", "countA": 22, "countB": 126 }
+{ "tweetid": 23i64, "user": { "screen-name": "NevadaCattley#858", "lang": "en", "friends-count": 74, "statuses-count": 389, "name": "Nevada Cattley", "followers-count": 17 }, "sender-location": point("43.6,93.24"), "send-time": datetime("2007-03-11T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " hate at&t the network is horrible", "countA": 23, "countB": 74 }
+{ "tweetid": 24i64, "user": { "screen-name": "ReannonEisenhart#637", "lang": "en", "friends-count": 34, "statuses-count": 235, "name": "Reannon Eisenhart", "followers-count": 145 }, "sender-location": point("47.56,76.49"), "send-time": datetime("2012-03-20T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-clarity" }}, "message-text": " can't stand samsung its voice-clarity is terrible:(", "countA": 24, "countB": 70 }
+{ "tweetid": 25i64, "user": { "screen-name": "LillyMang#928", "lang": "en", "friends-count": 20, "statuses-count": 40, "name": "Lilly Mang", "followers-count": 47 }, "sender-location": point("38.68,94.93"), "send-time": datetime("2006-04-21T10:10:00.000Z"), "referred-topics": {{ "sprint", "customization" }}, "message-text": " love sprint its customization is awesome", "countA": 25, "countB": 86 }
+{ "tweetid": 26i64, "user": { "screen-name": "MicaBusk$903", "lang": "en", "friends-count": 87, "statuses-count": 164, "name": "Mica Busk", "followers-count": 92 }, "sender-location": point("45.47,90.97"), "send-time": datetime("2008-01-24T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " hate motorola the touch-screen is terrible", "countA": 26, "countB": 11 }
+{ "tweetid": 27i64, "user": { "screen-name": "PiaHildyard_915", "lang": "en", "friends-count": 92, "statuses-count": 302, "name": "Pia Hildyard", "followers-count": 16 }, "sender-location": point("43.76,68.58"), "send-time": datetime("2007-08-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " dislike iphone its platform is bad:(", "countA": 27, "countB": 125 }
+{ "tweetid": 28i64, "user": { "screen-name": "CamelliaSiegrist_676", "lang": "en", "friends-count": 73, "statuses-count": 392, "name": "Camellia Siegrist", "followers-count": 193 }, "sender-location": point("24.94,77.95"), "send-time": datetime("2007-12-26T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-command" }}, "message-text": " can't stand verizon its voice-command is bad", "countA": 28, "countB": 123 }
+{ "tweetid": 29i64, "user": { "screen-name": "BurtTaggart_922", "lang": "en", "friends-count": 49, "statuses-count": 62, "name": "Burt Taggart", "followers-count": 134 }, "sender-location": point("35.67,97.43"), "send-time": datetime("2011-07-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is awesome", "countA": 29, "countB": 115 }
+{ "tweetid": 30i64, "user": { "screen-name": "MarlaHill@215", "lang": "en", "friends-count": 84, "statuses-count": 305, "name": "Marla Hill", "followers-count": 71 }, "sender-location": point("24.29,84.28"), "send-time": datetime("2012-07-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "3G" }}, "message-text": " like t-mobile the 3G is awesome", "countA": 30, "countB": 36 }
+{ "tweetid": 31i64, "user": { "screen-name": "CaitlynChristman@452", "lang": "en", "friends-count": 57, "statuses-count": 414, "name": "Caitlyn Christman", "followers-count": 67 }, "sender-location": point("41.04,85.13"), "send-time": datetime("2005-11-05T10:10:00.000Z"), "referred-topics": {{ "samsung", "voicemail-service" }}, "message-text": " love samsung the voicemail-service is mind-blowing:)", "countA": 31, "countB": 59 }
+{ "tweetid": 32i64, "user": { "screen-name": "BraxtonBonner#527", "lang": "en", "friends-count": 21, "statuses-count": 427, "name": "Braxton Bonner", "followers-count": 168 }, "sender-location": point("34.25,86.09"), "send-time": datetime("2011-10-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " dislike motorola its 3G is horrible:(", "countA": 32, "countB": 38 }
+{ "tweetid": 33i64, "user": { "screen-name": "WilmaSouthern@238", "lang": "en", "friends-count": 83, "statuses-count": 413, "name": "Wilma Southern", "followers-count": 24 }, "sender-location": point("34.71,69.57"), "send-time": datetime("2005-02-19T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " dislike sprint the wireless is OMG:(", "countA": 33, "countB": 105 }
+{ "tweetid": 34i64, "user": { "screen-name": "MaxNash$802", "lang": "en", "friends-count": 13, "statuses-count": 189, "name": "Max Nash", "followers-count": 39 }, "sender-location": point("48.12,89.23"), "send-time": datetime("2012-02-17T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " dislike motorola its touch-screen is horrible", "countA": 34, "countB": 185 }
+{ "tweetid": 35i64, "user": { "screen-name": "HannahWarrick_843", "lang": "en", "friends-count": 14, "statuses-count": 10, "name": "Hannah Warrick", "followers-count": 2 }, "sender-location": point("32.75,69.94"), "send-time": datetime("2007-09-14T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "plan" }}, "message-text": " like t-mobile the plan is amazing:)", "countA": 35, "countB": 176 }
+{ "tweetid": 36i64, "user": { "screen-name": "SherikaBarth#732", "lang": "en", "friends-count": 80, "statuses-count": 277, "name": "Sherika Barth", "followers-count": 138 }, "sender-location": point("34.85,66.87"), "send-time": datetime("2011-10-13T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " can't stand at&t the plan is OMG", "countA": 36, "countB": 147 }
+{ "tweetid": 37i64, "user": { "screen-name": "SabinaCattley$355", "lang": "en", "friends-count": 67, "statuses-count": 20, "name": "Sabina Cattley", "followers-count": 104 }, "sender-location": point("40.22,71.18"), "send-time": datetime("2007-04-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-clarity" }}, "message-text": " can't stand samsung its voice-clarity is bad:(", "countA": 37, "countB": 55 }
+{ "tweetid": 38i64, "user": { "screen-name": "KimberlyVeith$848", "lang": "en", "friends-count": 43, "statuses-count": 274, "name": "Kimberly Veith", "followers-count": 14 }, "sender-location": point("34.94,83.17"), "send-time": datetime("2011-07-15T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " hate samsung the speed is horrible:(", "countA": 38, "countB": 89 }
+{ "tweetid": 39i64, "user": { "screen-name": "AdrianneMackendoerfer_478", "lang": "en", "friends-count": 2, "statuses-count": 125, "name": "Adrianne Mackendoerfer", "followers-count": 113 }, "sender-location": point("40.14,78.49"), "send-time": datetime("2010-10-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-command" }}, "message-text": " hate motorola its voice-command is OMG", "countA": 39, "countB": 97 }
+{ "tweetid": 40i64, "user": { "screen-name": "MunroWire@995", "lang": "en", "friends-count": 89, "statuses-count": 336, "name": "Munro Wire", "followers-count": 181 }, "sender-location": point("30.94,80.83"), "send-time": datetime("2009-05-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " love verizon the network is good", "countA": 40, "countB": 193 }
+{ "tweetid": 41i64, "user": { "screen-name": "AmadaAft@648", "lang": "en", "friends-count": 50, "statuses-count": 127, "name": "Amada Aft", "followers-count": 20 }, "sender-location": point("32.88,81.46"), "send-time": datetime("2010-04-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " can't stand iphone its customer-service is OMG:(", "countA": 41, "countB": 169 }
+{ "tweetid": 42i64, "user": { "screen-name": "SalenaMcfall_717", "lang": "en", "friends-count": 30, "statuses-count": 93, "name": "Salena Mcfall", "followers-count": 184 }, "sender-location": point("47.86,71.93"), "send-time": datetime("2010-02-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "network" }}, "message-text": " can't stand t-mobile its network is bad", "countA": 42, "countB": 130 }
+{ "tweetid": 43i64, "user": { "screen-name": "JeniferCanham$317", "lang": "en", "friends-count": 63, "statuses-count": 344, "name": "Jenifer Canham", "followers-count": 132 }, "sender-location": point("25.68,81.87"), "send-time": datetime("2010-04-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " hate motorola the 3G is OMG:(", "countA": 43, "countB": 153 }
+{ "tweetid": 44i64, "user": { "screen-name": "NannieBender$656", "lang": "en", "friends-count": 26, "statuses-count": 84, "name": "Nannie Bender", "followers-count": 184 }, "sender-location": point("47.46,85.04"), "send-time": datetime("2007-06-08T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " dislike samsung its voice-command is bad", "countA": 44, "countB": 12 }
+{ "tweetid": 45i64, "user": { "screen-name": "ThaoKooser@875", "lang": "en", "friends-count": 60, "statuses-count": 289, "name": "Thao Kooser", "followers-count": 8 }, "sender-location": point("37.02,87.94"), "send-time": datetime("2005-11-28T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " like verizon its network is amazing:)", "countA": 45, "countB": 151 }
+{ "tweetid": 46i64, "user": { "screen-name": "AugustaBaumgartner_385", "lang": "en", "friends-count": 17, "statuses-count": 70, "name": "Augusta Baumgartner", "followers-count": 162 }, "sender-location": point("24.83,73.16"), "send-time": datetime("2008-09-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " like verizon its network is mind-blowing", "countA": 46, "countB": 37 }
+{ "tweetid": 47i64, "user": { "screen-name": "OtisHill_124", "lang": "en", "friends-count": 46, "statuses-count": 68, "name": "Otis Hill", "followers-count": 29 }, "sender-location": point("36.01,86.76"), "send-time": datetime("2011-05-16T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " dislike t-mobile the touch-screen is horrible:(", "countA": 47, "countB": 27 }
+{ "tweetid": 48i64, "user": { "screen-name": "ZolaJudge@572", "lang": "en", "friends-count": 8, "statuses-count": 39, "name": "Zola Judge", "followers-count": 36 }, "sender-location": point("42.67,91.8"), "send-time": datetime("2009-03-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " like motorola the network is awesome", "countA": 48, "countB": 16 }
+{ "tweetid": 49i64, "user": { "screen-name": "ChristianaWeisgarber$35", "lang": "en", "friends-count": 40, "statuses-count": 427, "name": "Christiana Weisgarber", "followers-count": 20 }, "sender-location": point("36.91,86.0"), "send-time": datetime("2010-04-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " hate sprint its plan is terrible", "countA": 49, "countB": 28 }
+{ "tweetid": 50i64, "user": { "screen-name": "MollyGarneis_210", "lang": "en", "friends-count": 6, "statuses-count": 453, "name": "Molly Garneis", "followers-count": 185 }, "sender-location": point("44.42,87.86"), "send-time": datetime("2012-04-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " love t-mobile its wireless is good:)", "countA": 50, "countB": 123 }
+{ "tweetid": 51i64, "user": { "screen-name": "HarlanHanseu#420", "lang": "en", "friends-count": 66, "statuses-count": 295, "name": "Harlan Hanseu", "followers-count": 99 }, "sender-location": point("37.65,70.54"), "send-time": datetime("2006-05-11T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " can't stand samsung the speed is terrible:(", "countA": 51, "countB": 94 }
+{ "tweetid": 52i64, "user": { "screen-name": "DelorseSloan#229", "lang": "en", "friends-count": 84, "statuses-count": 287, "name": "Delorse Sloan", "followers-count": 20 }, "sender-location": point("27.12,78.69"), "send-time": datetime("2011-02-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " dislike motorola its touch-screen is horrible:(", "countA": 52, "countB": 156 }
+{ "tweetid": 53i64, "user": { "screen-name": "MylesEwing@54", "lang": "en", "friends-count": 45, "statuses-count": 411, "name": "Myles Ewing", "followers-count": 23 }, "sender-location": point("25.82,97.9"), "send-time": datetime("2007-11-10T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " like at&t its network is mind-blowing:)", "countA": 53, "countB": 174 }
+{ "tweetid": 54i64, "user": { "screen-name": "OprahClark_160", "lang": "en", "friends-count": 26, "statuses-count": 299, "name": "Oprah Clark", "followers-count": 161 }, "sender-location": point("36.1,87.24"), "send-time": datetime("2010-04-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile its shortcut-menu is good", "countA": 54, "countB": 128 }
+{ "tweetid": 55i64, "user": { "screen-name": "CoreyRichards#130", "lang": "en", "friends-count": 45, "statuses-count": 420, "name": "Corey Richards", "followers-count": 102 }, "sender-location": point("42.77,72.16"), "send-time": datetime("2012-07-20T10:10:00.000Z"), "referred-topics": {{ "at&t", "customer-service" }}, "message-text": " can't stand at&t its customer-service is horrible", "countA": 55, "countB": 30 }
+{ "tweetid": 56i64, "user": { "screen-name": "GwendolenHahn#673", "lang": "en", "friends-count": 62, "statuses-count": 426, "name": "Gwendolen Hahn", "followers-count": 158 }, "sender-location": point("39.76,90.94"), "send-time": datetime("2009-05-10T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " dislike samsung its 3G is bad:(", "countA": 56, "countB": 50 }
+{ "tweetid": 57i64, "user": { "screen-name": "DewayneBallou@258", "lang": "en", "friends-count": 42, "statuses-count": 215, "name": "Dewayne Ballou", "followers-count": 85 }, "sender-location": point("28.45,75.02"), "send-time": datetime("2006-08-07T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile the speed is mind-blowing:)", "countA": 57, "countB": 61 }
+{ "tweetid": 58i64, "user": { "screen-name": "RenayReese@543", "lang": "en", "friends-count": 80, "statuses-count": 459, "name": "Renay Reese", "followers-count": 102 }, "sender-location": point("38.09,77.66"), "send-time": datetime("2007-12-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " love motorola the speed is awesome:)", "countA": 58, "countB": 35 }
+{ "tweetid": 59i64, "user": { "screen-name": "SamuelHoffhants@740", "lang": "en", "friends-count": 53, "statuses-count": 64, "name": "Samuel Hoffhants", "followers-count": 150 }, "sender-location": point("48.86,97.19"), "send-time": datetime("2009-11-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " dislike sprint its speed is horrible", "countA": 59, "countB": 95 }
+{ "tweetid": 60i64, "user": { "screen-name": "GarlandOneal@886", "lang": "en", "friends-count": 12, "statuses-count": 481, "name": "Garland Oneal", "followers-count": 13 }, "sender-location": point("37.6,75.74"), "send-time": datetime("2005-05-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint its touch-screen is OMG:(", "countA": 60, "countB": 2 }
+{ "tweetid": 61i64, "user": { "screen-name": "LukeHoopengarner@327", "lang": "en", "friends-count": 77, "statuses-count": 224, "name": "Luke Hoopengarner", "followers-count": 107 }, "sender-location": point("46.38,80.88"), "send-time": datetime("2006-11-13T10:10:00.000Z"), "referred-topics": {{ "sprint", "voicemail-service" }}, "message-text": " can't stand sprint its voicemail-service is bad:(", "countA": 61, "countB": 12 }
+{ "tweetid": 62i64, "user": { "screen-name": "AudieStahl@296", "lang": "en", "friends-count": 89, "statuses-count": 90, "name": "Audie Stahl", "followers-count": 74 }, "sender-location": point("32.48,96.01"), "send-time": datetime("2010-04-23T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t its platform is terrible:(", "countA": 62, "countB": 47 }
+{ "tweetid": 63i64, "user": { "screen-name": "ArielleErrett_963", "lang": "en", "friends-count": 15, "statuses-count": 385, "name": "Arielle Errett", "followers-count": 34 }, "sender-location": point("25.39,82.02"), "send-time": datetime("2012-02-20T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " love verizon its signal is awesome:)", "countA": 63, "countB": 149 }
+{ "tweetid": 64i64, "user": { "screen-name": "MiltonWeldi#571", "lang": "en", "friends-count": 72, "statuses-count": 236, "name": "Milton Weldi", "followers-count": 128 }, "sender-location": point("26.08,85.94"), "send-time": datetime("2007-09-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " like sprint the reachability is awesome", "countA": 64, "countB": 158 }
+{ "tweetid": 65i64, "user": { "screen-name": "CarlineAft_666", "lang": "en", "friends-count": 25, "statuses-count": 352, "name": "Carline Aft", "followers-count": 59 }, "sender-location": point("29.33,78.49"), "send-time": datetime("2012-01-18T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " can't stand samsung the customization is terrible:(", "countA": 65, "countB": 77 }
+{ "tweetid": 66i64, "user": { "screen-name": "TenaGronko#55", "lang": "en", "friends-count": 91, "statuses-count": 2, "name": "Tena Gronko", "followers-count": 19 }, "sender-location": point("40.14,73.21"), "send-time": datetime("2011-03-18T10:10:00.000Z"), "referred-topics": {{ "at&t", "voice-command" }}, "message-text": " hate at&t the voice-command is horrible", "countA": 66, "countB": 73 }
+{ "tweetid": 67i64, "user": { "screen-name": "DanialBrinigh#499", "lang": "en", "friends-count": 67, "statuses-count": 413, "name": "Danial Brinigh", "followers-count": 4 }, "sender-location": point("41.26,97.09"), "send-time": datetime("2008-02-28T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " can't stand motorola the 3G is OMG:(", "countA": 67, "countB": 47 }
+{ "tweetid": 68i64, "user": { "screen-name": "GretaBusk#270", "lang": "en", "friends-count": 30, "statuses-count": 150, "name": "Greta Busk", "followers-count": 124 }, "sender-location": point("30.35,86.51"), "send-time": datetime("2008-03-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola the reachability is bad:(", "countA": 68, "countB": 21 }
+{ "tweetid": 69i64, "user": { "screen-name": "DanielBurch@155", "lang": "en", "friends-count": 5, "statuses-count": 268, "name": "Daniel Burch", "followers-count": 178 }, "sender-location": point("45.31,66.89"), "send-time": datetime("2009-06-27T10:10:00.000Z"), "referred-topics": {{ "samsung", "network" }}, "message-text": " hate samsung its network is horrible", "countA": 69, "countB": 153 }
+{ "tweetid": 70i64, "user": { "screen-name": "JaynaBash@532", "lang": "en", "friends-count": 90, "statuses-count": 244, "name": "Jayna Bash", "followers-count": 184 }, "sender-location": point("43.92,69.28"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " can't stand samsung the platform is bad", "countA": 70, "countB": 133 }
+{ "tweetid": 71i64, "user": { "screen-name": "PatriciaCason#475", "lang": "en", "friends-count": 50, "statuses-count": 149, "name": "Patricia Cason", "followers-count": 114 }, "sender-location": point("43.74,69.29"), "send-time": datetime("2009-08-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is amazing:)", "countA": 71, "countB": 185 }
+{ "tweetid": 72i64, "user": { "screen-name": "KatharineElsas_215", "lang": "en", "friends-count": 69, "statuses-count": 128, "name": "Katharine Elsas", "followers-count": 114 }, "sender-location": point("29.05,94.41"), "send-time": datetime("2010-09-25T10:10:00.000Z"), "referred-topics": {{ "samsung", "signal" }}, "message-text": " can't stand samsung the signal is OMG:(", "countA": 72, "countB": 31 }
+{ "tweetid": 73i64, "user": { "screen-name": "YorkSanborn_951", "lang": "en", "friends-count": 69, "statuses-count": 375, "name": "York Sanborn", "followers-count": 15 }, "sender-location": point("43.92,94.49"), "send-time": datetime("2010-09-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " dislike motorola its reachability is bad:(", "countA": 73, "countB": 61 }
+{ "tweetid": 74i64, "user": { "screen-name": "AlbertoDull$598", "lang": "en", "friends-count": 29, "statuses-count": 181, "name": "Alberto Dull", "followers-count": 192 }, "sender-location": point("25.6,85.23"), "send-time": datetime("2005-09-22T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " dislike samsung the wireless is OMG:(", "countA": 74, "countB": 32 }
+{ "tweetid": 75i64, "user": { "screen-name": "EnriqueFaast$123", "lang": "en", "friends-count": 9, "statuses-count": 24, "name": "Enrique Faast", "followers-count": 24 }, "sender-location": point("30.09,72.93"), "send-time": datetime("2009-10-17T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voicemail-service" }}, "message-text": " dislike t-mobile its voicemail-service is horrible", "countA": 75, "countB": 185 }
+{ "tweetid": 76i64, "user": { "screen-name": "AndreaBruxner$43", "lang": "en", "friends-count": 37, "statuses-count": 279, "name": "Andrea Bruxner", "followers-count": 118 }, "sender-location": point("30.39,92.92"), "send-time": datetime("2011-04-18T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " like t-mobile its customization is awesome", "countA": 76, "countB": 146 }
+{ "tweetid": 77i64, "user": { "screen-name": "LashawnaKemble$318", "lang": "en", "friends-count": 53, "statuses-count": 44, "name": "Lashawna Kemble", "followers-count": 102 }, "sender-location": point("46.29,93.16"), "send-time": datetime("2010-12-09T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " dislike samsung the voice-command is horrible", "countA": 77, "countB": 0 }
+{ "tweetid": 78i64, "user": { "screen-name": "RodolfoWoodworth#419", "lang": "en", "friends-count": 2, "statuses-count": 82, "name": "Rodolfo Woodworth", "followers-count": 16 }, "sender-location": point("44.92,70.03"), "send-time": datetime("2008-12-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " like motorola the touch-screen is awesome", "countA": 78, "countB": 140 }
+{ "tweetid": 79i64, "user": { "screen-name": "AbramCourtney_384", "lang": "en", "friends-count": 10, "statuses-count": 33, "name": "Abram Courtney", "followers-count": 138 }, "sender-location": point("34.9,96.91"), "send-time": datetime("2007-03-15T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " hate at&t the plan is bad:(", "countA": 79, "countB": 193 }
+{ "tweetid": 80i64, "user": { "screen-name": "LaurindaRosensteel@202", "lang": "en", "friends-count": 19, "statuses-count": 222, "name": "Laurinda Rosensteel", "followers-count": 20 }, "sender-location": point("47.86,92.66"), "send-time": datetime("2008-09-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " hate samsung its platform is horrible:(", "countA": 80, "countB": 39 }
+{ "tweetid": 81i64, "user": { "screen-name": "JarrettBratton@573", "lang": "en", "friends-count": 90, "statuses-count": 287, "name": "Jarrett Bratton", "followers-count": 18 }, "sender-location": point("33.85,75.61"), "send-time": datetime("2006-05-06T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " like sprint its network is amazing:)", "countA": 81, "countB": 105 }
+{ "tweetid": 82i64, "user": { "screen-name": "EleanorBicknell$880", "lang": "en", "friends-count": 35, "statuses-count": 444, "name": "Eleanor Bicknell", "followers-count": 199 }, "sender-location": point("42.31,95.69"), "send-time": datetime("2008-04-19T10:10:00.000Z"), "referred-topics": {{ "samsung", "plan" }}, "message-text": " can't stand samsung its plan is terrible", "countA": 82, "countB": 140 }
+{ "tweetid": 83i64, "user": { "screen-name": "MannyWerner#209", "lang": "en", "friends-count": 85, "statuses-count": 476, "name": "Manny Werner", "followers-count": 78 }, "sender-location": point("35.76,80.06"), "send-time": datetime("2007-11-17T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " dislike samsung the platform is terrible", "countA": 83, "countB": 163 }
+{ "tweetid": 84i64, "user": { "screen-name": "TeshaReade_713", "lang": "en", "friends-count": 74, "statuses-count": 99, "name": "Tesha Reade", "followers-count": 112 }, "sender-location": point("26.69,82.05"), "send-time": datetime("2007-01-04T10:10:00.000Z"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon the shortcut-menu is mind-blowing", "countA": 84, "countB": 172 }
+{ "tweetid": 85i64, "user": { "screen-name": "WanCrissman$283", "lang": "en", "friends-count": 37, "statuses-count": 384, "name": "Wan Crissman", "followers-count": 176 }, "sender-location": point("42.3,69.18"), "send-time": datetime("2006-10-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " love at&t its network is awesome:)", "countA": 85, "countB": 77 }
+{ "tweetid": 86i64, "user": { "screen-name": "DeedeeMccallum#158", "lang": "en", "friends-count": 81, "statuses-count": 370, "name": "Deedee Mccallum", "followers-count": 104 }, "sender-location": point("46.35,92.36"), "send-time": datetime("2011-09-22T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-command" }}, "message-text": " like verizon the voice-command is amazing:)", "countA": 86, "countB": 156 }
+{ "tweetid": 87i64, "user": { "screen-name": "AileenAft@340", "lang": "en", "friends-count": 87, "statuses-count": 476, "name": "Aileen Aft", "followers-count": 14 }, "sender-location": point("41.44,95.97"), "send-time": datetime("2012-07-16T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " love motorola its network is mind-blowing", "countA": 87, "countB": 155 }
+{ "tweetid": 88i64, "user": { "screen-name": "BurtonLinton_390", "lang": "en", "friends-count": 13, "statuses-count": 462, "name": "Burton Linton", "followers-count": 34 }, "sender-location": point("45.22,88.29"), "send-time": datetime("2011-12-15T10:10:00.000Z"), "referred-topics": {{ "iphone", "reachability" }}, "message-text": " love iphone the reachability is awesome", "countA": 88, "countB": 95 }
+{ "tweetid": 89i64, "user": { "screen-name": "DamionJoghs_943", "lang": "en", "friends-count": 18, "statuses-count": 388, "name": "Damion Joghs", "followers-count": 111 }, "sender-location": point("36.32,83.38"), "send-time": datetime("2011-06-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " love motorola its touch-screen is awesome", "countA": 89, "countB": 85 }
+{ "tweetid": 90i64, "user": { "screen-name": "LatoshaCowart_858", "lang": "en", "friends-count": 14, "statuses-count": 318, "name": "Latosha Cowart", "followers-count": 27 }, "sender-location": point("26.63,82.77"), "send-time": datetime("2011-10-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " love motorola the plan is good:)", "countA": 90, "countB": 139 }
+{ "tweetid": 91i64, "user": { "screen-name": "LoganPowers$336", "lang": "en", "friends-count": 52, "statuses-count": 154, "name": "Logan Powers", "followers-count": 28 }, "sender-location": point("30.66,96.22"), "send-time": datetime("2011-02-25T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " hate verizon the reachability is OMG:(", "countA": 91, "countB": 198 }
+{ "tweetid": 92i64, "user": { "screen-name": "NeilParkinson#794", "lang": "en", "friends-count": 18, "statuses-count": 365, "name": "Neil Parkinson", "followers-count": 27 }, "sender-location": point("31.25,71.75"), "send-time": datetime("2009-12-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t the platform is terrible:(", "countA": 92, "countB": 59 }
+{ "tweetid": 93i64, "user": { "screen-name": "GoddardFiscina$655", "lang": "en", "friends-count": 10, "statuses-count": 388, "name": "Goddard Fiscina", "followers-count": 142 }, "sender-location": point("33.4,72.55"), "send-time": datetime("2009-04-15T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " like samsung the speed is mind-blowing", "countA": 93, "countB": 55 }
+{ "tweetid": 94i64, "user": { "screen-name": "JacindaCressman_698", "lang": "en", "friends-count": 50, "statuses-count": 380, "name": "Jacinda Cressman", "followers-count": 112 }, "sender-location": point("33.68,85.33"), "send-time": datetime("2010-09-07T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " like sprint its network is amazing:)", "countA": 94, "countB": 128 }
+{ "tweetid": 95i64, "user": { "screen-name": "NelsonWilks_476", "lang": "en", "friends-count": 43, "statuses-count": 249, "name": "Nelson Wilks", "followers-count": 47 }, "sender-location": point("26.2,74.63"), "send-time": datetime("2010-10-28T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " love iphone the shortcut-menu is mind-blowing", "countA": 95, "countB": 144 }
+{ "tweetid": 96i64, "user": { "screen-name": "FelipeBeach_761", "lang": "en", "friends-count": 70, "statuses-count": 191, "name": "Felipe Beach", "followers-count": 56 }, "sender-location": point("38.59,75.94"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "sprint", "customization" }}, "message-text": " like sprint the customization is awesome:)", "countA": 96, "countB": 66 }
+{ "tweetid": 97i64, "user": { "screen-name": "MaximaPoehl$770", "lang": "en", "friends-count": 9, "statuses-count": 99, "name": "Maxima Poehl", "followers-count": 198 }, "sender-location": point("46.94,66.2"), "send-time": datetime("2008-03-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " like iphone the customization is good", "countA": 97, "countB": 83 }
+{ "tweetid": 98i64, "user": { "screen-name": "IraLombardi#278", "lang": "en", "friends-count": 10, "statuses-count": 282, "name": "Ira Lombardi", "followers-count": 26 }, "sender-location": point("44.99,93.61"), "send-time": datetime("2011-02-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " love iphone its platform is awesome:)", "countA": 98, "countB": 78 }
+{ "tweetid": 99i64, "user": { "screen-name": "RexHincken_917", "lang": "en", "friends-count": 88, "statuses-count": 292, "name": "Rex Hincken", "followers-count": 74 }, "sender-location": point("42.0,81.22"), "send-time": datetime("2008-09-01T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voicemail-service" }}, "message-text": " love t-mobile the voicemail-service is amazing:)", "countA": 99, "countB": 51 }
+{ "tweetid": 100i64, "user": { "screen-name": "DakotaTeagarden_163", "lang": "en", "friends-count": 54, "statuses-count": 391, "name": "Dakota Teagarden", "followers-count": 160 }, "sender-location": point("43.59,92.49"), "send-time": datetime("2010-11-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "shortcut-menu" }}, "message-text": " can't stand samsung its shortcut-menu is OMG:(", "countA": 100, "countB": 184 }
+{ "tweetid": 101i64, "user": { "screen-name": "ChetMilliron_934", "lang": "en", "friends-count": 62, "statuses-count": 453, "name": "Chet Milliron", "followers-count": 53 }, "sender-location": point("47.95,77.58"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " like samsung the 3G is mind-blowing:)", "countA": 101, "countB": 142 }
+{ "tweetid": 102i64, "user": { "screen-name": "ZackLosey_956", "lang": "en", "friends-count": 90, "statuses-count": 6, "name": "Zack Losey", "followers-count": 116 }, "sender-location": point("26.53,80.6"), "send-time": datetime("2005-01-11T10:10:00.000Z"), "referred-topics": {{ "verizon", "platform" }}, "message-text": " like verizon its platform is amazing", "countA": 102, "countB": 190 }
+{ "tweetid": 103i64, "user": { "screen-name": "BrionyLafortune$483", "lang": "en", "friends-count": 87, "statuses-count": 496, "name": "Briony Lafortune", "followers-count": 4 }, "sender-location": point("42.2,73.96"), "send-time": datetime("2010-12-05T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " can't stand samsung its voice-command is OMG", "countA": 103, "countB": 27 }
+{ "tweetid": 104i64, "user": { "screen-name": "SaraGraham@726", "lang": "en", "friends-count": 38, "statuses-count": 398, "name": "Sara Graham", "followers-count": 68 }, "sender-location": point("32.83,81.29"), "send-time": datetime("2009-10-25T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " like iphone its platform is awesome", "countA": 104, "countB": 55 }
+{ "tweetid": 105i64, "user": { "screen-name": "EvanBarnes_217", "lang": "en", "friends-count": 42, "statuses-count": 239, "name": "Evan Barnes", "followers-count": 108 }, "sender-location": point("44.7,90.98"), "send-time": datetime("2008-07-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " like sprint its reachability is mind-blowing", "countA": 105, "countB": 3 }
+{ "tweetid": 106i64, "user": { "screen-name": "JulianeNorthey#34", "lang": "en", "friends-count": 69, "statuses-count": 94, "name": "Juliane Northey", "followers-count": 187 }, "sender-location": point("26.32,67.64"), "send-time": datetime("2007-06-26T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " can't stand sprint the voice-clarity is terrible", "countA": 106, "countB": 127 }
+{ "tweetid": 107i64, "user": { "screen-name": "ShannahBailey$196", "lang": "en", "friends-count": 47, "statuses-count": 215, "name": "Shannah Bailey", "followers-count": 67 }, "sender-location": point("40.85,87.01"), "send-time": datetime("2009-10-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " love verizon the reachability is awesome:)", "countA": 107, "countB": 31 }
+{ "tweetid": 108i64, "user": { "screen-name": "GranvilleKnisely$497", "lang": "en", "friends-count": 57, "statuses-count": 117, "name": "Granville Knisely", "followers-count": 52 }, "sender-location": point("35.46,78.27"), "send-time": datetime("2006-06-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " hate samsung its customization is horrible", "countA": 108, "countB": 148 }
+{ "tweetid": 109i64, "user": { "screen-name": "LeonardoJardine@763", "lang": "en", "friends-count": 48, "statuses-count": 415, "name": "Leonardo Jardine", "followers-count": 96 }, "sender-location": point("27.7,92.32"), "send-time": datetime("2010-12-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " dislike verizon the signal is bad", "countA": 109, "countB": 29 }
+{ "tweetid": 110i64, "user": { "screen-name": "AuroraMcelroy@927", "lang": "en", "friends-count": 79, "statuses-count": 297, "name": "Aurora Mcelroy", "followers-count": 119 }, "sender-location": point("48.56,85.12"), "send-time": datetime("2005-06-14T10:10:00.000Z"), "referred-topics": {{ "motorola", "shortcut-menu" }}, "message-text": " dislike motorola the shortcut-menu is bad:(", "countA": 110, "countB": 157 }
+{ "tweetid": 111i64, "user": { "screen-name": "NoleneLeslie#166", "lang": "en", "friends-count": 30, "statuses-count": 3, "name": "Nolene Leslie", "followers-count": 18 }, "sender-location": point("31.07,78.53"), "send-time": datetime("2005-10-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "platform" }}, "message-text": " like t-mobile its platform is good", "countA": 111, "countB": 50 }
+{ "tweetid": 112i64, "user": { "screen-name": "EusebioBeedell@329", "lang": "en", "friends-count": 94, "statuses-count": 341, "name": "Eusebio Beedell", "followers-count": 89 }, "sender-location": point("32.75,68.79"), "send-time": datetime("2007-08-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " can't stand verizon its network is terrible:(", "countA": 112, "countB": 30 }
+{ "tweetid": 113i64, "user": { "screen-name": "WoodySaltser$873", "lang": "en", "friends-count": 68, "statuses-count": 365, "name": "Woody Saltser", "followers-count": 132 }, "sender-location": point("37.57,88.05"), "send-time": datetime("2012-01-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voice-clarity" }}, "message-text": " love t-mobile its voice-clarity is awesome", "countA": 113, "countB": 177 }
+{ "tweetid": 114i64, "user": { "screen-name": "ReannaSeelig#553", "lang": "en", "friends-count": 31, "statuses-count": 291, "name": "Reanna Seelig", "followers-count": 175 }, "sender-location": point("42.87,72.38"), "send-time": datetime("2006-03-20T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " dislike motorola the signal is OMG:(", "countA": 114, "countB": 59 }
+{ "tweetid": 115i64, "user": { "screen-name": "AllannahNapier@336", "lang": "en", "friends-count": 34, "statuses-count": 359, "name": "Allannah Napier", "followers-count": 50 }, "sender-location": point("31.29,88.73"), "send-time": datetime("2010-07-28T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " hate samsung the wireless is bad:(", "countA": 115, "countB": 175 }
+{ "tweetid": 116i64, "user": { "screen-name": "AlaynaOsteen_327", "lang": "en", "friends-count": 59, "statuses-count": 237, "name": "Alayna Osteen", "followers-count": 12 }, "sender-location": point("30.6,71.97"), "send-time": datetime("2007-07-25T10:10:00.000Z"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is amazing:)", "countA": 116, "countB": 70 }
+{ "tweetid": 117i64, "user": { "screen-name": "LeticiaMillard#139", "lang": "en", "friends-count": 95, "statuses-count": 46, "name": "Leticia Millard", "followers-count": 72 }, "sender-location": point("26.53,73.37"), "send-time": datetime("2005-06-22T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " dislike iphone its 3G is horrible", "countA": 117, "countB": 168 }
+{ "tweetid": 118i64, "user": { "screen-name": "WinifredMckee_639", "lang": "en", "friends-count": 48, "statuses-count": 442, "name": "Winifred Mckee", "followers-count": 199 }, "sender-location": point("27.51,76.65"), "send-time": datetime("2012-06-19T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " can't stand sprint the reachability is bad", "countA": 118, "countB": 70 }
+{ "tweetid": 119i64, "user": { "screen-name": "SungShea#585", "lang": "en", "friends-count": 38, "statuses-count": 193, "name": "Sung Shea", "followers-count": 149 }, "sender-location": point("28.86,83.73"), "send-time": datetime("2009-04-22T10:10:00.000Z"), "referred-topics": {{ "samsung", "customer-service" }}, "message-text": " can't stand samsung its customer-service is horrible:(", "countA": 119, "countB": 155 }
+{ "tweetid": 120i64, "user": { "screen-name": "BernadineSutton@199", "lang": "en", "friends-count": 72, "statuses-count": 46, "name": "Bernadine Sutton", "followers-count": 105 }, "sender-location": point("40.19,77.94"), "send-time": datetime("2007-11-12T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " like sprint the voice-clarity is good:)", "countA": 120, "countB": 63 }
+{ "tweetid": 121i64, "user": { "screen-name": "DeedeeJerome#182", "lang": "en", "friends-count": 74, "statuses-count": 342, "name": "Deedee Jerome", "followers-count": 170 }, "sender-location": point("42.66,73.84"), "send-time": datetime("2012-07-03T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " can't stand sprint its reachability is horrible", "countA": 121, "countB": 163 }
+{ "tweetid": 122i64, "user": { "screen-name": "NigelPrechtl$759", "lang": "en", "friends-count": 10, "statuses-count": 133, "name": "Nigel Prechtl", "followers-count": 137 }, "sender-location": point("37.22,80.92"), "send-time": datetime("2012-01-05T10:10:00.000Z"), "referred-topics": {{ "at&t", "speed" }}, "message-text": " dislike at&t its speed is terrible", "countA": 122, "countB": 23 }
+{ "tweetid": 123i64, "user": { "screen-name": "KimmyWynne$198", "lang": "en", "friends-count": 16, "statuses-count": 40, "name": "Kimmy Wynne", "followers-count": 6 }, "sender-location": point("26.49,70.55"), "send-time": datetime("2008-10-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " dislike t-mobile the signal is terrible", "countA": 123, "countB": 44 }
+{ "tweetid": 124i64, "user": { "screen-name": "ByronHarshman$352", "lang": "en", "friends-count": 26, "statuses-count": 133, "name": "Byron Harshman", "followers-count": 144 }, "sender-location": point("26.4,88.43"), "send-time": datetime("2012-03-21T10:10:00.000Z"), "referred-topics": {{ "samsung", "touch-screen" }}, "message-text": " dislike samsung the touch-screen is terrible", "countA": 124, "countB": 44 }
+{ "tweetid": 125i64, "user": { "screen-name": "PlacidPrevatt#865", "lang": "en", "friends-count": 3, "statuses-count": 493, "name": "Placid Prevatt", "followers-count": 10 }, "sender-location": point("43.09,84.0"), "send-time": datetime("2010-07-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "voicemail-service" }}, "message-text": " love motorola its voicemail-service is amazing:)", "countA": 125, "countB": 7 }
+{ "tweetid": 126i64, "user": { "screen-name": "TranterGarneis_456", "lang": "en", "friends-count": 89, "statuses-count": 151, "name": "Tranter Garneis", "followers-count": 166 }, "sender-location": point("41.6,93.6"), "send-time": datetime("2007-08-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone its shortcut-menu is amazing:)", "countA": 126, "countB": 76 }
+{ "tweetid": 127i64, "user": { "screen-name": "DonyaWilliamson$23", "lang": "en", "friends-count": 62, "statuses-count": 325, "name": "Donya Williamson", "followers-count": 101 }, "sender-location": point("27.75,66.01"), "send-time": datetime("2005-11-02T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " can't stand verizon its reachability is OMG", "countA": 127, "countB": 184 }
+{ "tweetid": 128i64, "user": { "screen-name": "GalinaJoghs$90", "lang": "en", "friends-count": 61, "statuses-count": 86, "name": "Galina Joghs", "followers-count": 169 }, "sender-location": point("30.95,71.04"), "send-time": datetime("2010-06-01T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " can't stand motorola its signal is horrible", "countA": 128, "countB": 24 }
+{ "tweetid": 129i64, "user": { "screen-name": "SamsonWerner#683", "lang": "en", "friends-count": 92, "statuses-count": 171, "name": "Samson Werner", "followers-count": 108 }, "sender-location": point("36.53,92.04"), "send-time": datetime("2009-08-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " hate motorola its network is terrible:(", "countA": 129, "countB": 80 }
+{ "tweetid": 130i64, "user": { "screen-name": "GabrielleMang#424", "lang": "en", "friends-count": 66, "statuses-count": 8, "name": "Gabrielle Mang", "followers-count": 80 }, "sender-location": point("36.74,96.64"), "send-time": datetime("2006-04-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " love sprint its plan is amazing:)", "countA": 130, "countB": 157 }
+{ "tweetid": 131i64, "user": { "screen-name": "ZachariasBaldwin#74", "lang": "en", "friends-count": 5, "statuses-count": 205, "name": "Zacharias Baldwin", "followers-count": 87 }, "sender-location": point("25.44,72.7"), "send-time": datetime("2006-01-21T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " like sprint the platform is amazing", "countA": 131, "countB": 192 }
+{ "tweetid": 132i64, "user": { "screen-name": "FanniePoorbaugh@315", "lang": "en", "friends-count": 59, "statuses-count": 441, "name": "Fannie Poorbaugh", "followers-count": 114 }, "sender-location": point("48.89,68.65"), "send-time": datetime("2005-05-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " hate verizon the reachability is terrible", "countA": 132, "countB": 142 }
+{ "tweetid": 133i64, "user": { "screen-name": "SandraTeagarden$747", "lang": "en", "friends-count": 60, "statuses-count": 353, "name": "Sandra Teagarden", "followers-count": 141 }, "sender-location": point("44.37,76.54"), "send-time": datetime("2008-05-25T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t its platform is bad", "countA": 133, "countB": 17 }
+{ "tweetid": 134i64, "user": { "screen-name": "SteveMayers_702", "lang": "en", "friends-count": 70, "statuses-count": 196, "name": "Steve Mayers", "followers-count": 22 }, "sender-location": point("41.52,91.39"), "send-time": datetime("2012-08-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "speed" }}, "message-text": " love iphone its speed is amazing", "countA": 134, "countB": 86 }
+{ "tweetid": 135i64, "user": { "screen-name": "RosalynPullman@789", "lang": "en", "friends-count": 17, "statuses-count": 470, "name": "Rosalyn Pullman", "followers-count": 123 }, "sender-location": point("48.33,86.41"), "send-time": datetime("2009-12-08T10:10:00.000Z"), "referred-topics": {{ "motorola", "customization" }}, "message-text": " hate motorola its customization is bad:(", "countA": 135, "countB": 171 }
+{ "tweetid": 136i64, "user": { "screen-name": "LamarChauvin$832", "lang": "en", "friends-count": 21, "statuses-count": 234, "name": "Lamar Chauvin", "followers-count": 184 }, "sender-location": point("36.83,89.48"), "send-time": datetime("2011-06-26T10:10:00.000Z"), "referred-topics": {{ "samsung", "reachability" }}, "message-text": " love samsung its reachability is awesome:)", "countA": 136, "countB": 77 }
+{ "tweetid": 137i64, "user": { "screen-name": "EleaseReade#477", "lang": "en", "friends-count": 24, "statuses-count": 299, "name": "Elease Reade", "followers-count": 24 }, "sender-location": point("45.55,93.09"), "send-time": datetime("2012-07-19T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " hate at&t the network is bad:(", "countA": 137, "countB": 25 }
+{ "tweetid": 138i64, "user": { "screen-name": "LeviPhilbrick$328", "lang": "en", "friends-count": 73, "statuses-count": 77, "name": "Levi Philbrick", "followers-count": 179 }, "sender-location": point("33.51,68.88"), "send-time": datetime("2009-08-28T10:10:00.000Z"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " dislike verizon its shortcut-menu is horrible", "countA": 138, "countB": 5 }
+{ "tweetid": 139i64, "user": { "screen-name": "RaeburnNickolson_295", "lang": "en", "friends-count": 72, "statuses-count": 176, "name": "Raeburn Nickolson", "followers-count": 103 }, "sender-location": point("38.42,74.16"), "send-time": datetime("2008-05-28T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " like at&t its voicemail-service is good:)", "countA": 139, "countB": 175 }
+{ "tweetid": 140i64, "user": { "screen-name": "NerissaBallou@177", "lang": "en", "friends-count": 17, "statuses-count": 447, "name": "Nerissa Ballou", "followers-count": 183 }, "sender-location": point("34.65,81.44"), "send-time": datetime("2008-07-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " hate at&t its plan is OMG", "countA": 140, "countB": 192 }
+{ "tweetid": 141i64, "user": { "screen-name": "DanyelWalker#602", "lang": "en", "friends-count": 22, "statuses-count": 397, "name": "Danyel Walker", "followers-count": 154 }, "sender-location": point("40.38,78.39"), "send-time": datetime("2012-01-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "3G" }}, "message-text": " hate t-mobile its 3G is OMG", "countA": 141, "countB": 9 }
+{ "tweetid": 142i64, "user": { "screen-name": "YaronLeichter_45", "lang": "en", "friends-count": 80, "statuses-count": 103, "name": "Yaron Leichter", "followers-count": 115 }, "sender-location": point("42.59,71.72"), "send-time": datetime("2010-04-24T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " like verizon the signal is mind-blowing:)", "countA": 142, "countB": 130 }
+{ "tweetid": 143i64, "user": { "screen-name": "DeemerCable_599", "lang": "en", "friends-count": 26, "statuses-count": 371, "name": "Deemer Cable", "followers-count": 34 }, "sender-location": point("29.89,70.29"), "send-time": datetime("2012-08-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " like motorola the wireless is awesome", "countA": 143, "countB": 50 }
+{ "tweetid": 144i64, "user": { "screen-name": "AliaHay_860", "lang": "en", "friends-count": 34, "statuses-count": 470, "name": "Alia Hay", "followers-count": 111 }, "sender-location": point("46.39,96.22"), "send-time": datetime("2010-12-18T10:10:00.000Z"), "referred-topics": {{ "at&t", "shortcut-menu" }}, "message-text": " like at&t its shortcut-menu is amazing:)", "countA": 144, "countB": 184 }
+{ "tweetid": 145i64, "user": { "screen-name": "AdamMoore$384", "lang": "en", "friends-count": 42, "statuses-count": 198, "name": "Adam Moore", "followers-count": 17 }, "sender-location": point("27.99,84.55"), "send-time": datetime("2007-03-16T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " love sprint its network is good", "countA": 145, "countB": 162 }
+{ "tweetid": 146i64, "user": { "screen-name": "AileenSouthern$868", "lang": "en", "friends-count": 62, "statuses-count": 388, "name": "Aileen Southern", "followers-count": 131 }, "sender-location": point("30.85,66.62"), "send-time": datetime("2005-07-14T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " like t-mobile the signal is amazing", "countA": 146, "countB": 102 }
+{ "tweetid": 147i64, "user": { "screen-name": "JewellWise_154", "lang": "en", "friends-count": 2, "statuses-count": 279, "name": "Jewell Wise", "followers-count": 107 }, "sender-location": point("46.72,83.98"), "send-time": datetime("2006-01-09T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " hate motorola the plan is terrible:(", "countA": 147, "countB": 174 }
+{ "tweetid": 148i64, "user": { "screen-name": "DanielJowers#519", "lang": "en", "friends-count": 23, "statuses-count": 22, "name": "Daniel Jowers", "followers-count": 131 }, "sender-location": point("34.26,72.22"), "send-time": datetime("2008-07-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-clarity" }}, "message-text": " like motorola its voice-clarity is mind-blowing:)", "countA": 148, "countB": 6 }
+{ "tweetid": 149i64, "user": { "screen-name": "DillonWilliams_557", "lang": "en", "friends-count": 18, "statuses-count": 136, "name": "Dillon Williams", "followers-count": 35 }, "sender-location": point("46.63,97.38"), "send-time": datetime("2011-05-09T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " love motorola the network is good", "countA": 149, "countB": 20 }
+{ "tweetid": 150i64, "user": { "screen-name": "DerrickBullard$202", "lang": "en", "friends-count": 16, "statuses-count": 9, "name": "Derrick Bullard", "followers-count": 100 }, "sender-location": point("41.89,90.62"), "send-time": datetime("2012-08-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " like t-mobile its touch-screen is mind-blowing", "countA": 150, "countB": 145 }
+{ "tweetid": 151i64, "user": { "screen-name": "LuigiMcfall_976", "lang": "en", "friends-count": 31, "statuses-count": 215, "name": "Luigi Mcfall", "followers-count": 79 }, "sender-location": point("45.38,70.52"), "send-time": datetime("2005-11-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " hate motorola its reachability is OMG:(", "countA": 151, "countB": 43 }
+{ "tweetid": 152i64, "user": { "screen-name": "MartinPinney_858", "lang": "en", "friends-count": 21, "statuses-count": 465, "name": "Martin Pinney", "followers-count": 16 }, "sender-location": point("32.87,75.66"), "send-time": datetime("2007-10-12T10:10:00.000Z"), "referred-topics": {{ "at&t", "reachability" }}, "message-text": " hate at&t its reachability is terrible", "countA": 152, "countB": 56 }
+{ "tweetid": 153i64, "user": { "screen-name": "JackieAft_623", "lang": "en", "friends-count": 70, "statuses-count": 413, "name": "Jackie Aft", "followers-count": 138 }, "sender-location": point("29.9,73.29"), "send-time": datetime("2010-07-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " like verizon the voice-clarity is amazing", "countA": 153, "countB": 154 }
+{ "tweetid": 154i64, "user": { "screen-name": "SherriWickes#118", "lang": "en", "friends-count": 20, "statuses-count": 31, "name": "Sherri Wickes", "followers-count": 59 }, "sender-location": point("39.2,79.2"), "send-time": datetime("2006-07-13T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " hate sprint the network is OMG:(", "countA": 154, "countB": 124 }
+{ "tweetid": 155i64, "user": { "screen-name": "CarlieCowher@103", "lang": "en", "friends-count": 81, "statuses-count": 127, "name": "Carlie Cowher", "followers-count": 184 }, "sender-location": point("30.3,76.43"), "send-time": datetime("2010-10-04T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " can't stand sprint the voice-clarity is bad", "countA": 155, "countB": 39 }
+{ "tweetid": 156i64, "user": { "screen-name": "AndraWardle@74", "lang": "en", "friends-count": 41, "statuses-count": 35, "name": "Andra Wardle", "followers-count": 168 }, "sender-location": point("45.49,93.97"), "send-time": datetime("2009-02-18T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " love t-mobile its wireless is amazing:)", "countA": 156, "countB": 23 }
+{ "tweetid": 157i64, "user": { "screen-name": "KanishaPinney@150", "lang": "en", "friends-count": 89, "statuses-count": 315, "name": "Kanisha Pinney", "followers-count": 173 }, "sender-location": point("24.72,77.36"), "send-time": datetime("2005-06-10T10:10:00.000Z"), "referred-topics": {{ "verizon", "touch-screen" }}, "message-text": " like verizon the touch-screen is amazing", "countA": 157, "countB": 153 }
+{ "tweetid": 158i64, "user": { "screen-name": "GlyndaSchere@104", "lang": "en", "friends-count": 6, "statuses-count": 111, "name": "Glynda Schere", "followers-count": 120 }, "sender-location": point("33.86,67.49"), "send-time": datetime("2010-11-26T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " like verizon its reachability is amazing:)", "countA": 158, "countB": 96 }
+{ "tweetid": 159i64, "user": { "screen-name": "JenelleNehling@461", "lang": "en", "friends-count": 4, "statuses-count": 384, "name": "Jenelle Nehling", "followers-count": 57 }, "sender-location": point("32.65,89.38"), "send-time": datetime("2010-12-06T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone its shortcut-menu is good", "countA": 159, "countB": 113 }
+{ "tweetid": 160i64, "user": { "screen-name": "DelWheeler@286", "lang": "en", "friends-count": 23, "statuses-count": 403, "name": "Del Wheeler", "followers-count": 177 }, "sender-location": point("39.39,78.05"), "send-time": datetime("2011-07-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " love samsung the platform is mind-blowing", "countA": 160, "countB": 163 }
+{ "tweetid": 161i64, "user": { "screen-name": "TrinityCowart@360", "lang": "en", "friends-count": 85, "statuses-count": 204, "name": "Trinity Cowart", "followers-count": 145 }, "sender-location": point("32.11,76.64"), "send-time": datetime("2010-10-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " like t-mobile its wireless is amazing:)", "countA": 161, "countB": 62 }
+{ "tweetid": 162i64, "user": { "screen-name": "HudsonBasmanoff_348", "lang": "en", "friends-count": 3, "statuses-count": 394, "name": "Hudson Basmanoff", "followers-count": 114 }, "sender-location": point("39.24,78.13"), "send-time": datetime("2007-09-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " can't stand motorola its wireless is OMG:(", "countA": 162, "countB": 57 }
+{ "tweetid": 163i64, "user": { "screen-name": "MatthewPowers_801", "lang": "en", "friends-count": 2, "statuses-count": 203, "name": "Matthew Powers", "followers-count": 199 }, "sender-location": point("33.79,69.57"), "send-time": datetime("2012-04-05T10:10:00.000Z"), "referred-topics": {{ "at&t", "customer-service" }}, "message-text": " love at&t its customer-service is awesome", "countA": 163, "countB": 115 }
+{ "tweetid": 164i64, "user": { "screen-name": "TitaniaKern$100", "lang": "en", "friends-count": 98, "statuses-count": 300, "name": "Titania Kern", "followers-count": 118 }, "sender-location": point("45.86,67.64"), "send-time": datetime("2005-11-04T10:10:00.000Z"), "referred-topics": {{ "at&t", "shortcut-menu" }}, "message-text": " love at&t the shortcut-menu is amazing", "countA": 164, "countB": 148 }
+{ "tweetid": 165i64, "user": { "screen-name": "EhtelCrissman#778", "lang": "en", "friends-count": 33, "statuses-count": 286, "name": "Ehtel Crissman", "followers-count": 63 }, "sender-location": point("32.98,82.49"), "send-time": datetime("2011-08-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is good:)", "countA": 165, "countB": 127 }
+{ "tweetid": 166i64, "user": { "screen-name": "WilletteLeslie@682", "lang": "en", "friends-count": 38, "statuses-count": 491, "name": "Willette Leslie", "followers-count": 75 }, "sender-location": point("32.57,84.97"), "send-time": datetime("2012-04-18T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " like verizon its reachability is mind-blowing", "countA": 166, "countB": 162 }
+{ "tweetid": 167i64, "user": { "screen-name": "DarellHincken_722", "lang": "en", "friends-count": 33, "statuses-count": 111, "name": "Darell Hincken", "followers-count": 74 }, "sender-location": point("48.57,77.77"), "send-time": datetime("2008-06-27T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile its speed is good:)", "countA": 167, "countB": 93 }
+{ "tweetid": 168i64, "user": { "screen-name": "DuaneKing@956", "lang": "en", "friends-count": 5, "statuses-count": 44, "name": "Duane King", "followers-count": 169 }, "sender-location": point("31.26,68.61"), "send-time": datetime("2011-03-24T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " like sprint its touch-screen is mind-blowing:)", "countA": 168, "countB": 174 }
+{ "tweetid": 169i64, "user": { "screen-name": "AmbroseKeilbach$300", "lang": "en", "friends-count": 76, "statuses-count": 278, "name": "Ambrose Keilbach", "followers-count": 54 }, "sender-location": point("29.75,71.35"), "send-time": datetime("2012-02-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "signal" }}, "message-text": " can't stand sprint its signal is horrible", "countA": 169, "countB": 129 }
+{ "tweetid": 170i64, "user": { "screen-name": "KarlBrooks#97", "lang": "en", "friends-count": 76, "statuses-count": 150, "name": "Karl Brooks", "followers-count": 117 }, "sender-location": point("30.77,85.78"), "send-time": datetime("2006-09-16T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " hate motorola the plan is horrible", "countA": 170, "countB": 174 }
+{ "tweetid": 171i64, "user": { "screen-name": "ShainaMayers$261", "lang": "en", "friends-count": 76, "statuses-count": 240, "name": "Shaina Mayers", "followers-count": 194 }, "sender-location": point("26.11,78.33"), "send-time": datetime("2005-06-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "voicemail-service" }}, "message-text": " love sprint its voicemail-service is mind-blowing:)", "countA": 171, "countB": 69 }
+{ "tweetid": 172i64, "user": { "screen-name": "LakeshaPery_35", "lang": "en", "friends-count": 58, "statuses-count": 300, "name": "Lakesha Pery", "followers-count": 51 }, "sender-location": point("38.45,75.31"), "send-time": datetime("2009-12-20T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " like motorola its 3G is good:)", "countA": 172, "countB": 127 }
+{ "tweetid": 173i64, "user": { "screen-name": "DoranMingle#901", "lang": "en", "friends-count": 3, "statuses-count": 302, "name": "Doran Mingle", "followers-count": 152 }, "sender-location": point("47.76,91.28"), "send-time": datetime("2009-06-07T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " dislike at&t the customization is OMG", "countA": 173, "countB": 41 }
+{ "tweetid": 174i64, "user": { "screen-name": "AmadaHatcher#710", "lang": "en", "friends-count": 3, "statuses-count": 12, "name": "Amada Hatcher", "followers-count": 193 }, "sender-location": point("28.64,89.42"), "send-time": datetime("2011-09-16T10:10:00.000Z"), "referred-topics": {{ "samsung", "network" }}, "message-text": " hate samsung the network is OMG", "countA": 174, "countB": 0 }
+{ "tweetid": 175i64, "user": { "screen-name": "CorianderMoon@658", "lang": "en", "friends-count": 28, "statuses-count": 117, "name": "Coriander Moon", "followers-count": 77 }, "sender-location": point("43.82,87.23"), "send-time": datetime("2011-03-26T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " dislike samsung the customization is bad:(", "countA": 175, "countB": 5 }
+{ "tweetid": 176i64, "user": { "screen-name": "PhilomenaEiford#608", "lang": "en", "friends-count": 53, "statuses-count": 467, "name": "Philomena Eiford", "followers-count": 80 }, "sender-location": point("30.06,71.08"), "send-time": datetime("2006-05-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " like motorola the signal is amazing", "countA": 176, "countB": 89 }
+{ "tweetid": 177i64, "user": { "screen-name": "JacobCongdon$162", "lang": "en", "friends-count": 61, "statuses-count": 161, "name": "Jacob Congdon", "followers-count": 19 }, "sender-location": point("36.36,78.43"), "send-time": datetime("2006-12-10T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " dislike samsung its platform is horrible", "countA": 177, "countB": 15 }
+{ "tweetid": 178i64, "user": { "screen-name": "GeorgeRichards$777", "lang": "en", "friends-count": 12, "statuses-count": 213, "name": "George Richards", "followers-count": 72 }, "sender-location": point("44.78,90.69"), "send-time": datetime("2006-03-17T10:10:00.000Z"), "referred-topics": {{ "verizon", "speed" }}, "message-text": " can't stand verizon its speed is horrible", "countA": 178, "countB": 64 }
+{ "tweetid": 179i64, "user": { "screen-name": "BridgerHamilton@431", "lang": "en", "friends-count": 51, "statuses-count": 396, "name": "Bridger Hamilton", "followers-count": 110 }, "sender-location": point("32.82,81.54"), "send-time": datetime("2008-12-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " love verizon the plan is awesome", "countA": 179, "countB": 141 }
+{ "tweetid": 180i64, "user": { "screen-name": "FemieLucy@34", "lang": "en", "friends-count": 68, "statuses-count": 221, "name": "Femie Lucy", "followers-count": 11 }, "sender-location": point("31.58,82.78"), "send-time": datetime("2010-02-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " hate iphone its customization is OMG", "countA": 180, "countB": 191 }
+{ "tweetid": 181i64, "user": { "screen-name": "JodiNapier@338", "lang": "en", "friends-count": 96, "statuses-count": 467, "name": "Jodi Napier", "followers-count": 69 }, "sender-location": point("26.14,78.5"), "send-time": datetime("2007-01-06T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " love verizon the plan is mind-blowing", "countA": 181, "countB": 13 }
+{ "tweetid": 182i64, "user": { "screen-name": "MitsueRawls_424", "lang": "en", "friends-count": 91, "statuses-count": 70, "name": "Mitsue Rawls", "followers-count": 193 }, "sender-location": point("47.77,70.41"), "send-time": datetime("2007-04-12T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " dislike iphone its customer-service is OMG", "countA": 182, "countB": 59 }
+{ "tweetid": 183i64, "user": { "screen-name": "DeshawnAultman_690", "lang": "en", "friends-count": 49, "statuses-count": 330, "name": "Deshawn Aultman", "followers-count": 39 }, "sender-location": point("40.65,79.37"), "send-time": datetime("2005-08-11T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " like sprint the shortcut-menu is amazing:)", "countA": 183, "countB": 199 }
+{ "tweetid": 184i64, "user": { "screen-name": "BradfordEiford#127", "lang": "en", "friends-count": 90, "statuses-count": 425, "name": "Bradford Eiford", "followers-count": 10 }, "sender-location": point("40.85,91.17"), "send-time": datetime("2005-09-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "voicemail-service" }}, "message-text": " dislike iphone its voicemail-service is horrible:(", "countA": 184, "countB": 141 }
+{ "tweetid": 185i64, "user": { "screen-name": "MadelynGaskins_356", "lang": "en", "friends-count": 48, "statuses-count": 455, "name": "Madelyn Gaskins", "followers-count": 66 }, "sender-location": point("33.81,88.32"), "send-time": datetime("2011-10-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " love at&t its voicemail-service is good:)", "countA": 185, "countB": 189 }
+{ "tweetid": 186i64, "user": { "screen-name": "SophiaMang@768", "lang": "en", "friends-count": 28, "statuses-count": 86, "name": "Sophia Mang", "followers-count": 125 }, "sender-location": point("37.45,68.47"), "send-time": datetime("2010-08-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "touch-screen" }}, "message-text": " love iphone its touch-screen is awesome:)", "countA": 186, "countB": 149 }
+{ "tweetid": 187i64, "user": { "screen-name": "VernonKnisely#170", "lang": "en", "friends-count": 63, "statuses-count": 406, "name": "Vernon Knisely", "followers-count": 31 }, "sender-location": point("32.93,94.65"), "send-time": datetime("2006-03-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " love sprint the reachability is awesome", "countA": 187, "countB": 98 }
+{ "tweetid": 188i64, "user": { "screen-name": "AmyEndsley@85", "lang": "en", "friends-count": 83, "statuses-count": 43, "name": "Amy Endsley", "followers-count": 7 }, "sender-location": point("27.33,82.34"), "send-time": datetime("2005-01-11T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " love at&t its customization is good", "countA": 188, "countB": 48 }
+{ "tweetid": 189i64, "user": { "screen-name": "AntonChristner#166", "lang": "en", "friends-count": 12, "statuses-count": 10, "name": "Anton Christner", "followers-count": 66 }, "sender-location": point("25.29,89.55"), "send-time": datetime("2006-11-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " can't stand motorola its wireless is terrible:(", "countA": 189, "countB": 160 }
+{ "tweetid": 190i64, "user": { "screen-name": "DeshawnHarris#34", "lang": "en", "friends-count": 32, "statuses-count": 488, "name": "Deshawn Harris", "followers-count": 178 }, "sender-location": point("45.46,76.04"), "send-time": datetime("2007-05-13T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " like samsung the customization is awesome:)", "countA": 190, "countB": 86 }
+{ "tweetid": 191i64, "user": { "screen-name": "MarioHolts_870", "lang": "en", "friends-count": 20, "statuses-count": 192, "name": "Mario Holts", "followers-count": 71 }, "sender-location": point("29.69,71.42"), "send-time": datetime("2005-09-15T10:10:00.000Z"), "referred-topics": {{ "iphone", "speed" }}, "message-text": " hate iphone the speed is horrible:(", "countA": 191, "countB": 150 }
+{ "tweetid": 192i64, "user": { "screen-name": "DeanHall#220", "lang": "en", "friends-count": 55, "statuses-count": 236, "name": "Dean Hall", "followers-count": 68 }, "sender-location": point("48.12,72.0"), "send-time": datetime("2006-11-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " love sprint the platform is awesome", "countA": 192, "countB": 199 }
+{ "tweetid": 193i64, "user": { "screen-name": "LoanClarke_206", "lang": "en", "friends-count": 32, "statuses-count": 173, "name": "Loan Clarke", "followers-count": 186 }, "sender-location": point("45.39,96.01"), "send-time": datetime("2009-05-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "customization" }}, "message-text": " can't stand verizon its customization is horrible:(", "countA": 193, "countB": 147 }
+{ "tweetid": 194i64, "user": { "screen-name": "DonnetteGoodman@627", "lang": "en", "friends-count": 61, "statuses-count": 202, "name": "Donnette Goodman", "followers-count": 106 }, "sender-location": point("44.72,73.13"), "send-time": datetime("2005-04-27T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "plan" }}, "message-text": " love t-mobile its plan is good", "countA": 194, "countB": 142 }
+{ "tweetid": 195i64, "user": { "screen-name": "TamekaPorter#315", "lang": "en", "friends-count": 79, "statuses-count": 63, "name": "Tameka Porter", "followers-count": 7 }, "sender-location": point("37.68,81.78"), "send-time": datetime("2006-06-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " like samsung its wireless is good", "countA": 195, "countB": 126 }
+{ "tweetid": 196i64, "user": { "screen-name": "GarlandClark@425", "lang": "en", "friends-count": 15, "statuses-count": 375, "name": "Garland Clark", "followers-count": 24 }, "sender-location": point("44.9,70.1"), "send-time": datetime("2006-01-08T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint the touch-screen is OMG:(", "countA": 196, "countB": 197 }
+{ "tweetid": 197i64, "user": { "screen-name": "RupertSanner$868", "lang": "en", "friends-count": 49, "statuses-count": 414, "name": "Rupert Sanner", "followers-count": 189 }, "sender-location": point("40.45,94.94"), "send-time": datetime("2012-06-20T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " can't stand iphone the customer-service is terrible:(", "countA": 197, "countB": 195 }
+{ "tweetid": 198i64, "user": { "screen-name": "JenZoucks#841", "lang": "en", "friends-count": 99, "statuses-count": 73, "name": "Jen Zoucks", "followers-count": 71 }, "sender-location": point("38.77,70.33"), "send-time": datetime("2010-07-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "signal" }}, "message-text": " like iphone its signal is good", "countA": 198, "countB": 94 }
+{ "tweetid": 199i64, "user": { "screen-name": "IsmaelLlora@529", "lang": "en", "friends-count": 63, "statuses-count": 239, "name": "Ismael Llora", "followers-count": 135 }, "sender-location": point("35.31,82.71"), "send-time": datetime("2007-03-19T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is amazing:)", "countA": 199, "countB": 172 }
+{ "tweetid": 200i64, "user": { "screen-name": "JenniferHoltzer_459", "lang": "en", "friends-count": 93, "statuses-count": 172, "name": "Jennifer Holtzer", "followers-count": 51 }, "sender-location": point("30.44,81.57"), "send-time": datetime("2007-11-24T10:10:00.000Z"), "referred-topics": {{ "at&t", "wireless" }}, "message-text": " love at&t the wireless is amazing", "countA": 200, "countB": 20 }
+{ "tweetid": 201i64, "user": { "screen-name": "EvanJardine$78", "lang": "en", "friends-count": 80, "statuses-count": 349, "name": "Evan Jardine", "followers-count": 155 }, "sender-location": point("39.12,68.0"), "send-time": datetime("2010-01-13T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " like t-mobile its signal is awesome:)", "countA": 201, "countB": 65 }
+{ "tweetid": 202i64, "user": { "screen-name": "KerenBard_268", "lang": "en", "friends-count": 28, "statuses-count": 348, "name": "Keren Bard", "followers-count": 158 }, "sender-location": point("32.72,77.86"), "send-time": datetime("2008-07-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " love sprint its platform is awesome:)", "countA": 202, "countB": 117 }
+{ "tweetid": 203i64, "user": { "screen-name": "GilbertLosey#710", "lang": "en", "friends-count": 67, "statuses-count": 141, "name": "Gilbert Losey", "followers-count": 192 }, "sender-location": point("44.66,84.32"), "send-time": datetime("2011-09-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "signal" }}, "message-text": " dislike at&t its signal is horrible", "countA": 203, "countB": 182 }
+{ "tweetid": 204i64, "user": { "screen-name": "TawnyGraham_816", "lang": "en", "friends-count": 63, "statuses-count": 13, "name": "Tawny Graham", "followers-count": 155 }, "sender-location": point("43.08,86.89"), "send-time": datetime("2012-05-10T10:10:00.000Z"), "referred-topics": {{ "verizon", "touch-screen" }}, "message-text": " dislike verizon its touch-screen is terrible", "countA": 204, "countB": 104 }
+{ "tweetid": 205i64, "user": { "screen-name": "LamontWilkins_977", "lang": "en", "friends-count": 19, "statuses-count": 102, "name": "Lamont Wilkins", "followers-count": 127 }, "sender-location": point("31.77,87.68"), "send-time": datetime("2005-04-12T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " dislike samsung its 3G is bad", "countA": 205, "countB": 57 }
+{ "tweetid": 206i64, "user": { "screen-name": "VincentRiggle$122", "lang": "en", "friends-count": 99, "statuses-count": 105, "name": "Vincent Riggle", "followers-count": 173 }, "sender-location": point("34.28,74.71"), "send-time": datetime("2007-05-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " like motorola the wireless is amazing:)", "countA": 206, "countB": 3 }
+{ "tweetid": 207i64, "user": { "screen-name": "AdelaJones_400", "lang": "en", "friends-count": 74, "statuses-count": 121, "name": "Adela Jones", "followers-count": 161 }, "sender-location": point("28.5,94.52"), "send-time": datetime("2010-01-20T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "platform" }}, "message-text": " like t-mobile the platform is good:)", "countA": 207, "countB": 110 }
+{ "tweetid": 208i64, "user": { "screen-name": "PrestonLittle$360", "lang": "en", "friends-count": 61, "statuses-count": 430, "name": "Preston Little", "followers-count": 115 }, "sender-location": point("35.79,78.52"), "send-time": datetime("2010-03-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " dislike sprint the speed is terrible:(", "countA": 208, "countB": 149 }
+{ "tweetid": 209i64, "user": { "screen-name": "TaraAnderson#214", "lang": "en", "friends-count": 60, "statuses-count": 219, "name": "Tara Anderson", "followers-count": 110 }, "sender-location": point("34.04,79.5"), "send-time": datetime("2006-12-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " dislike motorola its plan is OMG", "countA": 209, "countB": 164 }
+{ "tweetid": 210i64, "user": { "screen-name": "DeshawnWallace$305", "lang": "en", "friends-count": 29, "statuses-count": 279, "name": "Deshawn Wallace", "followers-count": 27 }, "sender-location": point("39.49,76.58"), "send-time": datetime("2011-01-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " dislike t-mobile the customization is OMG", "countA": 210, "countB": 70 }
+{ "tweetid": 211i64, "user": { "screen-name": "OliFisher$694", "lang": "en", "friends-count": 39, "statuses-count": 131, "name": "Oli Fisher", "followers-count": 101 }, "sender-location": point("44.91,94.72"), "send-time": datetime("2010-04-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " love iphone the 3G is mind-blowing", "countA": 211, "countB": 95 }
+{ "tweetid": 212i64, "user": { "screen-name": "LizaMathews#376", "lang": "en", "friends-count": 40, "statuses-count": 107, "name": "Liza Mathews", "followers-count": 70 }, "sender-location": point("46.01,77.85"), "send-time": datetime("2009-07-20T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " like verizon the plan is mind-blowing:)", "countA": 212, "countB": 189 }
+{ "tweetid": 213i64, "user": { "screen-name": "MylesRahl#433", "lang": "en", "friends-count": 51, "statuses-count": 144, "name": "Myles Rahl", "followers-count": 90 }, "sender-location": point("45.41,95.69"), "send-time": datetime("2006-04-03T10:10:00.000Z"), "referred-topics": {{ "verizon", "3G" }}, "message-text": " love verizon its 3G is mind-blowing:)", "countA": 213, "countB": 190 }
+{ "tweetid": 214i64, "user": { "screen-name": "BertEve@968", "lang": "en", "friends-count": 84, "statuses-count": 110, "name": "Bert Eve", "followers-count": 122 }, "sender-location": point("43.25,87.82"), "send-time": datetime("2009-11-07T10:10:00.000Z"), "referred-topics": {{ "at&t", "voice-command" }}, "message-text": " dislike at&t the voice-command is terrible:(", "countA": 214, "countB": 142 }
+{ "tweetid": 215i64, "user": { "screen-name": "MelissaLaurenzi_383", "lang": "en", "friends-count": 78, "statuses-count": 318, "name": "Melissa Laurenzi", "followers-count": 19 }, "sender-location": point("37.82,86.96"), "send-time": datetime("2010-05-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " like iphone its customer-service is mind-blowing", "countA": 215, "countB": 67 }
+{ "tweetid": 216i64, "user": { "screen-name": "ReneaPennington#175", "lang": "en", "friends-count": 42, "statuses-count": 16, "name": "Renea Pennington", "followers-count": 52 }, "sender-location": point("39.06,78.62"), "send-time": datetime("2006-05-23T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " dislike iphone its 3G is bad:(", "countA": 216, "countB": 50 }
+{ "tweetid": 217i64, "user": { "screen-name": "UrsulaMitchell@26", "lang": "en", "friends-count": 50, "statuses-count": 150, "name": "Ursula Mitchell", "followers-count": 191 }, "sender-location": point("34.02,71.3"), "send-time": datetime("2005-03-26T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " like iphone its customer-service is awesome", "countA": 217, "countB": 154 }
+{ "tweetid": 218i64, "user": { "screen-name": "WillyLambert_669", "lang": "en", "friends-count": 6, "statuses-count": 44, "name": "Willy Lambert", "followers-count": 60 }, "sender-location": point("32.84,73.25"), "send-time": datetime("2011-10-04T10:10:00.000Z"), "referred-topics": {{ "verizon", "customization" }}, "message-text": " like verizon the customization is amazing", "countA": 218, "countB": 135 }
+{ "tweetid": 219i64, "user": { "screen-name": "ChristineLaurence_912", "lang": "en", "friends-count": 77, "statuses-count": 458, "name": "Christine Laurence", "followers-count": 166 }, "sender-location": point("35.39,90.54"), "send-time": datetime("2009-12-01T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " like at&t the customization is awesome:)", "countA": 219, "countB": 170 }
+{ "tweetid": 220i64, "user": { "screen-name": "JedFiddler_540", "lang": "en", "friends-count": 11, "statuses-count": 264, "name": "Jed Fiddler", "followers-count": 48 }, "sender-location": point("44.11,91.45"), "send-time": datetime("2009-10-10T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " like at&t its voicemail-service is awesome:)", "countA": 220, "countB": 188 }
+{ "tweetid": 221i64, "user": { "screen-name": "RubinMueller_263", "lang": "en", "friends-count": 7, "statuses-count": 374, "name": "Rubin Mueller", "followers-count": 77 }, "sender-location": point("25.65,78.68"), "send-time": datetime("2006-09-20T10:10:00.000Z"), "referred-topics": {{ "sprint", "signal" }}, "message-text": " love sprint its signal is amazing", "countA": 221, "countB": 185 }
+{ "tweetid": 222i64, "user": { "screen-name": "JeniWeldi#255", "lang": "en", "friends-count": 71, "statuses-count": 259, "name": "Jeni Weldi", "followers-count": 63 }, "sender-location": point("27.21,67.74"), "send-time": datetime("2009-11-24T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " like iphone its customization is mind-blowing:)", "countA": 222, "countB": 51 }
+{ "tweetid": 223i64, "user": { "screen-name": "RochelleSaline@265", "lang": "en", "friends-count": 18, "statuses-count": 319, "name": "Rochelle Saline", "followers-count": 11 }, "sender-location": point("33.44,67.08"), "send-time": datetime("2007-10-23T10:10:00.000Z"), "referred-topics": {{ "iphone", "voice-command" }}, "message-text": " can't stand iphone its voice-command is OMG", "countA": 223, "countB": 84 }
+{ "tweetid": 224i64, "user": { "screen-name": "TabbyEckhardstein@204", "lang": "en", "friends-count": 5, "statuses-count": 207, "name": "Tabby Eckhardstein", "followers-count": 5 }, "sender-location": point("24.33,84.9"), "send-time": datetime("2005-02-01T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " hate samsung its speed is horrible", "countA": 224, "countB": 192 }
+{ "tweetid": 225i64, "user": { "screen-name": "FranklynBurns$23", "lang": "en", "friends-count": 73, "statuses-count": 475, "name": "Franklyn Burns", "followers-count": 131 }, "sender-location": point("26.5,94.14"), "send-time": datetime("2007-08-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " love motorola its reachability is good", "countA": 225, "countB": 65 }
+{ "tweetid": 226i64, "user": { "screen-name": "CorianderFischer#204", "lang": "en", "friends-count": 4, "statuses-count": 205, "name": "Coriander Fischer", "followers-count": 30 }, "sender-location": point("46.59,73.39"), "send-time": datetime("2010-03-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola its reachability is horrible", "countA": 226, "countB": 73 }
+{ "tweetid": 227i64, "user": { "screen-name": "EmikoBarth$856", "lang": "en", "friends-count": 28, "statuses-count": 7, "name": "Emiko Barth", "followers-count": 98 }, "sender-location": point("42.43,97.13"), "send-time": datetime("2005-03-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " can't stand verizon the reachability is bad:(", "countA": 227, "countB": 184 }
+{ "tweetid": 228i64, "user": { "screen-name": "LavoneStroh$831", "lang": "en", "friends-count": 53, "statuses-count": 2, "name": "Lavone Stroh", "followers-count": 44 }, "sender-location": point("32.04,66.17"), "send-time": datetime("2011-04-25T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " love t-mobile its touch-screen is awesome:)", "countA": 228, "countB": 38 }
+{ "tweetid": 229i64, "user": { "screen-name": "ShantelLaurence@745", "lang": "en", "friends-count": 78, "statuses-count": 195, "name": "Shantel Laurence", "followers-count": 124 }, "sender-location": point("46.58,97.58"), "send-time": datetime("2007-12-02T10:10:00.000Z"), "referred-topics": {{ "iphone", "wireless" }}, "message-text": " love iphone its wireless is awesome:)", "countA": 229, "countB": 26 }
+{ "tweetid": 230i64, "user": { "screen-name": "PiedadMosser@971", "lang": "en", "friends-count": 84, "statuses-count": 80, "name": "Piedad Mosser", "followers-count": 198 }, "sender-location": point("40.44,70.59"), "send-time": datetime("2005-06-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " hate motorola the wireless is bad", "countA": 230, "countB": 46 }
+{ "tweetid": 231i64, "user": { "screen-name": "CorrieHindman$963", "lang": "en", "friends-count": 54, "statuses-count": 78, "name": "Corrie Hindman", "followers-count": 180 }, "sender-location": point("26.04,77.36"), "send-time": datetime("2012-03-07T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " like sprint its wireless is mind-blowing:)", "countA": 231, "countB": 77 }
+{ "tweetid": 232i64, "user": { "screen-name": "DignaPorter_932", "lang": "en", "friends-count": 63, "statuses-count": 278, "name": "Digna Porter", "followers-count": 139 }, "sender-location": point("41.76,97.51"), "send-time": datetime("2009-12-01T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is mind-blowing:)", "countA": 232, "countB": 85 }
+{ "tweetid": 233i64, "user": { "screen-name": "TerrellStoddard@17", "lang": "en", "friends-count": 74, "statuses-count": 3, "name": "Terrell Stoddard", "followers-count": 35 }, "sender-location": point("34.89,73.83"), "send-time": datetime("2007-01-28T10:10:00.000Z"), "referred-topics": {{ "at&t", "touch-screen" }}, "message-text": " can't stand at&t the touch-screen is terrible", "countA": 233, "countB": 28 }
+{ "tweetid": 234i64, "user": { "screen-name": "EzraField#602", "lang": "en", "friends-count": 7, "statuses-count": 41, "name": "Ezra Field", "followers-count": 169 }, "sender-location": point("26.51,74.01"), "send-time": datetime("2011-03-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " love motorola the speed is good", "countA": 234, "countB": 94 }
+{ "tweetid": 235i64, "user": { "screen-name": "NathanielWentzel$505", "lang": "en", "friends-count": 40, "statuses-count": 178, "name": "Nathaniel Wentzel", "followers-count": 83 }, "sender-location": point("47.03,84.5"), "send-time": datetime("2007-07-02T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-clarity" }}, "message-text": " hate motorola its voice-clarity is bad", "countA": 235, "countB": 53 }
+{ "tweetid": 236i64, "user": { "screen-name": "AlanePycroft$112", "lang": "en", "friends-count": 61, "statuses-count": 222, "name": "Alane Pycroft", "followers-count": 125 }, "sender-location": point("31.82,81.86"), "send-time": datetime("2010-09-09T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is amazing:)", "countA": 236, "countB": 133 }
+{ "tweetid": 237i64, "user": { "screen-name": "EliseoMunson$584", "lang": "en", "friends-count": 21, "statuses-count": 353, "name": "Eliseo Munson", "followers-count": 9 }, "sender-location": point("39.12,74.86"), "send-time": datetime("2011-05-02T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " hate sprint its speed is terrible", "countA": 237, "countB": 100 }
+{ "tweetid": 238i64, "user": { "screen-name": "LupeCram@152", "lang": "en", "friends-count": 89, "statuses-count": 57, "name": "Lupe Cram", "followers-count": 51 }, "sender-location": point("40.34,71.58"), "send-time": datetime("2010-12-13T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is amazing", "countA": 238, "countB": 47 }
+{ "tweetid": 239i64, "user": { "screen-name": "AuroraChristman#544", "lang": "en", "friends-count": 96, "statuses-count": 435, "name": "Aurora Christman", "followers-count": 88 }, "sender-location": point("35.44,67.79"), "send-time": datetime("2009-08-11T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is mind-blowing", "countA": 239, "countB": 114 }
+{ "tweetid": 240i64, "user": { "screen-name": "JermainePotter_329", "lang": "en", "friends-count": 20, "statuses-count": 85, "name": "Jermaine Potter", "followers-count": 177 }, "sender-location": point("35.26,90.47"), "send-time": datetime("2012-04-17T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "network" }}, "message-text": " like t-mobile its network is mind-blowing:)", "countA": 240, "countB": 104 }
+{ "tweetid": 241i64, "user": { "screen-name": "KaetyHall$972", "lang": "en", "friends-count": 23, "statuses-count": 416, "name": "Kaety Hall", "followers-count": 87 }, "sender-location": point("25.06,93.28"), "send-time": datetime("2011-02-13T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " can't stand verizon its network is bad:(", "countA": 241, "countB": 45 }
+{ "tweetid": 242i64, "user": { "screen-name": "LaurenChristopher#195", "lang": "en", "friends-count": 12, "statuses-count": 149, "name": "Lauren Christopher", "followers-count": 183 }, "sender-location": point("24.03,79.65"), "send-time": datetime("2012-05-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " love t-mobile the touch-screen is amazing", "countA": 242, "countB": 179 }
+{ "tweetid": 243i64, "user": { "screen-name": "KoreyBonner$296", "lang": "en", "friends-count": 56, "statuses-count": 412, "name": "Korey Bonner", "followers-count": 190 }, "sender-location": point("42.54,97.81"), "send-time": datetime("2006-10-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "touch-screen" }}, "message-text": " like iphone its touch-screen is amazing:)", "countA": 243, "countB": 176 }
+{ "tweetid": 244i64, "user": { "screen-name": "LaetitiaWise@568", "lang": "en", "friends-count": 87, "statuses-count": 363, "name": "Laetitia Wise", "followers-count": 49 }, "sender-location": point("39.34,70.46"), "send-time": datetime("2010-11-25T10:10:00.000Z"), "referred-topics": {{ "iphone", "voicemail-service" }}, "message-text": " hate iphone its voicemail-service is terrible", "countA": 244, "countB": 150 }
+{ "tweetid": 245i64, "user": { "screen-name": "PatricaKellogg@123", "lang": "en", "friends-count": 4, "statuses-count": 80, "name": "Patrica Kellogg", "followers-count": 47 }, "sender-location": point("34.06,84.02"), "send-time": datetime("2005-05-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint its touch-screen is bad:(", "countA": 245, "countB": 83 }
+{ "tweetid": 246i64, "user": { "screen-name": "AdelleSwink$50", "lang": "en", "friends-count": 64, "statuses-count": 411, "name": "Adelle Swink", "followers-count": 165 }, "sender-location": point("28.66,71.42"), "send-time": datetime("2007-04-11T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " can't stand sprint the plan is horrible", "countA": 246, "countB": 128 }
+{ "tweetid": 247i64, "user": { "screen-name": "MckennaAlbright$534", "lang": "en", "friends-count": 94, "statuses-count": 374, "name": "Mckenna Albright", "followers-count": 61 }, "sender-location": point("45.26,93.62"), "send-time": datetime("2011-11-12T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " can't stand sprint the speed is OMG", "countA": 247, "countB": 194 }
+{ "tweetid": 248i64, "user": { "screen-name": "MerryReade$689", "lang": "en", "friends-count": 41, "statuses-count": 172, "name": "Merry Reade", "followers-count": 105 }, "sender-location": point("42.21,81.81"), "send-time": datetime("2009-08-18T10:10:00.000Z"), "referred-topics": {{ "verizon", "wireless" }}, "message-text": " like verizon its wireless is amazing", "countA": 248, "countB": 123 }
+{ "tweetid": 249i64, "user": { "screen-name": "CoryGoldvogel$187", "lang": "en", "friends-count": 91, "statuses-count": 359, "name": "Cory Goldvogel", "followers-count": 20 }, "sender-location": point("39.94,83.42"), "send-time": datetime("2005-09-11T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " dislike verizon its plan is bad:(", "countA": 249, "countB": 163 }
+{ "tweetid": 250i64, "user": { "screen-name": "RoystonRummel@500", "lang": "en", "friends-count": 34, "statuses-count": 154, "name": "Royston Rummel", "followers-count": 146 }, "sender-location": point("40.73,72.93"), "send-time": datetime("2005-11-20T10:10:00.000Z"), "referred-topics": {{ "samsung", "touch-screen" }}, "message-text": " love samsung its touch-screen is amazing:)", "countA": 250, "countB": 125 }
\ No newline at end of file
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
index 50adb57..523c59c 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
@@ -31,7 +31,7 @@
 
     protected List<Statement.Kind> getAllowedStatements() {
         Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPDATE, Kind.DML_CMD_LIST,
-                Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT };
+                Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT, Kind.EXTERNAL_DATASET_REFRESH };
         return Arrays.asList(statementsArray);
     }
 
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 35518a6..ef823ff 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -19,8 +19,10 @@
 import java.nio.ByteBuffer;
 import java.rmi.RemoteException;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -61,12 +63,16 @@
 import edu.uci.ics.asterix.aql.expression.NodeGroupDropStatement;
 import edu.uci.ics.asterix.aql.expression.NodegroupDecl;
 import edu.uci.ics.asterix.aql.expression.Query;
+import edu.uci.ics.asterix.aql.expression.RefreshExternalDatasetStatement;
 import edu.uci.ics.asterix.aql.expression.SetStatement;
 import edu.uci.ics.asterix.aql.expression.TypeDecl;
 import edu.uci.ics.asterix.aql.expression.TypeDropStatement;
 import edu.uci.ics.asterix.aql.expression.WriteStatement;
 import edu.uci.ics.asterix.aql.util.FunctionUtils;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.common.config.GlobalConfig;
 import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
@@ -75,6 +81,7 @@
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.file.DatasetOperations;
 import edu.uci.ics.asterix.file.DataverseOperations;
+import edu.uci.ics.asterix.file.ExternalIndexingOperations;
 import edu.uci.ics.asterix.file.FeedOperations;
 import edu.uci.ics.asterix.file.IndexOperations;
 import edu.uci.ics.asterix.formats.base.IDataFormat;
@@ -92,6 +99,7 @@
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
 import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
 import edu.uci.ics.asterix.metadata.entities.Feed;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity;
 import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
@@ -101,6 +109,7 @@
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
 import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
 import edu.uci.ics.asterix.metadata.feeds.FeedUtil;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -308,6 +317,11 @@
                     break;
                 }
 
+                case EXTERNAL_DATASET_REFRESH: {
+                    handleExternalDatasetRefreshStatement(metadataProvider, stmt, hcc);
+                    break;
+                }
+
                 case WRITE: {
                     Pair<IAWriterFactory, FileSplit> result = handleWriteStatement(metadataProvider, stmt);
                     if (result.first != null) {
@@ -480,7 +494,20 @@
                 case EXTERNAL: {
                     String adapter = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getAdapter();
                     Map<String, String> properties = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getProperties();
-                    datasetDetails = new ExternalDatasetDetails(adapter, properties);
+                    Identifier ngNameId = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getNodegroupName();
+                    String ngName = ngNameId != null ? ngNameId.getValue() : configureNodegroupForDataset(dd,
+                            dataverseName, mdTxnCtx);
+                    String compactionPolicy = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getCompactionPolicy();
+                    Map<String, String> compactionPolicyProperties = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl())
+                            .getCompactionPolicyProperties();
+                    if (compactionPolicy == null) {
+                        compactionPolicy = GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME;
+                        compactionPolicyProperties = GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES;
+                    } else {
+                        validateCompactionPolicy(compactionPolicy, compactionPolicyProperties, mdTxnCtx);
+                    }
+                    datasetDetails = new ExternalDatasetDetails(adapter, properties, ngName, new Date(),
+                            ExternalDatasetTransactionState.COMMIT, compactionPolicy, compactionPolicyProperties);
                     break;
                 }
 
@@ -625,7 +652,6 @@
 
     private void handleCreateIndexStatement(AqlMetadataProvider metadataProvider, Statement stmt,
             IHyracksClientConnection hcc) throws Exception {
-
         ProgressState progress = ProgressState.NO_PROGRESS;
         MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
         boolean bActiveTxn = true;
@@ -637,6 +663,12 @@
         String indexName = null;
         JobSpecification spec = null;
         Dataset ds = null;
+        // For external datasets
+        ArrayList<ExternalFile> externalFilesSnapshot = null;
+        boolean firstExternalDatasetIndex = false;
+        boolean filesIndexReplicated = false;
+        Index filesIndex = null;
+        boolean datasetLocked = false;
         try {
             CreateIndexStatement stmtCreateIndex = (CreateIndexStatement) stmt;
             dataverseName = getActiveDataverseName(stmtCreateIndex.getDataverseName());
@@ -669,17 +701,61 @@
                 }
             }
 
-            List<FeedActivity> feedActivities = MetadataManager.INSTANCE.getActiveFeeds(mdTxnCtx, dataverseName,
-                    datasetName);
-            if (feedActivities != null && !feedActivities.isEmpty()) {
-                StringBuilder builder = new StringBuilder();
+            if (ds.getDatasetType() == DatasetType.INTERNAL) {
+                List<FeedActivity> feedActivities = MetadataManager.INSTANCE.getActiveFeeds(mdTxnCtx, dataverseName,
+                        datasetName);
+                if (feedActivities != null && !feedActivities.isEmpty()) {
+                    StringBuilder builder = new StringBuilder();
 
-                for (FeedActivity fa : feedActivities) {
-                    builder.append(fa + "\n");
+                    for (FeedActivity fa : feedActivities) {
+                        builder.append(fa + "\n");
+                    }
+                    throw new AsterixException("Dataset" + datasetName
+                            + " is currently being fed into by the following feeds " + "." + builder.toString()
+                            + "\nOperation not supported.");
                 }
-                throw new AsterixException("Dataset" + datasetName
-                        + " is currently being fed into by the following feeds " + "." + builder.toString()
-                        + "\nOperation not supported.");
+
+            } else {
+                // External dataset
+                // Check if the dataset is indexible
+                if (!ExternalIndexingOperations.isIndexible((ExternalDatasetDetails) ds.getDatasetDetails())) {
+                    throw new AlgebricksException("dataset using "
+                            + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
+                            + " Adapter can't be indexed");
+                }
+                // check if the name of the index is valid
+                if (!ExternalIndexingOperations.isValidIndexName(datasetName, indexName)) {
+                    throw new AlgebricksException("external dataset index name is invalid");
+                }
+                // lock external dataset
+                ExternalDatasetsRegistry.INSTANCE.buildIndexBegin(ds);
+                datasetLocked = true;
+                // Check if the files index exist
+                filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                        datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+                firstExternalDatasetIndex = (filesIndex == null);
+                if (firstExternalDatasetIndex) {
+                    // Get snapshot from External File System
+                    externalFilesSnapshot = ExternalIndexingOperations.getSnapshotFromExternalFileSystem(ds);
+                    // Add an entry for the files index
+                    filesIndex = new Index(dataverseName, datasetName,
+                            ExternalIndexingOperations.getFilesIndexName(datasetName), IndexType.BTREE,
+                            ExternalIndexingOperations.FILE_INDEX_FIELDS, false, IMetadataEntity.PENDING_ADD_OP);
+                    MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
+                    // Add files to the external files index
+                    for (ExternalFile file : externalFilesSnapshot) {
+                        MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+                    }
+                    // This is the first index for the external dataset, replicate the files index
+                    spec = ExternalIndexingOperations.buildFilesIndexReplicationJobSpec(ds, externalFilesSnapshot,
+                            metadataProvider, true);
+                    if (spec == null) {
+                        throw new AsterixException(
+                                "Failed to create job spec for replicating Files Index For external dataset");
+                    }
+                    filesIndexReplicated = true;
+                    runJob(hcc, spec, true);
+                }
             }
 
             //#. add a new index with PendingAddOp
@@ -726,12 +802,41 @@
                     indexName);
             index.setPendingOp(IMetadataEntity.PENDING_NO_OP);
             MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
+            // add another new files index with PendingNoOp after deleting the index with PendingAddOp
+            if (firstExternalDatasetIndex) {
+                MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                        datasetName, filesIndex.getIndexName());
+                filesIndex.setPendingOp(IMetadataEntity.PENDING_NO_OP);
+                MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
+                // update transaction timestamp
+                ((ExternalDatasetDetails) ds.getDatasetDetails()).setRefreshTimestamp(new Date());
+                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
+            }
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
 
         } catch (Exception e) {
             if (bActiveTxn) {
                 abort(e, e, mdTxnCtx);
             }
+            // If files index was replicated for external dataset, it should be cleaned up on NC side
+            if (filesIndexReplicated) {
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                        ExternalIndexingOperations.getFilesIndexName(datasetName));
+                try {
+                    JobSpecification jobSpec = ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+                            metadataProvider, ds);
+                    MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    bActiveTxn = false;
+                    runJob(hcc, jobSpec, true);
+                } catch (Exception e2) {
+                    e.addSuppressed(e2);
+                    if (bActiveTxn) {
+                        abort(e, e2, mdTxnCtx);
+                    }
+                }
+            }
 
             if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
                 //#. execute compensation operations
@@ -754,7 +859,35 @@
                     }
                 }
 
-                //   remove the record from the metadata.
+                if (firstExternalDatasetIndex) {
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                    try {
+                        // Drop External Files from metadata
+                        MetadataManager.INSTANCE.dropDatasetExternalFiles(mdTxnCtx, ds);
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    } catch (Exception e2) {
+                        e.addSuppressed(e2);
+                        abort(e, e2, mdTxnCtx);
+                        throw new IllegalStateException("System is inconsistent state: pending files for("
+                                + dataverseName + "." + datasetName + ") couldn't be removed from the metadata", e);
+                    }
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                    try {
+                        // Drop the files index from metadata
+                        MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                                datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    } catch (Exception e2) {
+                        e.addSuppressed(e2);
+                        abort(e, e2, mdTxnCtx);
+                        throw new IllegalStateException("System is inconsistent state: pending index(" + dataverseName
+                                + "." + datasetName + "." + ExternalIndexingOperations.getFilesIndexName(datasetName)
+                                + ") couldn't be removed from the metadata", e);
+                    }
+                }
+                // remove the record from the metadata.
                 mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
                 metadataProvider.setMetadataTxnContext(mdTxnCtx);
                 try {
@@ -764,13 +897,16 @@
                 } catch (Exception e2) {
                     e.addSuppressed(e2);
                     abort(e, e2, mdTxnCtx);
-                    throw new IllegalStateException("System is inconsistent state: pending index(" + dataverseName
+                    throw new IllegalStateException("System is in inconsistent state: pending index(" + dataverseName
                             + "." + datasetName + "." + indexName + ") couldn't be removed from the metadata", e);
                 }
             }
             throw e;
         } finally {
             releaseWriteLatch();
+            if (datasetLocked) {
+                ExternalDatasetsRegistry.INSTANCE.buildIndexEnd(ds);
+            }
         }
     }
 
@@ -864,7 +1000,6 @@
                 String datasetName = datasets.get(j).getDatasetName();
                 DatasetType dsType = datasets.get(j).getDatasetType();
                 if (dsType == DatasetType.INTERNAL) {
-
                     List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
                             datasetName);
                     for (int k = 0; k < indexes.size(); k++) {
@@ -878,6 +1013,23 @@
 
                     CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
                     jobsToExecute.add(DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider));
+                } else {
+                    // External dataset
+                    List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
+                            datasetName);
+                    for (int k = 0; k < indexes.size(); k++) {
+                        if (ExternalIndexingOperations.isFileIndex(indexes.get(k))) {
+                            CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                    indexes.get(k).getIndexName());
+                            jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+                                    metadataProvider, datasets.get(j)));
+                        } else {
+                            CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                    indexes.get(k).getIndexName());
+                            jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider,
+                                    datasets.get(j)));
+                        }
+                    }
                 }
             }
             jobsToExecute.add(DataverseOperations.createDropDataverseJobSpec(dv, metadataProvider));
@@ -906,7 +1058,7 @@
             if (activeDefaultDataverse != null && activeDefaultDataverse.getDataverseName() == dataverseName) {
                 activeDefaultDataverse = null;
             }
-
+            ExternalDatasetsRegistry.INSTANCE.removeDataverse(dv);
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
             if (bActiveTxn) {
@@ -1029,16 +1181,52 @@
                 mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
                 bActiveTxn = true;
                 metadataProvider.setMetadataTxnContext(mdTxnCtx);
+            } else {
+                // External dataset
+                //#. prepare jobs to drop the datatset and the indexes in NC
+                List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
+                for (int j = 0; j < indexes.size(); j++) {
+                    if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
+                        CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                indexes.get(j).getIndexName());
+                        jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
+                    } else {
+                        CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                indexes.get(j).getIndexName());
+                        jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds, metadataProvider,
+                                ds));
+                    }
+                }
+
+                //#. mark the existing dataset as PendingDropOp
+                MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
+                MetadataManager.INSTANCE.addDataset(
+                        mdTxnCtx,
+                        new Dataset(dataverseName, datasetName, ds.getItemTypeName(), ds.getDatasetDetails(), ds
+                                .getHints(), ds.getDatasetType(), ds.getDatasetId(), IMetadataEntity.PENDING_DROP_OP));
+
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                bActiveTxn = false;
+                progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+
+                //#. run the jobs
+                for (JobSpecification jobSpec : jobsToExecute) {
+                    runJob(hcc, jobSpec, true);
+                }
+                if (indexes.size() > 0) {
+                    ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
+                }
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                metadataProvider.setMetadataTxnContext(mdTxnCtx);
             }
 
             //#. finally, delete the dataset.
             MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
             // Drop the associated nodegroup
-            if (ds.getDatasetType() == DatasetType.INTERNAL) {
-                String nodegroup = ((InternalDatasetDetails) ds.getDatasetDetails()).getNodeGroupName();
-                if (!nodegroup.equalsIgnoreCase(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)) {
-                    MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, dataverseName + ":" + datasetName);
-                }
+            String nodegroup = ds.getDatasetDetails().getNodeGroupName();
+            if (!nodegroup.equalsIgnoreCase(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)) {
+                MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, dataverseName + ":" + datasetName);
             }
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1092,6 +1280,7 @@
         String dataverseName = null;
         String datasetName = null;
         String indexName = null;
+        boolean dropFilesIndex = false;
         List<JobSpecification> jobsToExecute = new ArrayList<JobSpecification>();
         try {
             IndexDropStatement stmtIndexDrop = (IndexDropStatement) stmt;
@@ -1155,8 +1344,74 @@
                 //#. finally, delete the existing index
                 MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
             } else {
-                throw new AlgebricksException(datasetName
-                        + " is an external dataset. Indexes are not maintained for external datasets.");
+                // External dataset
+                indexName = stmtIndexDrop.getIndexName().getValue();
+                Index index = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, indexName);
+                if (index == null) {
+                    if (stmtIndexDrop.getIfExists()) {
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                        return;
+                    } else {
+                        throw new AlgebricksException("There is no index with this name " + indexName + ".");
+                    }
+                } else if (ExternalIndexingOperations.isFileIndex(index)) {
+                    throw new AlgebricksException("Dropping a dataset's files index is not allowed.");
+                }
+                //#. prepare a job to drop the index in NC.
+                CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName, indexName);
+                jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
+                List<Index> datasetIndexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
+                        datasetName);
+                if (datasetIndexes.size() == 2) {
+                    dropFilesIndex = true;
+                    // only one index + the files index, we need to delete both of the indexes
+                    for (Index externalIndex : datasetIndexes) {
+                        if (ExternalIndexingOperations.isFileIndex(externalIndex)) {
+                            cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                    externalIndex.getIndexName());
+                            jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+                                    metadataProvider, ds));
+                            //#. mark PendingDropOp on the existing files index
+                            MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName,
+                                    externalIndex.getIndexName());
+                            MetadataManager.INSTANCE.addIndex(
+                                    mdTxnCtx,
+                                    new Index(dataverseName, datasetName, externalIndex.getIndexName(), externalIndex
+                                            .getIndexType(), externalIndex.getKeyFieldNames(), externalIndex
+                                            .isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
+                        }
+                    }
+                }
+
+                //#. mark PendingDropOp on the existing index
+                MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
+                MetadataManager.INSTANCE.addIndex(mdTxnCtx,
+                        new Index(dataverseName, datasetName, indexName, index.getIndexType(),
+                                index.getKeyFieldNames(), index.isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
+
+                //#. commit the existing transaction before calling runJob. 
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                bActiveTxn = false;
+                progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+
+                for (JobSpecification jobSpec : jobsToExecute) {
+                    runJob(hcc, jobSpec, true);
+                }
+
+                //#. begin a new transaction
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+                //#. finally, delete the existing index
+                MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
+                if (dropFilesIndex) {
+                    // delete the files index too
+                    MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName,
+                            ExternalIndexingOperations.getFilesIndexName(datasetName));
+                    MetadataManager.INSTANCE.dropDatasetExternalFiles(mdTxnCtx, ds);
+                    ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
+                }
             }
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
 
@@ -1183,6 +1438,10 @@
                 try {
                     MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
                             datasetName, indexName);
+                    if (dropFilesIndex) {
+                        MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                                datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+                    }
                     MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
                 } catch (Exception e2) {
                     e.addSuppressed(e2);
@@ -1673,29 +1932,41 @@
             CompactStatement compactStatement = (CompactStatement) stmt;
             dataverseName = getActiveDataverseName(compactStatement.getDataverseName());
             datasetName = compactStatement.getDatasetName().getValue();
-
             Dataset ds = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
             if (ds == null) {
                 throw new AlgebricksException("There is no dataset with this name " + datasetName + " in dataverse "
                         + dataverseName + ".");
-            } else if (ds.getDatasetType() != DatasetType.INTERNAL) {
-                throw new AlgebricksException("Cannot compact the extrenal dataset " + datasetName + ".");
             }
-
-            // Prepare jobs to compact the datatset and its indexes
             List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
-            for (int j = 0; j < indexes.size(); j++) {
-                if (indexes.get(j).isSecondaryIndex()) {
-                    CompiledIndexCompactStatement cics = new CompiledIndexCompactStatement(dataverseName, datasetName,
-                            indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes.get(j)
-                                    .getGramLength(), indexes.get(j).getIndexType());
-                    jobsToExecute.add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, metadataProvider, ds));
-                }
+            if (indexes.size() == 0) {
+                throw new AlgebricksException("Cannot compact the extrenal dataset " + datasetName
+                        + " because it has no indexes");
             }
-            Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
-                    dataverseName);
-            jobsToExecute.add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider));
-
+            if (ds.getDatasetType() == DatasetType.INTERNAL) {
+                for (int j = 0; j < indexes.size(); j++) {
+                    if (indexes.get(j).isSecondaryIndex()) {
+                        CompiledIndexCompactStatement cics = new CompiledIndexCompactStatement(dataverseName,
+                                datasetName, indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes
+                                        .get(j).getGramLength(), indexes.get(j).getIndexType());
+                        jobsToExecute
+                                .add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, metadataProvider, ds));
+                    }
+                }
+                Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
+                        dataverseName);
+                jobsToExecute.add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider));
+            } else {
+                for (int j = 0; j < indexes.size(); j++) {
+                    if (!ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
+                        CompiledIndexCompactStatement cics = new CompiledIndexCompactStatement(dataverseName,
+                                datasetName, indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes
+                                        .get(j).getGramLength(), indexes.get(j).getIndexType());
+                        jobsToExecute
+                                .add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, metadataProvider, ds));
+                    }
+                }
+                jobsToExecute.add(ExternalIndexingOperations.compactFilesIndexJobSpec(ds, metadataProvider));
+            }
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
 
@@ -1720,9 +1991,9 @@
         boolean bActiveTxn = true;
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
         acquireReadLatch();
-
+        JobSpecification compiled = null;
         try {
-            JobSpecification compiled = rewriteCompileQuery(metadataProvider, query, null);
+            compiled = rewriteCompileQuery(metadataProvider, query, null);
 
             QueryResult queryResult = new QueryResult(query, metadataProvider.getResultSetId());
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1786,6 +2057,8 @@
             throw e;
         } finally {
             releaseReadLatch();
+            // release locks aquired during compilation of the query
+            ExternalDatasetsRegistry.INSTANCE.releaseAcquiredLocks(metadataProvider);
         }
     }
 
@@ -1819,6 +2092,232 @@
         }
     }
 
+    private void handleExternalDatasetRefreshStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+            IHyracksClientConnection hcc) throws Exception {
+        RefreshExternalDatasetStatement stmtRefresh = (RefreshExternalDatasetStatement) stmt;
+        ExternalDatasetTransactionState transactionState = ExternalDatasetTransactionState.COMMIT;
+        MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+        acquireWriteLatch();
+        boolean bActiveTxn = true;
+        metadataProvider.setMetadataTxnContext(mdTxnCtx);
+        String dataverseName = null;
+        String datasetName = null;
+        JobSpecification spec = null;
+        Dataset ds = null;
+        List<ExternalFile> metadataFiles = null;
+        List<ExternalFile> deletedFiles = null;
+        List<ExternalFile> addedFiles = null;
+        List<ExternalFile> appendedFiles = null;
+        List<Index> indexes = null;
+        Dataset transactionDataset = null;
+        boolean lockAquired = false;
+        boolean success = false;
+        try {
+            dataverseName = getActiveDataverseName(stmtRefresh.getDataverseName());
+            datasetName = stmtRefresh.getDatasetName().getValue();
+            ds = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
+                    datasetName);
+
+            // Dataset exists ?
+            if (ds == null) {
+                throw new AlgebricksException("There is no dataset with this name " + datasetName + " in dataverse "
+                        + dataverseName);
+            }
+            // Dataset external ?
+            if (ds.getDatasetType() != DatasetType.EXTERNAL) {
+                throw new AlgebricksException("dataset " + datasetName + " in dataverse " + dataverseName
+                        + " is not an external dataset");
+            }
+            // Dataset has indexes ?
+            indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
+            if (indexes.size() == 0) {
+                throw new AlgebricksException("External dataset " + datasetName + " in dataverse " + dataverseName
+                        + " doesn't have any index");
+            }
+
+            // Record transaction time
+            Date txnTime = new Date();
+
+            // refresh lock here
+            ExternalDatasetsRegistry.INSTANCE.refreshBegin(ds);
+            lockAquired = true;
+
+            // Get internal files
+            metadataFiles = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, ds);
+            deletedFiles = new ArrayList<ExternalFile>();
+            addedFiles = new ArrayList<ExternalFile>();
+            appendedFiles = new ArrayList<ExternalFile>();
+
+            // Compute delta
+            // Now we compare snapshot with external file system
+            if (ExternalIndexingOperations
+                    .isDatasetUptodate(ds, metadataFiles, addedFiles, deletedFiles, appendedFiles)) {
+                ((ExternalDatasetDetails) ds.getDatasetDetails()).setRefreshTimestamp(txnTime);
+                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                // latch will be released in the finally clause
+                return;
+            }
+
+            // At this point, we know data has changed in the external file system, record transaction in metadata and start            
+            transactionDataset = ExternalIndexingOperations.createTransactionDataset(ds);
+            /*
+             * Remove old dataset record and replace it with a new one
+             */
+            MetadataManager.INSTANCE.updateDataset(mdTxnCtx, transactionDataset);
+
+            // Add delta files to the metadata
+            for (ExternalFile file : addedFiles) {
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+            for (ExternalFile file : appendedFiles) {
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+            for (ExternalFile file : deletedFiles) {
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+
+            // Create the files index update job
+            spec = ExternalIndexingOperations.buildFilesIndexUpdateOp(ds, metadataFiles, deletedFiles, addedFiles,
+                    appendedFiles, metadataProvider);
+
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            bActiveTxn = false;
+            transactionState = ExternalDatasetTransactionState.BEGIN;
+
+            //run the files update job
+            runJob(hcc, spec, true);
+
+            for (Index index : indexes) {
+                if (!ExternalIndexingOperations.isFileIndex(index)) {
+                    spec = ExternalIndexingOperations.buildIndexUpdateOp(ds, index, metadataFiles, deletedFiles,
+                            addedFiles, appendedFiles, metadataProvider);
+                    //run the files update job
+                    runJob(hcc, spec, true);
+                }
+            }
+
+            // all index updates has completed successfully, record transaction state
+            spec = ExternalIndexingOperations.buildCommitJob(ds, indexes, metadataProvider);
+
+            // Aquire write latch again -> start a transaction and record the decision to commit
+            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+            metadataProvider.setMetadataTxnContext(mdTxnCtx);
+            bActiveTxn = true;
+            ((ExternalDatasetDetails) transactionDataset.getDatasetDetails())
+                    .setState(ExternalDatasetTransactionState.READY_TO_COMMIT);
+            ((ExternalDatasetDetails) transactionDataset.getDatasetDetails()).setRefreshTimestamp(txnTime);
+            MetadataManager.INSTANCE.updateDataset(mdTxnCtx, transactionDataset);
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            bActiveTxn = false;
+            transactionState = ExternalDatasetTransactionState.READY_TO_COMMIT;
+            // We don't release the latch since this job is expected to be quick
+            runJob(hcc, spec, true);
+            // Start a new metadata transaction to record the final state of the transaction
+            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+            metadataProvider.setMetadataTxnContext(mdTxnCtx);
+            bActiveTxn = true;
+
+            for (ExternalFile file : metadataFiles) {
+                if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP) {
+                    MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP) {
+                    Iterator<ExternalFile> iterator = appendedFiles.iterator();
+                    while (iterator.hasNext()) {
+                        ExternalFile appendedFile = iterator.next();
+                        if (file.getFileName().equals(appendedFile.getFileName())) {
+                            // delete existing file
+                            MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                            // delete existing appended file
+                            MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, appendedFile);
+                            // add the original file with appended information
+                            appendedFile.setFileNumber(file.getFileNumber());
+                            appendedFile.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                            MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, appendedFile);
+                            iterator.remove();
+                        }
+                    }
+                }
+            }
+
+            // remove the deleted files delta
+            for (ExternalFile file : deletedFiles) {
+                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+            }
+
+            // insert new files
+            for (ExternalFile file : addedFiles) {
+                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                file.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+
+            // mark the transaction as complete
+            ((ExternalDatasetDetails) transactionDataset.getDatasetDetails())
+                    .setState(ExternalDatasetTransactionState.COMMIT);
+            MetadataManager.INSTANCE.updateDataset(mdTxnCtx, transactionDataset);
+
+            // commit metadata transaction
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            success = true;
+        } catch (Exception e) {
+            if (bActiveTxn) {
+                abort(e, e, mdTxnCtx);
+            }
+            if (transactionState == ExternalDatasetTransactionState.READY_TO_COMMIT) {
+                throw new IllegalStateException("System is inconsistent state: commit of (" + dataverseName + "."
+                        + datasetName + ") refresh couldn't carry out the commit phase", e);
+            }
+            if (transactionState == ExternalDatasetTransactionState.COMMIT) {
+                // Nothing to do , everything should be clean
+                throw e;
+            }
+            if (transactionState == ExternalDatasetTransactionState.BEGIN) {
+                // transaction failed, need to do the following
+                // clean NCs removing transaction components
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                spec = ExternalIndexingOperations.buildAbortOp(ds, indexes, metadataProvider);
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                bActiveTxn = false;
+                try {
+                    runJob(hcc, spec, true);
+                } catch (Exception e2) {
+                    // This should never happen -- fix throw illegal
+                    e.addSuppressed(e2);
+                    throw new IllegalStateException("System is in inconsistent state. Failed to abort refresh", e);
+                }
+                // remove the delta of files
+                // return the state of the dataset to committed
+                try {
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    for (ExternalFile file : deletedFiles) {
+                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                    }
+                    for (ExternalFile file : addedFiles) {
+                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                    }
+                    for (ExternalFile file : appendedFiles) {
+                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                    }
+                    MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
+                    // commit metadata transaction
+                    MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                } catch (Exception e2) {
+                    abort(e, e2, mdTxnCtx);
+                    e.addSuppressed(e2);
+                    throw new IllegalStateException("System is in inconsistent state. Failed to drop delta files", e);
+                }
+            }
+        } finally {
+            releaseWriteLatch();
+            if (lockAquired) {
+                ExternalDatasetsRegistry.INSTANCE.refreshEnd(ds, success);
+            }
+        }
+    }
+
     private JobId runJob(IHyracksClientConnection hcc, JobSpecification spec, boolean waitForCompletion)
             throws Exception {
         spec.setFrameSize(OptimizationConfUtil.getPhysicalOptimizationConfig().getFrameSize());
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index ac0b919..19283ff 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -26,6 +26,7 @@
 import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
 import edu.uci.ics.asterix.common.config.GlobalConfig;
 import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -123,7 +124,7 @@
         JobSpecification specPrimary = JobSpecificationUtils.createJobSpecification();
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataset.getDataverseName(), datasetName,
+                .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName,
                         datasetName);
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
@@ -164,7 +165,7 @@
         int[] blooFilterKeyFields = DatasetUtils.createBloomFilterKeyFields(dataset);
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
         FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
         StringBuilder sb = new StringBuilder();
         for (int i = 0; i < fs.length; i++) {
@@ -225,10 +226,10 @@
         int[] blooFilterKeyFields = DatasetUtils.createBloomFilterKeyFields(dataset);
 
         ExternalDatasetDetails externalDatasetDetails = new ExternalDatasetDetails(loadStmt.getAdapter(),
-                loadStmt.getProperties());
+                loadStmt.getProperties(), null, null, ExternalDatasetTransactionState.COMMIT, null, null);
 
         Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = metadataProvider.buildExternalDataScannerRuntime(
-                spec, itemType, externalDatasetDetails, format);
+                spec, itemType, externalDatasetDetails, format, dataset);
         IOperatorDescriptor scanner = p.first;
         AlgebricksPartitionConstraint scannerPc = p.second;
         RecordDescriptor recDesc = computePayloadKeyRecordDescriptor(dataset, itemType, payloadSerde, format);
@@ -253,7 +254,7 @@
         fieldPermutation[numKeys] = 0;
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
 
         FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
         StringBuilder sb = new StringBuilder();
@@ -389,7 +390,7 @@
         int[] blooFilterKeyFields = DatasetUtils.createBloomFilterKeyFields(dataset);
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
 
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
 
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
new file mode 100644
index 0000000..bcd9ff9
--- /dev/null
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
@@ -0,0 +1,750 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.file;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
+import edu.uci.ics.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
+import edu.uci.ics.asterix.external.adapter.factory.HDFSAdapterFactory;
+import edu.uci.ics.asterix.external.adapter.factory.HDFSIndexingAdapterFactory;
+import edu.uci.ics.asterix.external.adapter.factory.HiveAdapterFactory;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalDatasetIndexesAbortOperatorDescriptor;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalDatasetIndexesCommitOperatorDescriptor;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalDatasetIndexesRecoverOperatorDescriptor;
+import edu.uci.ics.asterix.external.indexing.operators.IndexInfoOperatorDescriptor;
+import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
+import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.external.FilesIndexDescription;
+import edu.uci.ics.asterix.metadata.external.IndexingConstants;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
+import edu.uci.ics.asterix.tools.external.data.ExternalFilesIndexOperatorDescriptor;
+import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
+import edu.uci.ics.asterix.transaction.management.resource.ExternalBTreeLocalResourceMetadata;
+import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
+import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
+import edu.uci.ics.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
+import edu.uci.ics.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import edu.uci.ics.hyracks.storage.common.file.LocalResource;
+
+public class ExternalIndexingOperations {
+
+    public static final ArrayList<String> FILE_INDEX_FIELDS = new ArrayList<String>();
+    static {
+        FILE_INDEX_FIELDS.add("");
+    }
+
+    public static boolean isIndexible(ExternalDatasetDetails ds) {
+        String adapter = ds.getAdapter();
+        if (adapter.equalsIgnoreCase("hdfs") || adapter.equalsIgnoreCase("hive")
+                || adapter.equalsIgnoreCase("edu.uci.ics.asterix.external.dataset.adapter.HDFSAdapter")
+                || adapter.equalsIgnoreCase("edu.uci.ics.asterix.external.dataset.adapter.HIVEAdapter")) {
+            return true;
+        }
+        return false;
+    }
+
+    public static boolean isRefereshActive(ExternalDatasetDetails ds) {
+        return ds.getState() != ExternalDatasetTransactionState.COMMIT;
+    }
+
+    public static boolean datasetUsesHiveAdapter(ExternalDatasetDetails ds) {
+        String adapter = ds.getAdapter();
+        return (adapter.equalsIgnoreCase("hive") || adapter
+                .equalsIgnoreCase("edu.uci.ics.asterix.external.dataset.adapter.HIVEAdapter"));
+    }
+
+    public static boolean isValidIndexName(String datasetName, String indexName) {
+        return (!datasetName.concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX).equals(indexName));
+    }
+
+    public static String getFilesIndexName(String datasetName) {
+        return datasetName.concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX);
+    }
+
+    public static int getRIDSize(Dataset dataset) {
+        ExternalDatasetDetails dsd = ((ExternalDatasetDetails) dataset.getDatasetDetails());
+        return IndexingConstants.getRIDSize(dsd.getProperties().get(IndexingConstants.KEY_INPUT_FORMAT));
+    }
+
+    public static IBinaryComparatorFactory[] getComparatorFactories(Dataset dataset) {
+        ExternalDatasetDetails dsd = ((ExternalDatasetDetails) dataset.getDatasetDetails());
+        return IndexingConstants.getComparatorFactories((dsd.getProperties().get(IndexingConstants.KEY_INPUT_FORMAT)));
+    }
+
+    public static IBinaryComparatorFactory[] getBuddyBtreeComparatorFactories() {
+        return IndexingConstants.getBuddyBtreeComparatorFactories();
+    }
+
+    public static ArrayList<ExternalFile> getSnapshotFromExternalFileSystem(Dataset dataset) throws AlgebricksException {
+        ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
+        ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
+        try {
+            // Create the file system object
+            FileSystem fs = getFileSystemObject(datasetDetails.getProperties());
+            // If dataset uses hive adapter, add path to the dataset properties
+            if (datasetUsesHiveAdapter(datasetDetails)) {
+                HiveAdapterFactory.populateConfiguration(datasetDetails.getProperties());
+            }
+            // Get paths of dataset
+            String path = datasetDetails.getProperties().get(HDFSAdapterFactory.KEY_PATH);
+            String[] paths = path.split(",");
+
+            // Add fileStatuses to files
+            for (String aPath : paths) {
+                FileStatus[] fileStatuses = fs.listStatus(new Path(aPath));
+                for (int i = 0; i < fileStatuses.length; i++) {
+                    int nextFileNumber = files.size();
+                    if (fileStatuses[i].isDir()) {
+                        listSubFiles(dataset, fs, fileStatuses[i], files);
+                    } else {
+                        files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(),
+                                nextFileNumber, fileStatuses[i].getPath().toUri().getPath(), new Date(fileStatuses[i]
+                                        .getModificationTime()), fileStatuses[i].getLen(),
+                                ExternalFilePendingOp.PENDING_NO_OP));
+                    }
+                }
+            }
+            // Close file system
+            fs.close();
+            if (files.size() == 0) {
+                throw new AlgebricksException("File Snapshot retrieved from external file system is empty");
+            }
+            return files;
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new AlgebricksException("Unable to get list of HDFS files " + e);
+        }
+    }
+
+    /* list all files under the directory
+     * src is expected to be a folder
+     */
+    private static void listSubFiles(Dataset dataset, FileSystem srcFs, FileStatus src, ArrayList<ExternalFile> files)
+            throws IOException {
+        Path path = src.getPath();
+        FileStatus[] fileStatuses = srcFs.listStatus(path);
+        for (int i = 0; i < fileStatuses.length; i++) {
+            int nextFileNumber = files.size();
+            if (fileStatuses[i].isDir()) {
+                listSubFiles(dataset, srcFs, fileStatuses[i], files);
+            } else {
+                files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(), nextFileNumber,
+                        fileStatuses[i].getPath().toUri().getPath(), new Date(fileStatuses[i].getModificationTime()),
+                        fileStatuses[i].getLen(), ExternalFilePendingOp.PENDING_NO_OP));
+            }
+        }
+    }
+
+    public static FileSystem getFileSystemObject(Map<String, String> map) throws IOException {
+        Configuration conf = new Configuration();
+        conf.set("fs.default.name", ((String) map.get(HDFSAdapterFactory.KEY_HDFS_URL)).trim());
+        conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
+        return FileSystem.get(conf);
+    }
+
+    public static JobSpecification buildFilesIndexReplicationJobSpec(Dataset dataset,
+            ArrayList<ExternalFile> externalFilesSnapshot, AqlMetadataProvider metadataProvider, boolean createIndex)
+            throws MetadataException, AlgebricksException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(),
+                        dataset.getDatasetName(), getFilesIndexName(dataset.getDatasetName()), true);
+        IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
+        ILocalResourceMetadata localResourceMetadata = new ExternalBTreeLocalResourceMetadata(
+                FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
+                FilesIndexDescription.FILES_INDEX_COMP_FACTORIES, new int[] { 0 }, false, dataset.getDatasetId(),
+                mergePolicyFactory, mergePolicyFactoryProperties);
+        PersistentLocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
+                localResourceMetadata, LocalResource.ExternalBTreeResource);
+        ExternalBTreeDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeDataflowHelperFactory(
+                mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                        dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        ExternalFilesIndexOperatorDescriptor externalFilesOp = new ExternalFilesIndexOperatorDescriptor(spec,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                secondaryFileSplitProvider, indexDataflowHelperFactory, localResourceFactoryProvider,
+                externalFilesSnapshot, createIndex);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, externalFilesOp,
+                secondarySplitsAndConstraint.second);
+        spec.addRoot(externalFilesOp);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+
+    /**
+     * This method create an indexing operator that index records in HDFS
+     * 
+     * @param jobSpec
+     * @param itemType
+     * @param dataset
+     * @param format
+     * @param files
+     * @param indexerDesc
+     * @return
+     * @throws Exception
+     */
+    private static Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> getExternalDataIndexingOperator(
+            JobSpecification jobSpec, IAType itemType, Dataset dataset, List<ExternalFile> files,
+            RecordDescriptor indexerDesc, AqlMetadataProvider metadataProvider) throws Exception {
+        HDFSIndexingAdapterFactory adapterFactory = new HDFSIndexingAdapterFactory();
+        adapterFactory.setFiles(files);
+        adapterFactory.configure(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties(),
+                (ARecordType) itemType);
+        return new Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint>(
+                new ExternalDataScanOperatorDescriptor(jobSpec, indexerDesc, adapterFactory),
+                adapterFactory.getPartitionConstraint());
+    }
+
+    public static Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> createExternalIndexingOp(
+            JobSpecification spec, AqlMetadataProvider metadataProvider, Dataset dataset, ARecordType itemType,
+            RecordDescriptor indexerDesc, List<ExternalFile> files) throws Exception {
+        if (files == null) {
+            files = MetadataManager.INSTANCE.getDatasetExternalFiles(metadataProvider.getMetadataTxnContext(), dataset);
+        }
+        return getExternalDataIndexingOperator(spec, itemType, dataset, files, indexerDesc, metadataProvider);
+    }
+
+    /**
+     * At the end of this method, we expect to have 4 sets as follows:
+     * metadataFiles should contain only the files that are appended in their original state
+     * addedFiles should contain new files that has number assigned starting after the max original file number
+     * deleteedFiles should contain files that are no longer there in the file system
+     * appendedFiles should have the new file information of existing files
+     * The method should return false in case of zero delta
+     * 
+     * @param dataset
+     * @param metadataFiles
+     * @param addedFiles
+     * @param deletedFiles
+     * @param appendedFiles
+     * @return
+     * @throws MetadataException
+     * @throws AlgebricksException
+     */
+    public static boolean isDatasetUptodate(Dataset dataset, List<ExternalFile> metadataFiles,
+            List<ExternalFile> addedFiles, List<ExternalFile> deletedFiles, List<ExternalFile> appendedFiles)
+            throws MetadataException, AlgebricksException {
+        boolean uptodate = true;
+        int newFileNumber = metadataFiles.get(metadataFiles.size() - 1).getFileNumber() + 1;
+
+        ArrayList<ExternalFile> fileSystemFiles = getSnapshotFromExternalFileSystem(dataset);
+
+        // Loop over file system files < taking care of added files >
+        for (ExternalFile fileSystemFile : fileSystemFiles) {
+            boolean fileFound = false;
+            Iterator<ExternalFile> mdFilesIterator = metadataFiles.iterator();
+            while (mdFilesIterator.hasNext()) {
+                ExternalFile metadataFile = mdFilesIterator.next();
+                if (fileSystemFile.getFileName().equals(metadataFile.getFileName())) {
+                    // Same file name
+                    if (fileSystemFile.getLastModefiedTime().equals(metadataFile.getLastModefiedTime())) {
+                        // Same timestamp
+                        if (fileSystemFile.getSize() == metadataFile.getSize()) {
+                            // Same size -> no op
+                            mdFilesIterator.remove();
+                            fileFound = true;
+                        } else {
+                            // Different size -> append op
+                            metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_APPEND_OP);
+                            fileSystemFile.setPendingOp(ExternalFilePendingOp.PENDING_APPEND_OP);
+                            appendedFiles.add(fileSystemFile);
+                            fileFound = true;
+                            uptodate = false;
+                        }
+                    } else {
+                        // Same file name, Different file mod date -> delete and add
+                        metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_DROP_OP);
+                        deletedFiles.add(new ExternalFile(metadataFile.getDataverseName(), metadataFile
+                                .getDatasetName(), 0, metadataFile.getFileName(), metadataFile.getLastModefiedTime(),
+                                metadataFile.getSize(), ExternalFilePendingOp.PENDING_DROP_OP));
+                        fileSystemFile.setPendingOp(ExternalFilePendingOp.PENDING_ADD_OP);
+                        fileSystemFile.setFileNumber(newFileNumber);
+                        addedFiles.add(fileSystemFile);
+                        newFileNumber++;
+                        fileFound = true;
+                        uptodate = false;
+                    }
+                }
+                if (fileFound)
+                    break;
+            }
+            if (!fileFound) {
+                // File not stored previously in metadata -> pending add op
+                fileSystemFile.setPendingOp(ExternalFilePendingOp.PENDING_ADD_OP);
+                fileSystemFile.setFileNumber(newFileNumber);
+                addedFiles.add(fileSystemFile);
+                newFileNumber++;
+                uptodate = false;
+            }
+        }
+
+        // Done with files from external file system -> metadata files now contain both deleted files and appended ones
+        // first, correct number assignment to deleted and updated files
+        for (ExternalFile deletedFile : deletedFiles) {
+            deletedFile.setFileNumber(newFileNumber);
+            newFileNumber++;
+        }
+        for (ExternalFile appendedFile : appendedFiles) {
+            appendedFile.setFileNumber(newFileNumber);
+            newFileNumber++;
+        }
+
+        // include the remaining deleted files
+        Iterator<ExternalFile> mdFilesIterator = metadataFiles.iterator();
+        while (mdFilesIterator.hasNext()) {
+            ExternalFile metadataFile = mdFilesIterator.next();
+            if (metadataFile.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP) {
+                metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_DROP_OP);
+                deletedFiles.add(new ExternalFile(metadataFile.getDataverseName(), metadataFile.getDatasetName(),
+                        newFileNumber, metadataFile.getFileName(), metadataFile.getLastModefiedTime(), metadataFile
+                                .getSize(), metadataFile.getPendingOp()));
+                newFileNumber++;
+                uptodate = false;
+            }
+        }
+        return uptodate;
+    }
+
+    public static Dataset createTransactionDataset(Dataset dataset) {
+        ExternalDatasetDetails originalDsd = (ExternalDatasetDetails) dataset.getDatasetDetails();
+        ExternalDatasetDetails dsd = new ExternalDatasetDetails(originalDsd.getAdapter(), originalDsd.getProperties(),
+                originalDsd.getNodeGroupName(), originalDsd.getTimestamp(), ExternalDatasetTransactionState.BEGIN,
+                originalDsd.getCompactionPolicy(), originalDsd.getCompactionPolicyProperties());
+        Dataset transactionDatset = new Dataset(dataset.getDataverseName(), dataset.getDatasetName(),
+                dataset.getItemTypeName(), dsd, dataset.getHints(), DatasetType.EXTERNAL, dataset.getDatasetId(),
+                dataset.getPendingOp());
+        return transactionDatset;
+    }
+
+    public static boolean isFileIndex(Index index) {
+        return (index.getIndexName().equals(getFilesIndexName(index.getDatasetName())));
+    }
+
+    public static JobSpecification buildDropFilesIndexJobSpec(CompiledIndexDropStatement indexDropStmt,
+            AqlMetadataProvider metadataProvider, Dataset dataset) throws AlgebricksException, MetadataException {
+        String dataverseName = indexDropStmt.getDataverseName() == null ? metadataProvider.getDefaultDataverseName()
+                : indexDropStmt.getDataverseName();
+        String datasetName = indexDropStmt.getDatasetName();
+        String indexName = indexDropStmt.getIndexName();
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForFilesIndex(dataverseName, datasetName, indexName, true);
+        AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
+                metadataProvider.getMetadataTxnContext());
+        IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
+                        dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
+                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+                        storageProperties.getBloomFilterFalsePositiveRate(), false));
+        AlgebricksPartitionConstraintHelper
+                .setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
+        spec.addRoot(btreeDrop);
+
+        return spec;
+    }
+
+    public static JobSpecification buildFilesIndexUpdateOp(Dataset ds, List<ExternalFile> metadataFiles,
+            List<ExternalFile> deletedFiles, List<ExternalFile> addedFiles, List<ExternalFile> appendedFiles,
+            AqlMetadataProvider metadataProvider) throws MetadataException, AlgebricksException {
+        ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
+        for (ExternalFile file : metadataFiles) {
+            if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP)
+                files.add(file);
+            else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_APPEND_OP) {
+                for (ExternalFile appendedFile : appendedFiles) {
+                    if (appendedFile.getFileName().equals(file.getFileName())) {
+                        files.add(new ExternalFile(file.getDataverseName(), file.getDatasetName(),
+                                file.getFileNumber(), file.getFileName(), file.getLastModefiedTime(), appendedFile
+                                        .getSize(), ExternalFilePendingOp.PENDING_NO_OP));
+                    }
+                }
+            }
+        }
+        for (ExternalFile file : addedFiles) {
+            files.add(file);
+        }
+        Collections.sort(files);
+        return buildFilesIndexReplicationJobSpec(ds, files, metadataProvider, false);
+    }
+
+    public static JobSpecification buildIndexUpdateOp(Dataset ds, Index index, List<ExternalFile> metadataFiles,
+            List<ExternalFile> deletedFiles, List<ExternalFile> addedFiles, List<ExternalFile> appendedFiles,
+            AqlMetadataProvider metadataProvider) throws AsterixException, AlgebricksException {
+        // Create files list
+        ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
+
+        for (ExternalFile metadataFile : metadataFiles) {
+            if (metadataFile.getPendingOp() != ExternalFilePendingOp.PENDING_APPEND_OP) {
+                files.add(metadataFile);
+            } else {
+                metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                files.add(metadataFile);
+            }
+        }
+        // add new files
+        for (ExternalFile file : addedFiles) {
+            files.add(file);
+        }
+        // add appended files
+        for (ExternalFile file : appendedFiles) {
+            files.add(file);
+        }
+
+        CompiledCreateIndexStatement ccis = new CompiledCreateIndexStatement(index.getIndexName(),
+                index.getDataverseName(), index.getDatasetName(), index.getKeyFieldNames(), index.getGramLength(),
+                index.getIndexType());
+        return IndexOperations.buildSecondaryIndexLoadingJobSpec(ccis, metadataProvider, files);
+    }
+
+    public static JobSpecification buildCommitJob(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
+            throws AlgebricksException, AsterixException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                        getFilesIndexName(ds.getDatasetName()));
+        IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
+                mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
+        IndexInfoOperatorDescriptor filesIndexInfo = new IndexInfoOperatorDescriptor(filesIndexSplitProvider,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
+
+        ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory> btreeDataflowHelperFactories = new ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> btreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+        ArrayList<ExternalRTreeDataflowHelperFactory> rtreeDataflowHelperFactories = new ArrayList<ExternalRTreeDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> rtreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+
+        for (Index index : indexes) {
+            if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
+                Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
+                        .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                                index.getIndexName());
+                if (index.getIndexType() == IndexType.BTREE) {
+                    btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, spec));
+                    btreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                } else if (index.getIndexType() == IndexType.RTREE) {
+                    rtreeDataflowHelperFactories.add(getRTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, metadataProvider, spec));
+                    rtreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                }
+            }
+        }
+
+        ExternalDatasetIndexesCommitOperatorDescriptor op = new ExternalDatasetIndexesCommitOperatorDescriptor(spec,
+                filesIndexDataflowHelperFactory, filesIndexInfo, btreeDataflowHelperFactories, btreeInfos,
+                rtreeDataflowHelperFactories, rtreeInfos);
+
+        spec.addRoot(op);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op,
+                filesIndexSplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+
+    private static ExternalBTreeDataflowHelperFactory getFilesIndexDataflowHelperFactory(Dataset ds,
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
+            AsterixStorageProperties storageProperties, JobSpecification spec) {
+        return new ExternalBTreeDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
+                new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+                storageProperties.getBloomFilterFalsePositiveRate(),
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+    }
+
+    private static ExternalBTreeWithBuddyDataflowHelperFactory getBTreeDataflowHelperFactory(Dataset ds, Index index,
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
+            AsterixStorageProperties storageProperties, JobSpecification spec) {
+        return new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
+                new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+                storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static ExternalRTreeDataflowHelperFactory getRTreeDataflowHelperFactory(Dataset ds, Index index,
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
+            AsterixStorageProperties storageProperties, AqlMetadataProvider metadataProvider, JobSpecification spec)
+            throws AlgebricksException, AsterixException {
+        int numPrimaryKeys = getRIDSize(ds);
+        List<String> secondaryKeyFields = index.getKeyFieldNames();
+        secondaryKeyFields.size();
+        ARecordType itemType = (ARecordType) metadataProvider.findType(ds.getDataverseName(), ds.getItemTypeName());
+        Pair<IAType, Boolean> spatialTypePair = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0), itemType);
+        IAType spatialType = spatialTypePair.first;
+        if (spatialType == null) {
+            throw new AsterixException("Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
+        }
+        int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
+        int numNestedSecondaryKeyFields = numDimensions * 2;
+        IPrimitiveValueProviderFactory[] valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
+        IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
+
+        ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys
+                + numNestedSecondaryKeyFields];
+        ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numNestedSecondaryKeyFields + numPrimaryKeys];
+        IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(spatialType.getTypeTag());
+        ATypeTag keyType = nestedKeyType.getTypeTag();
+
+        keyType = nestedKeyType.getTypeTag();
+        for (int i = 0; i < numNestedSecondaryKeyFields; i++) {
+            ISerializerDeserializer keySerde = AqlSerializerDeserializerProvider.INSTANCE
+                    .getSerializerDeserializer(nestedKeyType);
+            secondaryRecFields[i] = keySerde;
+
+            secondaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(
+                    nestedKeyType, true);
+            secondaryTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
+            valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
+        }
+        // Add serializers and comparators for primary index fields.
+        for (int i = 0; i < numPrimaryKeys; i++) {
+            secondaryRecFields[numNestedSecondaryKeyFields + i] = IndexingConstants.getSerializerDeserializer(i);
+            secondaryTypeTraits[numNestedSecondaryKeyFields + i] = IndexingConstants.getTypeTraits(i);
+        }
+        int[] primaryKeyFields = new int[numPrimaryKeys];
+        for (int i = 0; i < primaryKeyFields.length; i++) {
+            primaryKeyFields[i] = i + numNestedSecondaryKeyFields;
+        }
+
+        return new ExternalRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                getBuddyBtreeComparatorFactories(), mergePolicyFactory, mergePolicyFactoryProperties,
+                new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
+                AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+                storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+    }
+
+    public static JobSpecification buildAbortOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
+            throws AlgebricksException, AsterixException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                        getFilesIndexName(ds.getDatasetName()));
+        IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
+                mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
+        IndexInfoOperatorDescriptor filesIndexInfo = new IndexInfoOperatorDescriptor(filesIndexSplitProvider,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
+
+        ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory> btreeDataflowHelperFactories = new ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> btreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+        ArrayList<ExternalRTreeDataflowHelperFactory> rtreeDataflowHelperFactories = new ArrayList<ExternalRTreeDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> rtreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+
+        for (Index index : indexes) {
+            if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
+                Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
+                        .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                                index.getIndexName());
+                if (index.getIndexType() == IndexType.BTREE) {
+                    btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, spec));
+                    btreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                } else if (index.getIndexType() == IndexType.RTREE) {
+                    rtreeDataflowHelperFactories.add(getRTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, metadataProvider, spec));
+                    rtreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                }
+            }
+        }
+
+        ExternalDatasetIndexesAbortOperatorDescriptor op = new ExternalDatasetIndexesAbortOperatorDescriptor(spec,
+                filesIndexDataflowHelperFactory, filesIndexInfo, btreeDataflowHelperFactories, btreeInfos,
+                rtreeDataflowHelperFactories, rtreeInfos);
+
+        spec.addRoot(op);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op,
+                filesIndexSplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+
+    }
+
+    public static JobSpecification buildRecoverOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
+            throws AlgebricksException, AsterixException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                        getFilesIndexName(ds.getDatasetName()));
+        IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
+                mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
+        IndexInfoOperatorDescriptor filesIndexInfo = new IndexInfoOperatorDescriptor(filesIndexSplitProvider,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
+
+        ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory> btreeDataflowHelperFactories = new ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> btreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+        ArrayList<ExternalRTreeDataflowHelperFactory> rtreeDataflowHelperFactories = new ArrayList<ExternalRTreeDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> rtreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+
+        for (Index index : indexes) {
+            if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
+                Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
+                        .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                                index.getIndexName());
+                if (index.getIndexType() == IndexType.BTREE) {
+                    btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, spec));
+                    btreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                } else if (index.getIndexType() == IndexType.RTREE) {
+                    rtreeDataflowHelperFactories.add(getRTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, metadataProvider, spec));
+                    rtreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                }
+            }
+        }
+
+        ExternalDatasetIndexesRecoverOperatorDescriptor op = new ExternalDatasetIndexesRecoverOperatorDescriptor(spec,
+                filesIndexDataflowHelperFactory, filesIndexInfo, btreeDataflowHelperFactories, btreeInfos,
+                rtreeDataflowHelperFactories, rtreeInfos);
+
+        spec.addRoot(op);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op,
+                filesIndexSplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+
+    public static JobSpecification compactFilesIndexJobSpec(Dataset dataset, AqlMetadataProvider metadataProvider)
+            throws MetadataException, AlgebricksException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(),
+                        dataset.getDatasetName(), getFilesIndexName(dataset.getDatasetName()), true);
+        IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeDataflowHelperFactory(
+                mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                        dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                secondaryFileSplitProvider, FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
+                FilesIndexDescription.FILES_INDEX_COMP_FACTORIES, new int[] { 0 }, indexDataflowHelperFactory,
+                NoOpOperationCallbackFactory.INSTANCE);
+        spec.addRoot(compactOp);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
+                secondarySplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index 5e3f7de..859e39c 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -14,6 +14,7 @@
  */
 package edu.uci.ics.asterix.file;
 
+import java.util.List;
 import java.util.Map;
 
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
@@ -24,6 +25,7 @@
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
@@ -66,6 +68,17 @@
                         physicalOptimizationConfig);
         return secondaryIndexHelper.buildLoadingJobSpec();
     }
+    
+    public static JobSpecification buildSecondaryIndexLoadingJobSpec(CompiledCreateIndexStatement createIndexStmt,
+            AqlMetadataProvider metadataProvider, List<ExternalFile> files) throws AsterixException, AlgebricksException {
+        SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+                .createIndexOperationsHelper(createIndexStmt.getIndexType(), createIndexStmt.getDataverseName(),
+                        createIndexStmt.getDatasetName(), createIndexStmt.getIndexName(),
+                        createIndexStmt.getKeyFields(), createIndexStmt.getGramLength(), metadataProvider,
+                        physicalOptimizationConfig);
+        secondaryIndexHelper.setExternalFiles(files);
+        return secondaryIndexHelper.buildLoadingJobSpec();
+    }
 
     public static JobSpecification buildDropSecondaryIndexJobSpec(CompiledIndexDropStatement indexDropStmt,
             AqlMetadataProvider metadataProvider, Dataset dataset) throws AlgebricksException, MetadataException {
@@ -76,7 +89,7 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, indexName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
                 metadataProvider.getMetadataTxnContext());
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
index 47b2567..597e835 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
@@ -16,12 +16,17 @@
 
 import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
 import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
 import edu.uci.ics.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
+import edu.uci.ics.asterix.transaction.management.resource.ExternalBTreeWithBuddyLocalResourceMetadata;
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
 import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
@@ -35,9 +40,12 @@
 import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
@@ -55,21 +63,39 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
-        ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
-                secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields, true,
-                dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties);
-        ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
-                localResourceMetadata, LocalResource.LSMBTreeResource);
-
+        ILocalResourceFactoryProvider localResourceFactoryProvider;
+        IIndexDataflowHelperFactory indexDataflowHelperFactory;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
+            ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    true, dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.LSMBTreeResource);
+            indexDataflowHelperFactory = new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
+                    dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
+                    new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+                    storageProperties.getBloomFilterFalsePositiveRate(), false);
+        } else {
+            // External dataset local resource and dataflow helper
+            int[] buddyBreeFields = new int[] { numSecondaryKeys };
+            ILocalResourceMetadata localResourceMetadata = new ExternalBTreeWithBuddyLocalResourceMetadata(
+                    dataset.getDatasetId(), secondaryComparatorFactories, secondaryRecDesc.getTypeTraits(),
+                    mergePolicyFactory, mergePolicyFactoryProperties, buddyBreeFields);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.ExternalBTreeWithBuddyResource);
+            indexDataflowHelperFactory = new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory,
+                    mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+                    storageProperties.getBloomFilterFalsePositiveRate(), buddyBreeFields,
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        }
         TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
-                secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
-                        dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), false), localResourceFactoryProvider,
+                secondaryBloomFilterKeyFields, indexDataflowHelperFactory, localResourceFactoryProvider,
                 NoOpOperationCallbackFactory.INSTANCE);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
                 secondaryPartitionConstraint);
@@ -81,49 +107,100 @@
     @Override
     public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+            /*
+             * In case of external data, this method is used to build loading jobs for both initial load on index creation
+             * and transaction load on dataset referesh
+             */
+            // Create external indexing scan operator
+            ExternalDataScanOperatorDescriptor primaryScanOp = createExternalIndexingOp(spec);
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec, numSecondaryKeys);
 
-        // Create dummy key provider for feeding the primary index scan. 
-        AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+            }
 
-        // Create primary index scan op.
-        BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
 
-        // Assign op.
-        AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp, numSecondaryKeys);
-
-        // If any of the secondary fields are nullable, then add a select op that filters nulls.
-        AlgebricksMetaOperatorDescriptor selectOp = null;
-        if (anySecondaryKeyIsNullable) {
-            selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
-        }
-
-        // Sort by secondary keys.
-        ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
-
-        AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        // Create secondary BTree bulk load op.
-        TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
-                spec,
-                numSecondaryKeys,
-                new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
-                        mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
-                                dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                        LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
-                                .getBloomFilterFalsePositiveRate(), false), BTree.DEFAULT_FILL_FACTOR);
-
-        // Connect the operators.
-        spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
-        spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
-        if (anySecondaryKeyIsNullable) {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
-            spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+            // Create secondary BTree bulk load op.
+            AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
+            ExternalBTreeWithBuddyDataflowHelperFactory dataflowHelperFactory = new ExternalBTreeWithBuddyDataflowHelperFactory(
+                    mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                            dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+                    storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+            if (externalFiles != null) {
+                // Transaction load
+                secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, numSecondaryKeys, dataflowHelperFactory,
+                        BTree.DEFAULT_FILL_FACTOR);
+            } else {
+                // Initial load
+                secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numSecondaryKeys, dataflowHelperFactory,
+                        BTree.DEFAULT_FILL_FACTOR);
+            }
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+            return spec;
         } else {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            // Create dummy key provider for feeding the primary index scan. 
+            AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
+
+            // Create primary index scan op.
+            BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
+
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp, numSecondaryKeys);
+
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+            }
+
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+            // Create secondary BTree bulk load op.
+            TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
+                    spec,
+                    numSecondaryKeys,
+                    new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
+                                    .getBloomFilterFalsePositiveRate(), false), BTree.DEFAULT_FILL_FACTOR);
+
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+            return spec;
         }
-        spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
-        spec.addRoot(secondaryBulkLoadOp);
-        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
-        return spec;
     }
 
     @Override
@@ -131,15 +208,32 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
-                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
-                secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
-                        dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), false),
-                NoOpOperationCallbackFactory.INSTANCE);
+        LSMTreeIndexCompactOperatorDescriptor compactOp;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
+                                    .getBloomFilterFalsePositiveRate(), false), NoOpOperationCallbackFactory.INSTANCE);
+        } else {
+            // External dataset
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, storageProperties
+                                    .getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
+                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+                    NoOpOperationCallbackFactory.INSTANCE);
+        }
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
                 secondaryPartitionConstraint);
         spec.addRoot(compactOp);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
index ca049a6..851de4a 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
@@ -22,6 +22,7 @@
 
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
 import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -31,6 +32,7 @@
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import edu.uci.ics.asterix.common.transactions.IRecoveryManager.ResourceType;
 import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalIndexBulkModifyOperatorDescriptor;
 import edu.uci.ics.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
 import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
@@ -38,7 +40,10 @@
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
 import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.external.IndexingConstants;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -123,6 +128,8 @@
     protected ILSMMergePolicyFactory mergePolicyFactory;
     protected Map<String, String> mergePolicyFactoryProperties;
 
+    protected List<ExternalFile> externalFiles;
+
     // Prevent public construction. Should be created via createIndexCreator().
     protected SecondaryIndexOperationsHelper(PhysicalOptimizationConfig physOptConf,
             IAsterixPropertiesProvider propertiesProvider) {
@@ -178,24 +185,25 @@
         if (dataset == null) {
             throw new AsterixException("Unknown dataset " + datasetName);
         }
-        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
-            throw new AsterixException("Cannot index an external dataset (" + datasetName + ").");
-        }
+
         itemType = (ARecordType) metadataProvider.findType(dataset.getDataverseName(), dataset.getItemTypeName());
         payloadSerde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
-        numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
         numSecondaryKeys = secondaryKeyFields.size();
-        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
-        primaryFileSplitProvider = primarySplitsAndConstraint.first;
-        primaryPartitionConstraint = primarySplitsAndConstraint.second;
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName,
-                        secondaryIndexName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName);
         secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
         secondaryPartitionConstraint = secondarySplitsAndConstraint.second;
-        // Must be called in this order.
-        setPrimaryRecDescAndComparators();
+
+        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+            numPrimaryKeys = ExternalIndexingOperations.getRIDSize(dataset);
+        } else {
+            numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
+            Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
+                    .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+            primaryFileSplitProvider = primarySplitsAndConstraint.first;
+            primaryPartitionConstraint = primarySplitsAndConstraint.second;
+            setPrimaryRecDescAndComparators();
+        }
         setSecondaryRecDescAndComparators(indexType, secondaryKeyFields, gramLength, metadataProvider);
         numElementsHint = metadataProvider.getCardinalityPerPartitionHint(dataset);
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
@@ -245,9 +253,11 @@
         ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
         IBinaryComparatorFactoryProvider comparatorFactoryProvider = metadataProvider.getFormat()
                 .getBinaryComparatorFactoryProvider();
+        // Record column is 0 for external datasets, numPrimaryKeys for internal ones
+        int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
         for (int i = 0; i < numSecondaryKeys; i++) {
             secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(
-                    itemType, secondaryKeyFields.get(i), numPrimaryKeys);
+                    itemType, secondaryKeyFields.get(i), recordColumn);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(i), itemType);
             IAType keyType = keyTypePair.first;
             anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
@@ -257,12 +267,23 @@
             secondaryTypeTraits[i] = typeTraitProvider.getTypeTrait(keyType);
             secondaryBloomFilterKeyFields[i] = i;
         }
-        // Add serializers and comparators for primary index fields.
-        for (int i = 0; i < numPrimaryKeys; i++) {
-            secondaryRecFields[numSecondaryKeys + i] = primaryRecDesc.getFields()[i];
-            secondaryTypeTraits[numSecondaryKeys + i] = primaryRecDesc.getTypeTraits()[i];
-            if (indexType != IndexType.RTREE) {
-                secondaryComparatorFactories[numSecondaryKeys + i] = primaryComparatorFactories[i];
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            // Add serializers and comparators for primary index fields.
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numSecondaryKeys + i] = primaryRecDesc.getFields()[i];
+                secondaryTypeTraits[numSecondaryKeys + i] = primaryRecDesc.getTypeTraits()[i];
+                if (indexType != IndexType.RTREE) {
+                    secondaryComparatorFactories[numSecondaryKeys + i] = primaryComparatorFactories[i];
+                }
+            }
+        } else {
+            // Add serializers and comparators for RID fields.
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numSecondaryKeys + i] = IndexingConstants.getSerializerDeserializer(i);
+                secondaryTypeTraits[numSecondaryKeys + i] = IndexingConstants.getTypeTraits(i);
+                if (indexType != IndexType.RTREE) {
+                    secondaryComparatorFactories[numSecondaryKeys + i] = IndexingConstants.getComparatorFactory(i);
+                }
             }
         }
         secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
@@ -314,8 +335,9 @@
                 new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
                         mergePolicyFactory, mergePolicyFactoryProperties, new PrimaryIndexOperationTrackerProvider(
                                 dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                        LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), true), false, false, null, searchCallbackFactory);
+                        LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
+                                .getBloomFilterFalsePositiveRate(), true), false, false, null,
+                searchCallbackFactory);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
                 primaryPartitionConstraint);
         return primarySearchOp;
@@ -409,4 +431,100 @@
                 primaryPartitionConstraint);
         return asterixSelectOp;
     }
+
+    // This method creates a source indexing operator for external data
+    protected ExternalDataScanOperatorDescriptor createExternalIndexingOp(JobSpecification spec)
+            throws AlgebricksException, AsterixException {
+        // A record + primary keys
+        ISerializerDeserializer[] serdes = new ISerializerDeserializer[1 + numPrimaryKeys];
+        ITypeTraits[] typeTraits = new ITypeTraits[1 + numPrimaryKeys];
+        // payload serde and type traits for the record slot
+        serdes[0] = payloadSerde;
+        typeTraits[0] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(itemType);
+        //  serdes and type traits for rid fields
+        for (int i = 1; i < serdes.length; i++) {
+            serdes[i] = IndexingConstants.getSerializerDeserializer(i - 1);
+            typeTraits[i] = IndexingConstants.getTypeTraits(i - 1);
+        }
+        // output record desc
+        RecordDescriptor indexerDesc = new RecordDescriptor(serdes, typeTraits);
+
+        // Create the operator and its partition constraits
+        Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> indexingOpAndConstraints;
+        try {
+            indexingOpAndConstraints = ExternalIndexingOperations.createExternalIndexingOp(spec, metadataProvider,
+                    dataset, itemType, indexerDesc, externalFiles);
+        } catch (Exception e) {
+            throw new AlgebricksException(e);
+        }
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexingOpAndConstraints.first,
+                indexingOpAndConstraints.second);
+
+        // Set the primary partition constraints to this partition constraints
+        primaryPartitionConstraint = indexingOpAndConstraints.second;
+        return indexingOpAndConstraints.first;
+    }
+
+    protected AlgebricksMetaOperatorDescriptor createExternalAssignOp(JobSpecification spec, int numSecondaryKeys) throws AlgebricksException {
+        int[] outColumns = new int[numSecondaryKeys];
+        int[] projectionList = new int[numSecondaryKeys + numPrimaryKeys];
+        for (int i = 0; i < numSecondaryKeys; i++) {
+            outColumns[i] = i + numPrimaryKeys + 1;
+            projectionList[i] = i + numPrimaryKeys + 1;
+        }
+
+        IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[secondaryFieldAccessEvalFactories.length];
+        for (int i = 0; i < secondaryFieldAccessEvalFactories.length; ++i) {
+            sefs[i] = new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.ScalarEvaluatorFactoryAdapter(
+                    secondaryFieldAccessEvalFactories[i]);
+        }
+        //add External RIDs to the projection list
+        for (int i = 0; i < numPrimaryKeys; i++) {
+            projectionList[numSecondaryKeys + i] = i + 1;
+        }
+
+        AssignRuntimeFactory assign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
+        AlgebricksMetaOperatorDescriptor asterixAssignOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 1,
+                new IPushRuntimeFactory[] { assign }, new RecordDescriptor[] { secondaryRecDesc });
+        return asterixAssignOp;
+    }
+
+    protected ExternalIndexBulkModifyOperatorDescriptor createExternalIndexBulkModifyOp(JobSpecification spec,
+            int numSecondaryKeyFields, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
+            throws MetadataException, AlgebricksException {
+        int[] fieldPermutation = new int[numSecondaryKeyFields + numPrimaryKeys];
+        for (int i = 0; i < numSecondaryKeyFields + numPrimaryKeys; i++) {
+            fieldPermutation[i] = i;
+        }
+        // create a list of file ids
+        int numOfDeletedFiles = 0;
+        for (ExternalFile file : externalFiles) {
+            if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP)
+                numOfDeletedFiles++;
+        }
+        int[] deletedFiles = new int[numOfDeletedFiles];
+        int i = 0;
+        for (ExternalFile file : externalFiles) {
+            if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP) {
+                deletedFiles[i] = file.getFileNumber();
+            }
+        }
+        ExternalIndexBulkModifyOperatorDescriptor treeIndexBulkLoadOp = new ExternalIndexBulkModifyOperatorDescriptor(
+                spec, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                dataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE, deletedFiles, fieldPermutation,
+                fillFactor, numElementsHint);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
+                secondaryPartitionConstraint);
+        return treeIndexBulkLoadOp;
+    }
+
+    public List<ExternalFile> getExternalFiles() {
+        return externalFiles;
+    }
+
+    public void setExternalFiles(List<ExternalFile> externalFiles) {
+        this.externalFiles = externalFiles;
+    }
 }
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
index 225ee53..37aa427 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -18,6 +18,7 @@
 
 import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -29,11 +30,15 @@
 import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.external.IndexingConstants;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
 import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
 import edu.uci.ics.asterix.transaction.management.resource.LSMRTreeLocalResourceMetadata;
+import edu.uci.ics.asterix.transaction.management.resource.ExternalRTreeLocalResourceMetadata;
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
 import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
@@ -53,10 +58,13 @@
 import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
 import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
 import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
@@ -80,26 +88,50 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
-        ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
-                secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
-                valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
-                        secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
-                mergePolicyFactoryProperties, primaryKeyFields);
-        ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
-                localResourceMetadata, LocalResource.LSMRTreeResource);
+        IIndexDataflowHelperFactory indexDataflowHelperFactory;
+        ILocalResourceFactoryProvider localResourceFactoryProvider;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
+            ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
+                    valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
+                    mergePolicyFactoryProperties, primaryKeyFields);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.LSMRTreeResource);
+            indexDataflowHelperFactory = new LSMRTreeDataflowHelperFactory(valueProviderFactories,
+                    RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
+                            dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
+                    new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
+                    AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+                    storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields);
+        } else {
+            // External dataset
+            // Prepare a LocalResourceMetadata which will be stored in NC's local resource repository
+            ILocalResourceMetadata localResourceMetadata = new ExternalRTreeLocalResourceMetadata(
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
+                    ExternalIndexingOperations.getBuddyBtreeComparatorFactories(), valueProviderFactories,
+                    RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
+                    mergePolicyFactoryProperties, primaryKeyFields);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.ExternalRTreeResource);
+
+            indexDataflowHelperFactory = new ExternalRTreeDataflowHelperFactory(valueProviderFactories,
+                    RTreePolicyType.RTREE, ExternalIndexingOperations.getBuddyBtreeComparatorFactories(),
+                    mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                            dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
+                    new int[] { numNestedSecondaryKeyFields },
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        }
 
         TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, null,
-                new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
-                        primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
-                        mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
-                                dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                        LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
-                                secondaryComparatorFactories.length), storageProperties
-                                .getBloomFilterFalsePositiveRate(), primaryKeyFields),
-                localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
+                indexDataflowHelperFactory, localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
                 secondaryPartitionConstraint);
         spec.addRoot(secondaryIndexCreateOp);
@@ -125,8 +157,9 @@
         }
         int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
         numNestedSecondaryKeyFields = numDimensions * 2;
+        int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
         secondaryFieldAccessEvalFactories = metadata.getFormat().createMBRFactory(itemType, secondaryKeyFields.get(0),
-                numPrimaryKeys, numDimensions);
+                recordColumn, numDimensions);
         secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
         valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
         ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys
@@ -144,9 +177,16 @@
             valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
         }
         // Add serializers and comparators for primary index fields.
-        for (int i = 0; i < numPrimaryKeys; i++) {
-            secondaryRecFields[numNestedSecondaryKeyFields + i] = primaryRecDesc.getFields()[i];
-            secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i];
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numNestedSecondaryKeyFields + i] = primaryRecDesc.getFields()[i];
+                secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i];
+            }
+        } else {
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numNestedSecondaryKeyFields + i] = IndexingConstants.getSerializerDeserializer(i);
+                secondaryTypeTraits[numNestedSecondaryKeyFields + i] = IndexingConstants.getTypeTraits(i);
+            }
         }
         secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
         primaryKeyFields = new int[numPrimaryKeys];
@@ -158,51 +198,107 @@
     @Override
     public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            // Create dummy key provider for feeding the primary index scan. 
+            AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
 
-        // Create dummy key provider for feeding the primary index scan. 
-        AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
+            // Create primary index scan op.
+            BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
 
-        // Create primary index scan op.
-        BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp,
+                    numNestedSecondaryKeyFields);
 
-        // Assign op.
-        AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp,
-                numNestedSecondaryKeyFields);
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
+            }
 
-        // If any of the secondary fields are nullable, then add a select op that filters nulls.
-        AlgebricksMetaOperatorDescriptor selectOp = null;
-        if (anySecondaryKeyIsNullable) {
-            selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
-        }
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
+                    new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length) }, secondaryRecDesc);
 
-        // Sort by secondary keys.
-        ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
-                new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
-                        secondaryComparatorFactories.length) }, secondaryRecDesc);
-
-        AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        // Create secondary RTree bulk load op.
-        TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec,
-                numNestedSecondaryKeyFields, new LSMRTreeDataflowHelperFactory(valueProviderFactories,
-                        RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
-                                dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
-                        AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
-                        storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields),
-                BTree.DEFAULT_FILL_FACTOR);
-        // Connect the operators.
-        spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
-        spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
-        if (anySecondaryKeyIsNullable) {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
-            spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+            // Create secondary RTree bulk load op.
+            TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
+                    spec,
+                    numNestedSecondaryKeyFields,
+                    new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                            primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                                    secondaryComparatorFactories.length), storageProperties
+                                    .getBloomFilterFalsePositiveRate(), primaryKeyFields), BTree.DEFAULT_FILL_FACTOR);
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         } else {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            // External dataset
+            /*
+             * In case of external data, this method is used to build loading jobs for both initial load on index creation
+             * and transaction load on dataset referesh
+             */
+            // Create external indexing scan operator
+            ExternalDataScanOperatorDescriptor primaryScanOp = createExternalIndexingOp(spec);
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec, numNestedSecondaryKeyFields);
+
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+            }
+
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
+                    new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length) }, secondaryRecDesc);
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+
+            // Create the dataflow helper factory
+            ExternalRTreeDataflowHelperFactory dataflowHelperFactory = new ExternalRTreeDataflowHelperFactory(
+                    valueProviderFactories, RTreePolicyType.RTREE, primaryComparatorFactories, mergePolicyFactory,
+                    mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
+                    AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+                    storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+            // Create secondary RTree bulk load op.
+            AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
+            if (externalFiles != null) {
+                // Transaction load
+                secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, numNestedSecondaryKeyFields,
+                        dataflowHelperFactory, BTree.DEFAULT_FILL_FACTOR);
+            } else {
+                // Initial load
+                secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numNestedSecondaryKeyFields,
+                        dataflowHelperFactory, BTree.DEFAULT_FILL_FACTOR);
+            }
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         }
-        spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
-        spec.addRoot(secondaryBulkLoadOp);
-        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         return spec;
     }
 
@@ -211,17 +307,37 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
-                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
-                secondaryBloomFilterKeyFields, new LSMRTreeDataflowHelperFactory(valueProviderFactories,
-                        RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
-                                dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
-                        AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
-                        storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields),
-                NoOpOperationCallbackFactory.INSTANCE);
+        LSMTreeIndexCompactOperatorDescriptor compactOp;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                            primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                                    secondaryComparatorFactories.length), storageProperties
+                                    .getBloomFilterFalsePositiveRate(), primaryKeyFields),
+                    NoOpOperationCallbackFactory.INSTANCE);
+        } else {
+            // External dataset
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new ExternalRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                            primaryComparatorFactories, mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                                    secondaryComparatorFactories.length), storageProperties
+                                    .getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
+                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+                    NoOpOperationCallbackFactory.INSTANCE);
+        }
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
                 secondaryPartitionConstraint);
         spec.addRoot(compactOp);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
new file mode 100644
index 0000000..0af6a7e
--- /dev/null
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.hyracks.bootstrap;
+
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import edu.uci.ics.asterix.file.ExternalIndexingOperations;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.api.IClusterEventsSubscriber;
+import edu.uci.ics.asterix.metadata.api.IClusterManagementWork;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
+import edu.uci.ics.asterix.metadata.cluster.IClusterManagementWorkResponse;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
+import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties.State;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class AsterixGlobalRecoveryManager implements IClusterEventsSubscriber {
+
+    private static State state;
+    private static final Logger LOGGER = Logger.getLogger(AsterixGlobalRecoveryManager.class.getName());
+    private HyracksConnection hcc;
+    public static AsterixGlobalRecoveryManager INSTANCE;
+
+    public AsterixGlobalRecoveryManager(HyracksConnection hcc) throws Exception {
+        state = AsterixClusterProperties.INSTANCE.getState();
+        this.hcc = hcc;
+    }
+
+    @Override
+    public Set<IClusterManagementWork> notifyNodeFailure(Set<String> deadNodeIds) {
+        state = AsterixClusterProperties.INSTANCE.getState();
+        AsterixClusterProperties.INSTANCE.setGlobalRecoveryCompleted(false);
+        return null;
+    }
+
+    @Override
+    public Set<IClusterManagementWork> notifyNodeJoin(String joinedNodeId) {
+        // perform global recovery if state changed to active
+        final State newState = AsterixClusterProperties.INSTANCE.getState();
+        boolean needToRecover = !newState.equals(state) && (newState == State.ACTIVE);
+        if (needToRecover) {
+            Thread recoveryThread = new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    LOGGER.info("Starting AsterixDB's Global Recovery");
+                    MetadataTransactionContext mdTxnCtx = null;
+                    try {
+                        Thread.sleep(4000);
+                        MetadataManager.INSTANCE.init();
+                        // Loop over datasets
+                        mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                        List<Dataverse> dataverses = MetadataManager.INSTANCE.getDataverses(mdTxnCtx);
+                        for (Dataverse dataverse : dataverses) {
+                            if (!dataverse.getDataverseName().equals(MetadataConstants.METADATA_DATAVERSE_NAME)) {
+                                AqlMetadataProvider metadataProvider = new AqlMetadataProvider(dataverse);
+                                List<Dataset> datasets = MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx,
+                                        dataverse.getDataverseName());
+                                for (Dataset dataset : datasets) {
+                                    if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+                                        // External dataset
+                                        // Get indexes
+                                        List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx,
+                                                dataset.getDataverseName(), dataset.getDatasetName());
+                                        if (indexes.size() > 0) {
+                                            // Get the state of the dataset
+                                            ExternalDatasetDetails dsd = (ExternalDatasetDetails) dataset
+                                                    .getDatasetDetails();
+                                            ExternalDatasetTransactionState datasetState = dsd.getState();
+                                            if (datasetState == ExternalDatasetTransactionState.BEGIN) {
+                                                List<ExternalFile> files = MetadataManager.INSTANCE
+                                                        .getDatasetExternalFiles(mdTxnCtx, dataset);
+                                                // if persumed abort, roll backward
+                                                // 1. delete all pending files
+                                                for (ExternalFile file : files) {
+                                                    if (file.getPendingOp() != ExternalFilePendingOp.PENDING_NO_OP) {
+                                                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                                                    }
+                                                }
+                                                // 2. clean artifacts in NCs
+                                                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                                                JobSpecification jobSpec = ExternalIndexingOperations.buildAbortOp(
+                                                        dataset, indexes, metadataProvider);
+                                                executeHyracksJob(jobSpec);
+                                                // 3. correct the dataset state
+                                                ((ExternalDatasetDetails) dataset.getDatasetDetails())
+                                                        .setState(ExternalDatasetTransactionState.COMMIT);
+                                                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, dataset);
+                                                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                                                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                                            } else if (datasetState == ExternalDatasetTransactionState.READY_TO_COMMIT) {
+                                                List<ExternalFile> files = MetadataManager.INSTANCE
+                                                        .getDatasetExternalFiles(mdTxnCtx, dataset);
+                                                // if ready to commit, roll forward
+                                                // 1. commit indexes in NCs
+                                                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                                                JobSpecification jobSpec = ExternalIndexingOperations.buildRecoverOp(
+                                                        dataset, indexes, metadataProvider);
+                                                executeHyracksJob(jobSpec);
+                                                // 2. add pending files in metadata
+                                                for (ExternalFile file : files) {
+                                                    if (file.getPendingOp() == ExternalFilePendingOp.PENDING_ADD_OP) {
+                                                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                                                        file.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                                                        MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+                                                    } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP) {
+                                                        // find original file
+                                                        for (ExternalFile originalFile : files) {
+                                                            if (originalFile.getFileName().equals(file.getFileName())) {
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        file);
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        originalFile);
+                                                                break;
+                                                            }
+                                                        }
+                                                    } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_APPEND_OP) {
+                                                        // find original file
+                                                        for (ExternalFile originalFile : files) {
+                                                            if (originalFile.getFileName().equals(file.getFileName())) {
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        file);
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        originalFile);
+                                                                originalFile.setSize(file.getSize());
+                                                                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx,
+                                                                        originalFile);
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                                // 3. correct the dataset state
+                                                ((ExternalDatasetDetails) dataset.getDatasetDetails())
+                                                        .setState(ExternalDatasetTransactionState.COMMIT);
+                                                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, dataset);
+                                                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                                                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    } catch (Exception e) {
+                        // This needs to be fixed <-- Needs to shutdown the system -->
+                        /*
+                         * Note: Throwing this illegal state exception will terminate this thread
+                         * and feeds listeners will not be notified.
+                         */
+                        LOGGER.severe("Global recovery was not completed successfully" + e);
+                        try {
+                            MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+                        } catch (Exception e1) {
+                            if (LOGGER.isLoggable(Level.SEVERE)) {
+                                LOGGER.severe("Exception in aborting" + e.getMessage());
+                            }
+                            throw new IllegalStateException(e1);
+                        }
+                    }
+                    AsterixClusterProperties.INSTANCE.setGlobalRecoveryCompleted(true);
+                    LOGGER.info("Global Recovery Completed");
+                }
+            });
+            state = newState;
+            recoveryThread.start();
+        }
+        return null;
+    }
+
+    private void executeHyracksJob(JobSpecification spec) throws Exception {
+        spec.setMaxReattempts(0);
+        JobId jobId = hcc.startJob(spec);
+        hcc.waitForCompletion(jobId);
+    }
+
+    @Override
+    public void notifyRequestCompletion(IClusterManagementWorkResponse response) {
+        // Do nothing
+    }
+
+    @Override
+    public void notifyStateChange(State previousState, State newState) {
+        // Do nothing?
+    }
+}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 6314d54..ce528bf 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -37,6 +37,7 @@
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
 import edu.uci.ics.asterix.metadata.bootstrap.AsterixStateProxy;
+import edu.uci.ics.asterix.metadata.cluster.ClusterManager;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
 import edu.uci.ics.hyracks.api.application.ICCApplicationEntryPoint;
@@ -94,6 +95,9 @@
 		setupFeedServer(externalProperties);
 		feedServer.start();
 
+		AsterixGlobalRecoveryManager.INSTANCE = new AsterixGlobalRecoveryManager((HyracksConnection) getNewHyracksClientConnection());
+		ClusterManager.INSTANCE.registerSubscriber(AsterixGlobalRecoveryManager.INSTANCE);
+		
 		ccAppCtx.addClusterLifecycleListener(ClusterLifecycleListener.INSTANCE);
 	}
 
diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
index b9d7491..0750a58 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
@@ -1,13 +1,14 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 13, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 2, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 8, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 3, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 1, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 10, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 11, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 12, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 7, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 4, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 9, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 5, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 13, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 2, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 8, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 3, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 1, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DataTypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "FileNumber" ], "PrimaryKey": [ "DataverseName", "DatasetName", "FileNumber" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 14, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 10, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 11, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 12, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 7, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 4, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 9, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 5, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm
index 66a5a68..4580e90 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm
@@ -1,70 +1,73 @@
-{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Mon Sep 23 00:04:06 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Thu Oct 24 01:40:50 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "FileNumber", "FieldType": "int32" }, { "FieldName": "FileName", "FieldType": "string" }, { "FieldName": "FileSize", "FieldType": "int64" }, { "FieldName": "FileModTime", "FieldType": "datetime" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "LastRefreshTime", "FieldType": "datetime" }, { "FieldName": "TransactionState", "FieldType": "int32" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm
index a8dfc95..c14e3f2 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm
@@ -1,16 +1,17 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "IndexName": "ExternalFile", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "FileNumber" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm b/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
index b9d7491..0750a58 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
@@ -1,13 +1,14 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 13, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 2, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 8, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 3, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 1, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 10, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 11, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 12, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 7, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 4, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 9, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 5, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 13, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 2, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 8, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 3, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 1, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DataTypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "FileNumber" ], "PrimaryKey": [ "DataverseName", "DatasetName", "FileNumber" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 14, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 10, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 11, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 12, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 7, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 4, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 9, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 5, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm b/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
index 4081076..4580e90 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
@@ -1,70 +1,73 @@
-{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Mon Sep 23 00:25:26 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Thu Oct 24 01:40:50 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "FileNumber", "FieldType": "int32" }, { "FieldName": "FileName", "FieldType": "string" }, { "FieldName": "FileSize", "FieldType": "int64" }, { "FieldName": "FileModTime", "FieldType": "datetime" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "LastRefreshTime", "FieldType": "datetime" }, { "FieldName": "TransactionState", "FieldType": "int32" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm b/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm
index a8dfc95..c14e3f2 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm
@@ -1,16 +1,17 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "IndexName": "ExternalFile", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "FileNumber" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.ddl.aql
new file mode 100644
index 0000000..55bda77f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.ddl.aql
@@ -0,0 +1,38 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+	screen-name: string,
+	lang: string,
+	friends-count: int32,
+	statuses-count: int32,
+	name: string,
+	followers-count: int32
+} 
+
+create type TweetMessageType as closed {
+	tweetid: int64,
+        user: TwitterUserType,
+        sender-location: point,
+	send-time: datetime,
+        referred-topics: {{ string }},
+	message-text: string,
+	countA: int32,
+	countB: int32
+}
+
+create external dataset TweetMessages(TweetMessageType) using hdfs(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/tw_for_indexleftouterjoin.adm"),("input-format"="text-input-format"),("format"="adm"));
+
+create index IdIx on TweetMessages(tweetid) type btree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.3.query.aql
new file mode 100644
index 0000000..50e72e3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.3.query.aql
@@ -0,0 +1,21 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+let $n :=  create-circle($t1.sender-location, 0.5)
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"loc1":$t1.sender-location,
+"nearby-message": for $t2 in dataset('TweetMessages')
+                             where spatial-intersect($t2.sender-location, $n) 
+                             order by $t2.tweetid 
+                             return {"tweetid2":$t2.tweetid, "loc2":$t2.sender-location}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.1.ddl.aql
new file mode 100644
index 0000000..ed033eb
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.1.ddl.aql
@@ -0,0 +1,36 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+	screen-name: string,
+	lang: string,
+	friends-count: int32,
+	statuses-count: int32,
+	name: string,
+	followers-count: int32
+} 
+
+create type TweetMessageType as closed {
+	tweetid: int64,
+        user: TwitterUserType,
+        sender-location: point,
+	send-time: datetime,
+        referred-topics: {{ string }},
+	message-text: string,
+	countA: int32,
+	countB: int32
+}
+
+create external dataset TweetMessages(TweetMessageType) using hdfs(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/tw_for_indexleftouterjoin.adm"),("input-format"="text-input-format"),("format"="adm"));
+
+create index IdIx on TweetMessages(tweetid) type btree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.2.update.aql
new file mode 100644
index 0000000..16cbac3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.2.update.aql
@@ -0,0 +1,6 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.3.query.aql
new file mode 100644
index 0000000..408a2e1
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.3.query.aql
@@ -0,0 +1,14 @@
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"count1":$t1.countA,
+"t2info": for $t2 in dataset('TweetMessages') 
+          where $t1.countA /* +indexnl */= $t2.countB
+          order by $t2.tweetid 
+          return {"tweetid2": $t2.tweetid,
+                  "count2":$t2.countB}
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.1.ddl.aql
new file mode 100644
index 0000000..4f95913
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.1.ddl.aql
@@ -0,0 +1,23 @@
+/*
+* Description  : Create an external dataset that contains records stored with rc hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type EmployeeType as closed {
+ id: int32,
+ name: string,
+ age: int32
+};
+
+create external dataset EmployeeDataset(EmployeeType)
+using hdfs
+(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.rc"),("input-format"="rc-input-format"),("format"="binary"),("parser"="hive-parser"),("hive-serde"="org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"));
+
+create index EmployeeAgeIdx on EmployeeDataset(age);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.2.update.aql
new file mode 100644
index 0000000..569c3e6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.2.update.aql
@@ -0,0 +1,7 @@
+/*
+* Description  : Create an external dataset that contains records stored with rc hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.3.query.aql
new file mode 100644
index 0000000..d46031c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.3.query.aql
@@ -0,0 +1,12 @@
+/*
+* Description  : Create an external dataset that contains records stored with rc hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+use dataverse test;
+
+for $emp in dataset EmployeeDataset
+where $emp.age = 22
+return $emp;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.1.ddl.aql
new file mode 100644
index 0000000..8bd00c5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.1.ddl.aql
@@ -0,0 +1,19 @@
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type MyRecord as closed {
+  id: int32,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+}
+
+create external dataset MyData(MyRecord) using hdfs(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/spatialData.json"),("input-format"="text-input-format"),("format"="adm"));
+
+create index rtree_index_point on MyData(point) type rtree;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.3.query.aql
new file mode 100644
index 0000000..9986216
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.3.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('MyData')
+where spatial-intersect($o.point, create-polygon([4.0,1.0,4.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.1.ddl.aql
new file mode 100644
index 0000000..f149f35
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.1.ddl.aql
@@ -0,0 +1,23 @@
+/*
+* Description  : Create an external dataset that contains records stored with sequence hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type EmployeeType as closed {
+ id: int32,
+ name: string,
+ age: int32
+};
+
+create external dataset EmployeeDataset(EmployeeType)
+using hdfs
+(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.seq"),("input-format"="sequence-input-format"),("format"="delimited-text"),("delimiter"="|"));
+
+create index EmployeeAgeIdx on EmployeeDataset(age);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.2.update.aql
new file mode 100644
index 0000000..593a13b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.2.update.aql
@@ -0,0 +1,7 @@
+/*
+* Description  : Create an external dataset that contains records stored with sequence hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.3.query.aql
new file mode 100644
index 0000000..b714671
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.3.query.aql
@@ -0,0 +1,12 @@
+/*
+* Description  : Create an external dataset that contains records stored with sequence hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+use dataverse test;
+
+for $emp in dataset EmployeeDataset
+where $emp.age = 22
+return $emp;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.1.ddl.aql
new file mode 100644
index 0000000..8109531
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.1.ddl.aql
@@ -0,0 +1,23 @@
+/*
+* Description  : Create an external dataset that contains records stored with text hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type EmployeeType as closed {
+ id: int32,
+ name: string,
+ age: int32
+};
+
+create external dataset EmployeeDataset(EmployeeType)
+using hdfs
+(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.txt"),("input-format"="text-input-format"),("format"="delimited-text"),("delimiter"="|"));
+
+create index EmployeeAgeIdx on EmployeeDataset(age);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.2.update.aql
new file mode 100644
index 0000000..4fb3db0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.2.update.aql
@@ -0,0 +1,7 @@
+/*
+* Description  : Create an external dataset that contains records stored with text hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.3.query.aql
new file mode 100644
index 0000000..954e877
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.3.query.aql
@@ -0,0 +1,12 @@
+/*
+* Description  : Create an external dataset that contains records stored with text hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+use dataverse test;
+
+for $emp in dataset EmployeeDataset
+where $emp.age = 22
+return $emp;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm b/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
index 62d8a6c..660e2a8 100644
--- a/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
@@ -1,7 +1,7 @@
-{ "DataverseName": "test1", "DatasetName": "TwitterData", "DataTypeName": "Tweet", "DatasetType": "EXTERNAL", "InternalDetails": null, "ExternalDetails": { "DatasourceAdapter": "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter", "Properties": [ { "Name": "path", "Value": "nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ] }, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 119, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t1", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 113, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 116, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 117, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 114, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 115, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t4", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 118, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "test1", "DatasetName": "TwitterData", "DataTypeName": "Tweet", "DatasetType": "EXTERNAL", "InternalDetails": null, "ExternalDetails": { "DatasourceAdapter": "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter", "Properties": [ { "Name": "path", "Value": "nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ], "GroupName": "DEFAULT_NG_ALL_NODES", "LastRefreshTime": datetime("2014-06-08T20:30:43.724Z"), "TransactionState": 0, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 107, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t1", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:42 PDT 2014", "DatasetId": 101, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 104, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 105, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:42 PDT 2014", "DatasetId": 102, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:42 PDT 2014", "DatasetId": 103, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t4", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 106, "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.adm
new file mode 100644
index 0000000..b4337b3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.adm
@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "loc1": point("42.83,72.44"), "nearby-message": [ { "tweetid2": 1i64, "loc2": point("42.83,72.44") }, { "tweetid2": 55i64, "loc2": point("42.77,72.16") }, { "tweetid2": 114i64, "loc2": point("42.87,72.38") } ] }
+{ "tweetid1": 2i64, "loc1": point("34.81,72.44"), "nearby-message": [ { "tweetid2": 2i64, "loc2": point("34.81,72.44") } ] }
+{ "tweetid1": 3i64, "loc1": point("24.54,82.66"), "nearby-message": [ { "tweetid2": 3i64, "loc2": point("24.54,82.66") } ] }
+{ "tweetid1": 4i64, "loc1": point("38.14,68.1"), "nearby-message": [ { "tweetid2": 4i64, "loc2": point("38.14,68.1") } ] }
+{ "tweetid1": 5i64, "loc1": point("35.4,68.89"), "nearby-message": [ { "tweetid2": 5i64, "loc2": point("35.4,68.89") } ] }
+{ "tweetid1": 6i64, "loc1": point("42.75,78.5"), "nearby-message": [ { "tweetid2": 6i64, "loc2": point("42.75,78.5") } ] }
+{ "tweetid1": 7i64, "loc1": point("48.16,71.59"), "nearby-message": [ { "tweetid2": 7i64, "loc2": point("48.16,71.59") }, { "tweetid2": 42i64, "loc2": point("47.86,71.93") }, { "tweetid2": 192i64, "loc2": point("48.12,72.0") } ] }
+{ "tweetid1": 8i64, "loc1": point("36.17,72.56"), "nearby-message": [ { "tweetid2": 8i64, "loc2": point("36.17,72.56") } ] }
+{ "tweetid1": 9i64, "loc1": point("38.02,70.38"), "nearby-message": [ { "tweetid2": 9i64, "loc2": point("38.02,70.38") }, { "tweetid2": 51i64, "loc2": point("37.65,70.54") } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin/leftouterjoin.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin/leftouterjoin.1.adm
new file mode 100644
index 0000000..1907bca
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin/leftouterjoin.1.adm
@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "count1": 1, "t2info": [  ] }
+{ "tweetid1": 2i64, "count1": 2, "t2info": [ { "tweetid2": 60i64, "count2": 2 } ] }
+{ "tweetid1": 3i64, "count1": 3, "t2info": [ { "tweetid2": 105i64, "count2": 3 }, { "tweetid2": 206i64, "count2": 3 } ] }
+{ "tweetid1": 4i64, "count1": 4, "t2info": [  ] }
+{ "tweetid1": 5i64, "count1": 5, "t2info": [ { "tweetid2": 138i64, "count2": 5 }, { "tweetid2": 175i64, "count2": 5 } ] }
+{ "tweetid1": 6i64, "count1": 6, "t2info": [ { "tweetid2": 148i64, "count2": 6 } ] }
+{ "tweetid1": 7i64, "count1": 7, "t2info": [ { "tweetid2": 125i64, "count2": 7 } ] }
+{ "tweetid1": 8i64, "count1": 8, "t2info": [  ] }
+{ "tweetid1": 9i64, "count1": 9, "t2info": [ { "tweetid2": 141i64, "count2": 9 } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/rc-format/rc-format.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/rc-format/rc-format.1.adm
new file mode 100644
index 0000000..3ee4a1e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/rc-format/rc-format.1.adm
@@ -0,0 +1,2 @@
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/rtree-index/rtree-index.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/rtree-index/rtree-index.1.adm
new file mode 100644
index 0000000..d22217a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/rtree-index/rtree-index.1.adm
@@ -0,0 +1,2 @@
+{ "id": 12 }
+{ "id": 20 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/sequence-format/sequence-format.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/sequence-format/sequence-format.1.adm
new file mode 100644
index 0000000..3ee4a1e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/sequence-format/sequence-format.1.adm
@@ -0,0 +1,2 @@
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/text-format/text-format.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/text-format/text-format.1.adm
new file mode 100644
index 0000000..3ee4a1e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/text-format/text-format.1.adm
@@ -0,0 +1,2 @@
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm b/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
index f3636a4..1b4a1f6 100644
--- a/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
@@ -1,6 +1,6 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 13, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 2, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 8, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 3, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 1, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 10, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 13, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 2, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 8, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 3, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 1, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DataTypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "FileNumber" ], "PrimaryKey": [ "DataverseName", "DatasetName", "FileNumber" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 14, "PendingOp": 0 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 8cf4cd9..6344bc1 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -4896,6 +4896,38 @@
       </compilation-unit>
     </test-case>
   </test-group>
+  <test-group name="external-indexing">
+  	<test-case FilePath="external-indexing">
+      <compilation-unit name="text-format">
+        <output-dir compare="Text">text-format</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="sequence-format">
+        <output-dir compare="Text">sequence-format</output-dir>
+      </compilation-unit>
+    </test-case>
+  	<test-case FilePath="external-indexing">
+      <compilation-unit name="rc-format">
+        <output-dir compare="Text">rc-format</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="rtree-index">
+        <output-dir compare="Text">rtree-index</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="leftouterjoin">
+        <output-dir compare="Text">leftouterjoin</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="leftouterjoin-rtree">
+        <output-dir compare="Text">leftouterjoin-rtree</output-dir>
+      </compilation-unit>
+    </test-case>
+  </test-group>
   <test-group name="temporal">
     <test-case FilePath="temporal">
      <compilation-unit name="parse_02">