Add external indexes This change include the following: 1. additional data parser for external data that parse hdfs records using Hive serdes. 2. allow users to create external data. this includes: a) changes in metadata external dataset details. b) addition of a new metadata index to store external file's statuses. c) the pipeline for building the B-Tree and R-Tree indexes. d) hyracks operators to fetch records with their RIDs using different formats. e) hyracks operators to lookup and parse external records. f) test cases for indexing and index access of different hdfs file formats. g) exposing the secondary indexes over external data to the compiler. 3. adding a new aql command to refresh external datasets. this includes a) global recovery on system startup. b) changes in the aql parser. c) construction of bulk modify pipelines and additional operators to perform local commit and abort operations (using 2PC protocol). 4. Added copyright header to all new files 5. Added additional test cases to test left outer join on external data Change-Id: I1065a473299f6027eb073aeeba3a56d137f6f98e Reviewed-on: http://fulliautomatix.ics.uci.edu:8443/70 Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu> Reviewed-by: Ian Maxon <imaxon@uci.edu>

commit: d905773df2460db0d163301f3d7685c972932204 [log] [tgz]
author: Abdullah Alamoudi <alamouda@uci.edu> Thu Jun 12 13:38:27 2014 -0700
committer: Ian Maxon <imaxon@uci.edu> Thu Jun 12 18:19:01 2014 -0800
tree: f6d31d0a1976d21465bc286be83ca67f908e739a
parent: d5f0fc5bb57f1b3b7bb2f647355d11c2fb137bba [diff]
diff --git a/asterix-app/data/hdfs/external-indexing-test.rc b/asterix-app/data/hdfs/external-indexing-test.rc
new file mode 100644
index 0000000..4801881
--- /dev/null
+++ b/asterix-app/data/hdfs/external-indexing-test.rc
Binary files differ

diff --git a/asterix-app/data/hdfs/external-indexing-test.seq b/asterix-app/data/hdfs/external-indexing-test.seq
new file mode 100644
index 0000000..0146c83
--- /dev/null
+++ b/asterix-app/data/hdfs/external-indexing-test.seq
Binary files differ

diff --git a/asterix-app/data/hdfs/external-indexing-test.txt b/asterix-app/data/hdfs/external-indexing-test.txt
new file mode 100644
index 0000000..555272e
--- /dev/null
+++ b/asterix-app/data/hdfs/external-indexing-test.txt

@@ -0,0 +1,11 @@
+1|Steve|50
+2|John|23
+3|Samuel|22
+4|Mary|29
+5|William|75
+6|Sarah|16
+7|Noel|33
+8|Carlos|40
+9|Joseph|45
+10|David|22
+11|Nadine|10

diff --git a/asterix-app/data/hdfs/spatialData.json b/asterix-app/data/hdfs/spatialData.json
new file mode 100644
index 0000000..9c78064
--- /dev/null
+++ b/asterix-app/data/hdfs/spatialData.json

@@ -0,0 +1,21 @@
+{"id": 1, "point": point("4.1,7.0"), "kwds": "sign ahead", "line1": line("4.0,7.0 9.0,7.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.2,7.0"), "circle" : circle("1.0,1.0 10.0")}
+{"id": 2, "point": point("40.2152,-75.0449"), "kwds": "factory hosedan", "line1": line("-4.0,2.0 2.0,2.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("2.0,3.0 2.0")}
+{"id": 3, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("3.0,0.0 0.0,4.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("5.5,1.0 10.0")}
+{"id": 4, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0 2.0,1.0 1.0,0.0"), "poly2": polygon("2.0,1.0 2.0,2.0 3.0,2.0 3.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("77.0,4.0 30.0")}
+{"id": 5, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("100.0,100.0 100.0,400.0 300.0,400.0 300.0,100.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("88.0,1.0 10.0")}
+{"id": 6, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 1.0,7.0"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("3.1,1.0 2.9,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,1.0 10.0")}
+{"id": 7, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("0.0,5.0 4.0,7.1"), "line2": line("4.0,7.0 2.0,-17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("3.0,1.0 3.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("3.0,6.0 5.0,7.0"), "circle" : circle("13.0,75.0 1.0")}
+{"id": 8, "point": point("43.5083,-79.3007"), "kwds": "enterprisecamp torcamp", "line1": line("4.0,7.0 2.0,17.0"), "line2": line("4.0,7.0 2.0,17.0"), "poly1": polygon("-5.0,-2.0 -4.0,-1.0 -3.0,-1.0 -2.0,-2.0 -4.0,-4.0 -5.0,-3.0"), "poly2": polygon("-3.0,-3.0 -1.0,-3.0 -3.0,-5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("76.0,87.0 50.0")}
+{"id": 9, "point": point("5.0,1.0"), "kwds": "sign ahead", "line1": line("5.0,1.0 5.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("1.0,1.0 1.0,4.0 3.0,4.0 3.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("11.0,14.0 15.0")}
+{"id": 10, "point": point("2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("6.01,1.0 6.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,76.0 17.0")}
+{"id": 11, "point": point("4.9,0.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("4.9,0.1 4.9,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("22.0,35.0 144.0")}
+{"id": 12, "point": point("6.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("4.0,1.0 4.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,23.0 12.0")}
+{"id": 13, "point": point("5.0,5.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("6.0,1.0 6.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("30.0,11.0 11.0")}
+{"id": 14, "point": point("5.1,5.1"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 5.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,66.0 17.0")}
+{"id": 15, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.1,1.0 5.1,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("12.0,87.0 10.0")}
+{"id": 16, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,35.0 10.0")}
+{"id": 17, "point": point("4.1,7.0"), "kwds": "sign ahead", "line1": line("4.0,7.0 9.0,7.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("0.0,6.0 0.0,0.0 3.0,0.0 4.0,1.0 6.0,1.0 8.0,0.0 12.0,0.0 13.0,2.0 8.0,2.0 8.0,4.0 11.0,4.0 11.0,6.0 6.0,6.0 4.0,3.0 2.0,6.0"), "poly2": polygon("5.0,1.0 5.0,4.0 7.0,4.0 7.0,1.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,51.0 10.0")}
+{"id": 18, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 7.0,1.0 7.0,4.0 6.0,2.0 5.0,4.0"), "poly2": polygon("6.0,3.0 7.0,5.0 6.0,7.0 5.0,5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("43.0,45.0 12.0")}
+{"id": 19, "point": point("-2.0,3.0"), "kwds": "sign ahead", "line1": line("1.0,2.0 3.0,4.0"), "line2": line("5.0,8.0 5.0,1.0"), "poly1": polygon("5.0,1.0 7.0,1.0 7.0,4.0 6.0,2.0 5.0,4.0"), "poly2": polygon("6.0,1.0 7.0,5.0 6.0,7.0 5.0,5.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("65.0,2.0 13.0")}
+{"id": 20, "point": point("4.0,3.0"), "kwds": "sign ahead", "line1": line("20.0,20.0 30.0,40.0"), "line2": line("5.0,8.0 0.0,1.0"), "poly1": polygon("4.0,1.0 4.0,4.0 12.0,4.0 12.0,1.0"), "poly2": polygon("50.0,10.0 50.0,40.0 70.0,40.0 70.0,10.0"), "rec": rectangle("0.0,0.0 4.0,4.0"), "circle" : circle("1.0,23.0 12.0")}
+{"id": 21, "point": point("0.0,5.0"), "kwds": "sign ahead", "line1": line("0.0,5.0 0.0,40.0"), "line2": line("5.0,8.0 0.0,1.0"), "poly1": polygon("5.1,5.1 14.0,14.0 22.0,14.0 22.0,10.0"), "poly2": polygon("50.0,10.0 50.0,40.0 70.0,40.0 70.0,10.0"), "rec": rectangle("0.0,0.0 5.1,5.1"), "circle" : circle("1.0,23.0 12.0")}
\ No newline at end of file

diff --git a/asterix-app/data/hdfs/tw_for_indexleftouterjoin.adm b/asterix-app/data/hdfs/tw_for_indexleftouterjoin.adm
new file mode 100644
index 0000000..405dd08
--- /dev/null
+++ b/asterix-app/data/hdfs/tw_for_indexleftouterjoin.adm

@@ -0,0 +1,250 @@
+{ "tweetid": 1i64, "user": { "screen-name": "WardLoewentsein@340", "lang": "en", "friends-count": 11, "statuses-count": 388, "name": "Ward Loewentsein", "followers-count": 129 }, "sender-location": point("42.83,72.44"), "send-time": datetime("2009-10-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile the speed is mind-blowing:)", "countA": 1, "countB": 26 }
+{ "tweetid": 2i64, "user": { "screen-name": "KyleGraham_120", "lang": "en", "friends-count": 55, "statuses-count": 231, "name": "Kyle Graham", "followers-count": 42 }, "sender-location": point("34.81,72.44"), "send-time": datetime("2011-09-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " hate samsung the 3G is horrible", "countA": 2, "countB": 131 }
+{ "tweetid": 3i64, "user": { "screen-name": "TateGarneys@542", "lang": "en", "friends-count": 74, "statuses-count": 370, "name": "Tate Garneys", "followers-count": 111 }, "sender-location": point("24.54,82.66"), "send-time": datetime("2009-12-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " hate iphone the shortcut-menu is bad", "countA": 3, "countB": 187 }
+{ "tweetid": 4i64, "user": { "screen-name": "BuckFields@708", "lang": "en", "friends-count": 20, "statuses-count": 469, "name": "Buck Fields", "followers-count": 181 }, "sender-location": point("38.14,68.1"), "send-time": datetime("2008-10-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " dislike samsung the speed is OMG", "countA": 4, "countB": 52 }
+{ "tweetid": 5i64, "user": { "screen-name": "NoreenBaldwin_373", "lang": "en", "friends-count": 89, "statuses-count": 144, "name": "Noreen Baldwin", "followers-count": 187 }, "sender-location": point("35.4,68.89"), "send-time": datetime("2008-10-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " hate motorola its 3G is OMG:(", "countA": 5, "countB": 35 }
+{ "tweetid": 6i64, "user": { "screen-name": "IselaHatcher_237", "lang": "en", "friends-count": 85, "statuses-count": 333, "name": "Isela Hatcher", "followers-count": 148 }, "sender-location": point("42.75,78.5"), "send-time": datetime("2011-10-15T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " hate sprint the wireless is terrible:(", "countA": 6, "countB": 61 }
+{ "tweetid": 7i64, "user": { "screen-name": "NicolaJolce$660", "lang": "en", "friends-count": 45, "statuses-count": 420, "name": "Nicola Jolce", "followers-count": 12 }, "sender-location": point("48.16,71.59"), "send-time": datetime("2005-11-23T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-command" }}, "message-text": " like motorola its voice-command is amazing", "countA": 7, "countB": 47 }
+{ "tweetid": 8i64, "user": { "screen-name": "MorganKeppel_176", "lang": "en", "friends-count": 74, "statuses-count": 190, "name": "Morgan Keppel", "followers-count": 2 }, "sender-location": point("36.17,72.56"), "send-time": datetime("2011-12-02T10:10:00.000Z"), "referred-topics": {{ "verizon", "3G" }}, "message-text": " hate verizon the 3G is OMG:(", "countA": 8, "countB": 98 }
+{ "tweetid": 9i64, "user": { "screen-name": "GerardMcdonald$43", "lang": "en", "friends-count": 72, "statuses-count": 151, "name": "Gerard Mcdonald", "followers-count": 96 }, "sender-location": point("38.02,70.38"), "send-time": datetime("2005-10-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " love sprint its voice-clarity is amazing", "countA": 9, "countB": 69 }
+{ "tweetid": 10i64, "user": { "screen-name": "WynonnaButler_286", "lang": "en", "friends-count": 30, "statuses-count": 375, "name": "Wynonna Butler", "followers-count": 78 }, "sender-location": point("38.71,90.05"), "send-time": datetime("2008-09-21T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " love motorola its wireless is good:)", "countA": 10, "countB": 75 }
+{ "tweetid": 11i64, "user": { "screen-name": "BrodyKing@977", "lang": "en", "friends-count": 3, "statuses-count": 62, "name": "Brody King", "followers-count": 106 }, "sender-location": point("32.26,73.48"), "send-time": datetime("2007-05-20T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " can't stand sprint the shortcut-menu is OMG", "countA": 11, "countB": 28 }
+{ "tweetid": 12i64, "user": { "screen-name": "ValentineSchofield@448", "lang": "en", "friends-count": 16, "statuses-count": 260, "name": "Valentine Schofield", "followers-count": 136 }, "sender-location": point("24.99,70.66"), "send-time": datetime("2009-07-26T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " can't stand sprint its shortcut-menu is terrible", "countA": 12, "countB": 159 }
+{ "tweetid": 13i64, "user": { "screen-name": "MaryroseBennett#483", "lang": "en", "friends-count": 99, "statuses-count": 496, "name": "Maryrose Bennett", "followers-count": 57 }, "sender-location": point("38.94,93.31"), "send-time": datetime("2005-01-06T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " love at&t its platform is awesome", "countA": 13, "countB": 13 }
+{ "tweetid": 14i64, "user": { "screen-name": "GarlandAlliman$490", "lang": "en", "friends-count": 23, "statuses-count": 146, "name": "Garland Alliman", "followers-count": 51 }, "sender-location": point("29.96,75.0"), "send-time": datetime("2009-09-26T10:10:00.000Z"), "referred-topics": {{ "motorola", "platform" }}, "message-text": " dislike motorola the platform is bad:(", "countA": 14, "countB": 74 }
+{ "tweetid": 15i64, "user": { "screen-name": "AnnabelPirl$171", "lang": "en", "friends-count": 43, "statuses-count": 402, "name": "Annabel Pirl", "followers-count": 137 }, "sender-location": point("27.22,86.32"), "send-time": datetime("2008-09-08T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " like at&t the network is amazing:)", "countA": 15, "countB": 19 }
+{ "tweetid": 16i64, "user": { "screen-name": "JarvisPickering@42", "lang": "en", "friends-count": 98, "statuses-count": 498, "name": "Jarvis Pickering", "followers-count": 20 }, "sender-location": point("30.54,81.6"), "send-time": datetime("2005-11-05T10:10:00.000Z"), "referred-topics": {{ "iphone", "wireless" }}, "message-text": " love iphone the wireless is awesome", "countA": 16, "countB": 27 }
+{ "tweetid": 17i64, "user": { "screen-name": "LillyHoffhants@595", "lang": "en", "friends-count": 35, "statuses-count": 391, "name": "Lilly Hoffhants", "followers-count": 129 }, "sender-location": point("47.96,95.21"), "send-time": datetime("2007-02-14T10:10:00.000Z"), "referred-topics": {{ "verizon", "customer-service" }}, "message-text": " like verizon its customer-service is amazing", "countA": 17, "countB": 55 }
+{ "tweetid": 18i64, "user": { "screen-name": "AllanPolson_455", "lang": "en", "friends-count": 36, "statuses-count": 227, "name": "Allan Polson", "followers-count": 113 }, "sender-location": point("46.39,73.85"), "send-time": datetime("2009-12-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " hate iphone the platform is bad", "countA": 18, "countB": 199 }
+{ "tweetid": 19i64, "user": { "screen-name": "DonnieWentzel#857", "lang": "en", "friends-count": 76, "statuses-count": 23, "name": "Donnie Wentzel", "followers-count": 78 }, "sender-location": point("37.21,95.76"), "send-time": datetime("2012-04-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "reachability" }}, "message-text": " like iphone the reachability is mind-blowing", "countA": 19, "countB": 68 }
+{ "tweetid": 20i64, "user": { "screen-name": "TraversFaast@428", "lang": "en", "friends-count": 42, "statuses-count": 70, "name": "Travers Faast", "followers-count": 116 }, "sender-location": point("26.54,74.71"), "send-time": datetime("2010-06-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone the shortcut-menu is mind-blowing", "countA": 20, "countB": 18 }
+{ "tweetid": 21i64, "user": { "screen-name": "KameronSandford#555", "lang": "en", "friends-count": 22, "statuses-count": 104, "name": "Kameron Sandford", "followers-count": 17 }, "sender-location": point("25.8,88.76"), "send-time": datetime("2007-11-20T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " can't stand at&t the network is bad:(", "countA": 21, "countB": 64 }
+{ "tweetid": 22i64, "user": { "screen-name": "TiannaArmitage_372", "lang": "en", "friends-count": 32, "statuses-count": 307, "name": "Tianna Armitage", "followers-count": 126 }, "sender-location": point("45.79,80.75"), "send-time": datetime("2005-04-14T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola its reachability is terrible:(", "countA": 22, "countB": 126 }
+{ "tweetid": 23i64, "user": { "screen-name": "NevadaCattley#858", "lang": "en", "friends-count": 74, "statuses-count": 389, "name": "Nevada Cattley", "followers-count": 17 }, "sender-location": point("43.6,93.24"), "send-time": datetime("2007-03-11T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " hate at&t the network is horrible", "countA": 23, "countB": 74 }
+{ "tweetid": 24i64, "user": { "screen-name": "ReannonEisenhart#637", "lang": "en", "friends-count": 34, "statuses-count": 235, "name": "Reannon Eisenhart", "followers-count": 145 }, "sender-location": point("47.56,76.49"), "send-time": datetime("2012-03-20T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-clarity" }}, "message-text": " can't stand samsung its voice-clarity is terrible:(", "countA": 24, "countB": 70 }
+{ "tweetid": 25i64, "user": { "screen-name": "LillyMang#928", "lang": "en", "friends-count": 20, "statuses-count": 40, "name": "Lilly Mang", "followers-count": 47 }, "sender-location": point("38.68,94.93"), "send-time": datetime("2006-04-21T10:10:00.000Z"), "referred-topics": {{ "sprint", "customization" }}, "message-text": " love sprint its customization is awesome", "countA": 25, "countB": 86 }
+{ "tweetid": 26i64, "user": { "screen-name": "MicaBusk$903", "lang": "en", "friends-count": 87, "statuses-count": 164, "name": "Mica Busk", "followers-count": 92 }, "sender-location": point("45.47,90.97"), "send-time": datetime("2008-01-24T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " hate motorola the touch-screen is terrible", "countA": 26, "countB": 11 }
+{ "tweetid": 27i64, "user": { "screen-name": "PiaHildyard_915", "lang": "en", "friends-count": 92, "statuses-count": 302, "name": "Pia Hildyard", "followers-count": 16 }, "sender-location": point("43.76,68.58"), "send-time": datetime("2007-08-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " dislike iphone its platform is bad:(", "countA": 27, "countB": 125 }
+{ "tweetid": 28i64, "user": { "screen-name": "CamelliaSiegrist_676", "lang": "en", "friends-count": 73, "statuses-count": 392, "name": "Camellia Siegrist", "followers-count": 193 }, "sender-location": point("24.94,77.95"), "send-time": datetime("2007-12-26T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-command" }}, "message-text": " can't stand verizon its voice-command is bad", "countA": 28, "countB": 123 }
+{ "tweetid": 29i64, "user": { "screen-name": "BurtTaggart_922", "lang": "en", "friends-count": 49, "statuses-count": 62, "name": "Burt Taggart", "followers-count": 134 }, "sender-location": point("35.67,97.43"), "send-time": datetime("2011-07-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is awesome", "countA": 29, "countB": 115 }
+{ "tweetid": 30i64, "user": { "screen-name": "MarlaHill@215", "lang": "en", "friends-count": 84, "statuses-count": 305, "name": "Marla Hill", "followers-count": 71 }, "sender-location": point("24.29,84.28"), "send-time": datetime("2012-07-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "3G" }}, "message-text": " like t-mobile the 3G is awesome", "countA": 30, "countB": 36 }
+{ "tweetid": 31i64, "user": { "screen-name": "CaitlynChristman@452", "lang": "en", "friends-count": 57, "statuses-count": 414, "name": "Caitlyn Christman", "followers-count": 67 }, "sender-location": point("41.04,85.13"), "send-time": datetime("2005-11-05T10:10:00.000Z"), "referred-topics": {{ "samsung", "voicemail-service" }}, "message-text": " love samsung the voicemail-service is mind-blowing:)", "countA": 31, "countB": 59 }
+{ "tweetid": 32i64, "user": { "screen-name": "BraxtonBonner#527", "lang": "en", "friends-count": 21, "statuses-count": 427, "name": "Braxton Bonner", "followers-count": 168 }, "sender-location": point("34.25,86.09"), "send-time": datetime("2011-10-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " dislike motorola its 3G is horrible:(", "countA": 32, "countB": 38 }
+{ "tweetid": 33i64, "user": { "screen-name": "WilmaSouthern@238", "lang": "en", "friends-count": 83, "statuses-count": 413, "name": "Wilma Southern", "followers-count": 24 }, "sender-location": point("34.71,69.57"), "send-time": datetime("2005-02-19T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " dislike sprint the wireless is OMG:(", "countA": 33, "countB": 105 }
+{ "tweetid": 34i64, "user": { "screen-name": "MaxNash$802", "lang": "en", "friends-count": 13, "statuses-count": 189, "name": "Max Nash", "followers-count": 39 }, "sender-location": point("48.12,89.23"), "send-time": datetime("2012-02-17T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " dislike motorola its touch-screen is horrible", "countA": 34, "countB": 185 }
+{ "tweetid": 35i64, "user": { "screen-name": "HannahWarrick_843", "lang": "en", "friends-count": 14, "statuses-count": 10, "name": "Hannah Warrick", "followers-count": 2 }, "sender-location": point("32.75,69.94"), "send-time": datetime("2007-09-14T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "plan" }}, "message-text": " like t-mobile the plan is amazing:)", "countA": 35, "countB": 176 }
+{ "tweetid": 36i64, "user": { "screen-name": "SherikaBarth#732", "lang": "en", "friends-count": 80, "statuses-count": 277, "name": "Sherika Barth", "followers-count": 138 }, "sender-location": point("34.85,66.87"), "send-time": datetime("2011-10-13T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " can't stand at&t the plan is OMG", "countA": 36, "countB": 147 }
+{ "tweetid": 37i64, "user": { "screen-name": "SabinaCattley$355", "lang": "en", "friends-count": 67, "statuses-count": 20, "name": "Sabina Cattley", "followers-count": 104 }, "sender-location": point("40.22,71.18"), "send-time": datetime("2007-04-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-clarity" }}, "message-text": " can't stand samsung its voice-clarity is bad:(", "countA": 37, "countB": 55 }
+{ "tweetid": 38i64, "user": { "screen-name": "KimberlyVeith$848", "lang": "en", "friends-count": 43, "statuses-count": 274, "name": "Kimberly Veith", "followers-count": 14 }, "sender-location": point("34.94,83.17"), "send-time": datetime("2011-07-15T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " hate samsung the speed is horrible:(", "countA": 38, "countB": 89 }
+{ "tweetid": 39i64, "user": { "screen-name": "AdrianneMackendoerfer_478", "lang": "en", "friends-count": 2, "statuses-count": 125, "name": "Adrianne Mackendoerfer", "followers-count": 113 }, "sender-location": point("40.14,78.49"), "send-time": datetime("2010-10-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-command" }}, "message-text": " hate motorola its voice-command is OMG", "countA": 39, "countB": 97 }
+{ "tweetid": 40i64, "user": { "screen-name": "MunroWire@995", "lang": "en", "friends-count": 89, "statuses-count": 336, "name": "Munro Wire", "followers-count": 181 }, "sender-location": point("30.94,80.83"), "send-time": datetime("2009-05-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " love verizon the network is good", "countA": 40, "countB": 193 }
+{ "tweetid": 41i64, "user": { "screen-name": "AmadaAft@648", "lang": "en", "friends-count": 50, "statuses-count": 127, "name": "Amada Aft", "followers-count": 20 }, "sender-location": point("32.88,81.46"), "send-time": datetime("2010-04-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " can't stand iphone its customer-service is OMG:(", "countA": 41, "countB": 169 }
+{ "tweetid": 42i64, "user": { "screen-name": "SalenaMcfall_717", "lang": "en", "friends-count": 30, "statuses-count": 93, "name": "Salena Mcfall", "followers-count": 184 }, "sender-location": point("47.86,71.93"), "send-time": datetime("2010-02-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "network" }}, "message-text": " can't stand t-mobile its network is bad", "countA": 42, "countB": 130 }
+{ "tweetid": 43i64, "user": { "screen-name": "JeniferCanham$317", "lang": "en", "friends-count": 63, "statuses-count": 344, "name": "Jenifer Canham", "followers-count": 132 }, "sender-location": point("25.68,81.87"), "send-time": datetime("2010-04-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " hate motorola the 3G is OMG:(", "countA": 43, "countB": 153 }
+{ "tweetid": 44i64, "user": { "screen-name": "NannieBender$656", "lang": "en", "friends-count": 26, "statuses-count": 84, "name": "Nannie Bender", "followers-count": 184 }, "sender-location": point("47.46,85.04"), "send-time": datetime("2007-06-08T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " dislike samsung its voice-command is bad", "countA": 44, "countB": 12 }
+{ "tweetid": 45i64, "user": { "screen-name": "ThaoKooser@875", "lang": "en", "friends-count": 60, "statuses-count": 289, "name": "Thao Kooser", "followers-count": 8 }, "sender-location": point("37.02,87.94"), "send-time": datetime("2005-11-28T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " like verizon its network is amazing:)", "countA": 45, "countB": 151 }
+{ "tweetid": 46i64, "user": { "screen-name": "AugustaBaumgartner_385", "lang": "en", "friends-count": 17, "statuses-count": 70, "name": "Augusta Baumgartner", "followers-count": 162 }, "sender-location": point("24.83,73.16"), "send-time": datetime("2008-09-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " like verizon its network is mind-blowing", "countA": 46, "countB": 37 }
+{ "tweetid": 47i64, "user": { "screen-name": "OtisHill_124", "lang": "en", "friends-count": 46, "statuses-count": 68, "name": "Otis Hill", "followers-count": 29 }, "sender-location": point("36.01,86.76"), "send-time": datetime("2011-05-16T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " dislike t-mobile the touch-screen is horrible:(", "countA": 47, "countB": 27 }
+{ "tweetid": 48i64, "user": { "screen-name": "ZolaJudge@572", "lang": "en", "friends-count": 8, "statuses-count": 39, "name": "Zola Judge", "followers-count": 36 }, "sender-location": point("42.67,91.8"), "send-time": datetime("2009-03-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " like motorola the network is awesome", "countA": 48, "countB": 16 }
+{ "tweetid": 49i64, "user": { "screen-name": "ChristianaWeisgarber$35", "lang": "en", "friends-count": 40, "statuses-count": 427, "name": "Christiana Weisgarber", "followers-count": 20 }, "sender-location": point("36.91,86.0"), "send-time": datetime("2010-04-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " hate sprint its plan is terrible", "countA": 49, "countB": 28 }
+{ "tweetid": 50i64, "user": { "screen-name": "MollyGarneis_210", "lang": "en", "friends-count": 6, "statuses-count": 453, "name": "Molly Garneis", "followers-count": 185 }, "sender-location": point("44.42,87.86"), "send-time": datetime("2012-04-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " love t-mobile its wireless is good:)", "countA": 50, "countB": 123 }
+{ "tweetid": 51i64, "user": { "screen-name": "HarlanHanseu#420", "lang": "en", "friends-count": 66, "statuses-count": 295, "name": "Harlan Hanseu", "followers-count": 99 }, "sender-location": point("37.65,70.54"), "send-time": datetime("2006-05-11T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " can't stand samsung the speed is terrible:(", "countA": 51, "countB": 94 }
+{ "tweetid": 52i64, "user": { "screen-name": "DelorseSloan#229", "lang": "en", "friends-count": 84, "statuses-count": 287, "name": "Delorse Sloan", "followers-count": 20 }, "sender-location": point("27.12,78.69"), "send-time": datetime("2011-02-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " dislike motorola its touch-screen is horrible:(", "countA": 52, "countB": 156 }
+{ "tweetid": 53i64, "user": { "screen-name": "MylesEwing@54", "lang": "en", "friends-count": 45, "statuses-count": 411, "name": "Myles Ewing", "followers-count": 23 }, "sender-location": point("25.82,97.9"), "send-time": datetime("2007-11-10T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " like at&t its network is mind-blowing:)", "countA": 53, "countB": 174 }
+{ "tweetid": 54i64, "user": { "screen-name": "OprahClark_160", "lang": "en", "friends-count": 26, "statuses-count": 299, "name": "Oprah Clark", "followers-count": 161 }, "sender-location": point("36.1,87.24"), "send-time": datetime("2010-04-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile its shortcut-menu is good", "countA": 54, "countB": 128 }
+{ "tweetid": 55i64, "user": { "screen-name": "CoreyRichards#130", "lang": "en", "friends-count": 45, "statuses-count": 420, "name": "Corey Richards", "followers-count": 102 }, "sender-location": point("42.77,72.16"), "send-time": datetime("2012-07-20T10:10:00.000Z"), "referred-topics": {{ "at&t", "customer-service" }}, "message-text": " can't stand at&t its customer-service is horrible", "countA": 55, "countB": 30 }
+{ "tweetid": 56i64, "user": { "screen-name": "GwendolenHahn#673", "lang": "en", "friends-count": 62, "statuses-count": 426, "name": "Gwendolen Hahn", "followers-count": 158 }, "sender-location": point("39.76,90.94"), "send-time": datetime("2009-05-10T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " dislike samsung its 3G is bad:(", "countA": 56, "countB": 50 }
+{ "tweetid": 57i64, "user": { "screen-name": "DewayneBallou@258", "lang": "en", "friends-count": 42, "statuses-count": 215, "name": "Dewayne Ballou", "followers-count": 85 }, "sender-location": point("28.45,75.02"), "send-time": datetime("2006-08-07T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile the speed is mind-blowing:)", "countA": 57, "countB": 61 }
+{ "tweetid": 58i64, "user": { "screen-name": "RenayReese@543", "lang": "en", "friends-count": 80, "statuses-count": 459, "name": "Renay Reese", "followers-count": 102 }, "sender-location": point("38.09,77.66"), "send-time": datetime("2007-12-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " love motorola the speed is awesome:)", "countA": 58, "countB": 35 }
+{ "tweetid": 59i64, "user": { "screen-name": "SamuelHoffhants@740", "lang": "en", "friends-count": 53, "statuses-count": 64, "name": "Samuel Hoffhants", "followers-count": 150 }, "sender-location": point("48.86,97.19"), "send-time": datetime("2009-11-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " dislike sprint its speed is horrible", "countA": 59, "countB": 95 }
+{ "tweetid": 60i64, "user": { "screen-name": "GarlandOneal@886", "lang": "en", "friends-count": 12, "statuses-count": 481, "name": "Garland Oneal", "followers-count": 13 }, "sender-location": point("37.6,75.74"), "send-time": datetime("2005-05-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint its touch-screen is OMG:(", "countA": 60, "countB": 2 }
+{ "tweetid": 61i64, "user": { "screen-name": "LukeHoopengarner@327", "lang": "en", "friends-count": 77, "statuses-count": 224, "name": "Luke Hoopengarner", "followers-count": 107 }, "sender-location": point("46.38,80.88"), "send-time": datetime("2006-11-13T10:10:00.000Z"), "referred-topics": {{ "sprint", "voicemail-service" }}, "message-text": " can't stand sprint its voicemail-service is bad:(", "countA": 61, "countB": 12 }
+{ "tweetid": 62i64, "user": { "screen-name": "AudieStahl@296", "lang": "en", "friends-count": 89, "statuses-count": 90, "name": "Audie Stahl", "followers-count": 74 }, "sender-location": point("32.48,96.01"), "send-time": datetime("2010-04-23T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t its platform is terrible:(", "countA": 62, "countB": 47 }
+{ "tweetid": 63i64, "user": { "screen-name": "ArielleErrett_963", "lang": "en", "friends-count": 15, "statuses-count": 385, "name": "Arielle Errett", "followers-count": 34 }, "sender-location": point("25.39,82.02"), "send-time": datetime("2012-02-20T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " love verizon its signal is awesome:)", "countA": 63, "countB": 149 }
+{ "tweetid": 64i64, "user": { "screen-name": "MiltonWeldi#571", "lang": "en", "friends-count": 72, "statuses-count": 236, "name": "Milton Weldi", "followers-count": 128 }, "sender-location": point("26.08,85.94"), "send-time": datetime("2007-09-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " like sprint the reachability is awesome", "countA": 64, "countB": 158 }
+{ "tweetid": 65i64, "user": { "screen-name": "CarlineAft_666", "lang": "en", "friends-count": 25, "statuses-count": 352, "name": "Carline Aft", "followers-count": 59 }, "sender-location": point("29.33,78.49"), "send-time": datetime("2012-01-18T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " can't stand samsung the customization is terrible:(", "countA": 65, "countB": 77 }
+{ "tweetid": 66i64, "user": { "screen-name": "TenaGronko#55", "lang": "en", "friends-count": 91, "statuses-count": 2, "name": "Tena Gronko", "followers-count": 19 }, "sender-location": point("40.14,73.21"), "send-time": datetime("2011-03-18T10:10:00.000Z"), "referred-topics": {{ "at&t", "voice-command" }}, "message-text": " hate at&t the voice-command is horrible", "countA": 66, "countB": 73 }
+{ "tweetid": 67i64, "user": { "screen-name": "DanialBrinigh#499", "lang": "en", "friends-count": 67, "statuses-count": 413, "name": "Danial Brinigh", "followers-count": 4 }, "sender-location": point("41.26,97.09"), "send-time": datetime("2008-02-28T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " can't stand motorola the 3G is OMG:(", "countA": 67, "countB": 47 }
+{ "tweetid": 68i64, "user": { "screen-name": "GretaBusk#270", "lang": "en", "friends-count": 30, "statuses-count": 150, "name": "Greta Busk", "followers-count": 124 }, "sender-location": point("30.35,86.51"), "send-time": datetime("2008-03-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola the reachability is bad:(", "countA": 68, "countB": 21 }
+{ "tweetid": 69i64, "user": { "screen-name": "DanielBurch@155", "lang": "en", "friends-count": 5, "statuses-count": 268, "name": "Daniel Burch", "followers-count": 178 }, "sender-location": point("45.31,66.89"), "send-time": datetime("2009-06-27T10:10:00.000Z"), "referred-topics": {{ "samsung", "network" }}, "message-text": " hate samsung its network is horrible", "countA": 69, "countB": 153 }
+{ "tweetid": 70i64, "user": { "screen-name": "JaynaBash@532", "lang": "en", "friends-count": 90, "statuses-count": 244, "name": "Jayna Bash", "followers-count": 184 }, "sender-location": point("43.92,69.28"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " can't stand samsung the platform is bad", "countA": 70, "countB": 133 }
+{ "tweetid": 71i64, "user": { "screen-name": "PatriciaCason#475", "lang": "en", "friends-count": 50, "statuses-count": 149, "name": "Patricia Cason", "followers-count": 114 }, "sender-location": point("43.74,69.29"), "send-time": datetime("2009-08-28T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is amazing:)", "countA": 71, "countB": 185 }
+{ "tweetid": 72i64, "user": { "screen-name": "KatharineElsas_215", "lang": "en", "friends-count": 69, "statuses-count": 128, "name": "Katharine Elsas", "followers-count": 114 }, "sender-location": point("29.05,94.41"), "send-time": datetime("2010-09-25T10:10:00.000Z"), "referred-topics": {{ "samsung", "signal" }}, "message-text": " can't stand samsung the signal is OMG:(", "countA": 72, "countB": 31 }
+{ "tweetid": 73i64, "user": { "screen-name": "YorkSanborn_951", "lang": "en", "friends-count": 69, "statuses-count": 375, "name": "York Sanborn", "followers-count": 15 }, "sender-location": point("43.92,94.49"), "send-time": datetime("2010-09-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " dislike motorola its reachability is bad:(", "countA": 73, "countB": 61 }
+{ "tweetid": 74i64, "user": { "screen-name": "AlbertoDull$598", "lang": "en", "friends-count": 29, "statuses-count": 181, "name": "Alberto Dull", "followers-count": 192 }, "sender-location": point("25.6,85.23"), "send-time": datetime("2005-09-22T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " dislike samsung the wireless is OMG:(", "countA": 74, "countB": 32 }
+{ "tweetid": 75i64, "user": { "screen-name": "EnriqueFaast$123", "lang": "en", "friends-count": 9, "statuses-count": 24, "name": "Enrique Faast", "followers-count": 24 }, "sender-location": point("30.09,72.93"), "send-time": datetime("2009-10-17T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voicemail-service" }}, "message-text": " dislike t-mobile its voicemail-service is horrible", "countA": 75, "countB": 185 }
+{ "tweetid": 76i64, "user": { "screen-name": "AndreaBruxner$43", "lang": "en", "friends-count": 37, "statuses-count": 279, "name": "Andrea Bruxner", "followers-count": 118 }, "sender-location": point("30.39,92.92"), "send-time": datetime("2011-04-18T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " like t-mobile its customization is awesome", "countA": 76, "countB": 146 }
+{ "tweetid": 77i64, "user": { "screen-name": "LashawnaKemble$318", "lang": "en", "friends-count": 53, "statuses-count": 44, "name": "Lashawna Kemble", "followers-count": 102 }, "sender-location": point("46.29,93.16"), "send-time": datetime("2010-12-09T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " dislike samsung the voice-command is horrible", "countA": 77, "countB": 0 }
+{ "tweetid": 78i64, "user": { "screen-name": "RodolfoWoodworth#419", "lang": "en", "friends-count": 2, "statuses-count": 82, "name": "Rodolfo Woodworth", "followers-count": 16 }, "sender-location": point("44.92,70.03"), "send-time": datetime("2008-12-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " like motorola the touch-screen is awesome", "countA": 78, "countB": 140 }
+{ "tweetid": 79i64, "user": { "screen-name": "AbramCourtney_384", "lang": "en", "friends-count": 10, "statuses-count": 33, "name": "Abram Courtney", "followers-count": 138 }, "sender-location": point("34.9,96.91"), "send-time": datetime("2007-03-15T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " hate at&t the plan is bad:(", "countA": 79, "countB": 193 }
+{ "tweetid": 80i64, "user": { "screen-name": "LaurindaRosensteel@202", "lang": "en", "friends-count": 19, "statuses-count": 222, "name": "Laurinda Rosensteel", "followers-count": 20 }, "sender-location": point("47.86,92.66"), "send-time": datetime("2008-09-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " hate samsung its platform is horrible:(", "countA": 80, "countB": 39 }
+{ "tweetid": 81i64, "user": { "screen-name": "JarrettBratton@573", "lang": "en", "friends-count": 90, "statuses-count": 287, "name": "Jarrett Bratton", "followers-count": 18 }, "sender-location": point("33.85,75.61"), "send-time": datetime("2006-05-06T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " like sprint its network is amazing:)", "countA": 81, "countB": 105 }
+{ "tweetid": 82i64, "user": { "screen-name": "EleanorBicknell$880", "lang": "en", "friends-count": 35, "statuses-count": 444, "name": "Eleanor Bicknell", "followers-count": 199 }, "sender-location": point("42.31,95.69"), "send-time": datetime("2008-04-19T10:10:00.000Z"), "referred-topics": {{ "samsung", "plan" }}, "message-text": " can't stand samsung its plan is terrible", "countA": 82, "countB": 140 }
+{ "tweetid": 83i64, "user": { "screen-name": "MannyWerner#209", "lang": "en", "friends-count": 85, "statuses-count": 476, "name": "Manny Werner", "followers-count": 78 }, "sender-location": point("35.76,80.06"), "send-time": datetime("2007-11-17T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " dislike samsung the platform is terrible", "countA": 83, "countB": 163 }
+{ "tweetid": 84i64, "user": { "screen-name": "TeshaReade_713", "lang": "en", "friends-count": 74, "statuses-count": 99, "name": "Tesha Reade", "followers-count": 112 }, "sender-location": point("26.69,82.05"), "send-time": datetime("2007-01-04T10:10:00.000Z"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon the shortcut-menu is mind-blowing", "countA": 84, "countB": 172 }
+{ "tweetid": 85i64, "user": { "screen-name": "WanCrissman$283", "lang": "en", "friends-count": 37, "statuses-count": 384, "name": "Wan Crissman", "followers-count": 176 }, "sender-location": point("42.3,69.18"), "send-time": datetime("2006-10-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " love at&t its network is awesome:)", "countA": 85, "countB": 77 }
+{ "tweetid": 86i64, "user": { "screen-name": "DeedeeMccallum#158", "lang": "en", "friends-count": 81, "statuses-count": 370, "name": "Deedee Mccallum", "followers-count": 104 }, "sender-location": point("46.35,92.36"), "send-time": datetime("2011-09-22T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-command" }}, "message-text": " like verizon the voice-command is amazing:)", "countA": 86, "countB": 156 }
+{ "tweetid": 87i64, "user": { "screen-name": "AileenAft@340", "lang": "en", "friends-count": 87, "statuses-count": 476, "name": "Aileen Aft", "followers-count": 14 }, "sender-location": point("41.44,95.97"), "send-time": datetime("2012-07-16T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " love motorola its network is mind-blowing", "countA": 87, "countB": 155 }
+{ "tweetid": 88i64, "user": { "screen-name": "BurtonLinton_390", "lang": "en", "friends-count": 13, "statuses-count": 462, "name": "Burton Linton", "followers-count": 34 }, "sender-location": point("45.22,88.29"), "send-time": datetime("2011-12-15T10:10:00.000Z"), "referred-topics": {{ "iphone", "reachability" }}, "message-text": " love iphone the reachability is awesome", "countA": 88, "countB": 95 }
+{ "tweetid": 89i64, "user": { "screen-name": "DamionJoghs_943", "lang": "en", "friends-count": 18, "statuses-count": 388, "name": "Damion Joghs", "followers-count": 111 }, "sender-location": point("36.32,83.38"), "send-time": datetime("2011-06-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "touch-screen" }}, "message-text": " love motorola its touch-screen is awesome", "countA": 89, "countB": 85 }
+{ "tweetid": 90i64, "user": { "screen-name": "LatoshaCowart_858", "lang": "en", "friends-count": 14, "statuses-count": 318, "name": "Latosha Cowart", "followers-count": 27 }, "sender-location": point("26.63,82.77"), "send-time": datetime("2011-10-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " love motorola the plan is good:)", "countA": 90, "countB": 139 }
+{ "tweetid": 91i64, "user": { "screen-name": "LoganPowers$336", "lang": "en", "friends-count": 52, "statuses-count": 154, "name": "Logan Powers", "followers-count": 28 }, "sender-location": point("30.66,96.22"), "send-time": datetime("2011-02-25T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " hate verizon the reachability is OMG:(", "countA": 91, "countB": 198 }
+{ "tweetid": 92i64, "user": { "screen-name": "NeilParkinson#794", "lang": "en", "friends-count": 18, "statuses-count": 365, "name": "Neil Parkinson", "followers-count": 27 }, "sender-location": point("31.25,71.75"), "send-time": datetime("2009-12-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t the platform is terrible:(", "countA": 92, "countB": 59 }
+{ "tweetid": 93i64, "user": { "screen-name": "GoddardFiscina$655", "lang": "en", "friends-count": 10, "statuses-count": 388, "name": "Goddard Fiscina", "followers-count": 142 }, "sender-location": point("33.4,72.55"), "send-time": datetime("2009-04-15T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " like samsung the speed is mind-blowing", "countA": 93, "countB": 55 }
+{ "tweetid": 94i64, "user": { "screen-name": "JacindaCressman_698", "lang": "en", "friends-count": 50, "statuses-count": 380, "name": "Jacinda Cressman", "followers-count": 112 }, "sender-location": point("33.68,85.33"), "send-time": datetime("2010-09-07T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " like sprint its network is amazing:)", "countA": 94, "countB": 128 }
+{ "tweetid": 95i64, "user": { "screen-name": "NelsonWilks_476", "lang": "en", "friends-count": 43, "statuses-count": 249, "name": "Nelson Wilks", "followers-count": 47 }, "sender-location": point("26.2,74.63"), "send-time": datetime("2010-10-28T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " love iphone the shortcut-menu is mind-blowing", "countA": 95, "countB": 144 }
+{ "tweetid": 96i64, "user": { "screen-name": "FelipeBeach_761", "lang": "en", "friends-count": 70, "statuses-count": 191, "name": "Felipe Beach", "followers-count": 56 }, "sender-location": point("38.59,75.94"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "sprint", "customization" }}, "message-text": " like sprint the customization is awesome:)", "countA": 96, "countB": 66 }
+{ "tweetid": 97i64, "user": { "screen-name": "MaximaPoehl$770", "lang": "en", "friends-count": 9, "statuses-count": 99, "name": "Maxima Poehl", "followers-count": 198 }, "sender-location": point("46.94,66.2"), "send-time": datetime("2008-03-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " like iphone the customization is good", "countA": 97, "countB": 83 }
+{ "tweetid": 98i64, "user": { "screen-name": "IraLombardi#278", "lang": "en", "friends-count": 10, "statuses-count": 282, "name": "Ira Lombardi", "followers-count": 26 }, "sender-location": point("44.99,93.61"), "send-time": datetime("2011-02-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " love iphone its platform is awesome:)", "countA": 98, "countB": 78 }
+{ "tweetid": 99i64, "user": { "screen-name": "RexHincken_917", "lang": "en", "friends-count": 88, "statuses-count": 292, "name": "Rex Hincken", "followers-count": 74 }, "sender-location": point("42.0,81.22"), "send-time": datetime("2008-09-01T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voicemail-service" }}, "message-text": " love t-mobile the voicemail-service is amazing:)", "countA": 99, "countB": 51 }
+{ "tweetid": 100i64, "user": { "screen-name": "DakotaTeagarden_163", "lang": "en", "friends-count": 54, "statuses-count": 391, "name": "Dakota Teagarden", "followers-count": 160 }, "sender-location": point("43.59,92.49"), "send-time": datetime("2010-11-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "shortcut-menu" }}, "message-text": " can't stand samsung its shortcut-menu is OMG:(", "countA": 100, "countB": 184 }
+{ "tweetid": 101i64, "user": { "screen-name": "ChetMilliron_934", "lang": "en", "friends-count": 62, "statuses-count": 453, "name": "Chet Milliron", "followers-count": 53 }, "sender-location": point("47.95,77.58"), "send-time": datetime("2012-08-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " like samsung the 3G is mind-blowing:)", "countA": 101, "countB": 142 }
+{ "tweetid": 102i64, "user": { "screen-name": "ZackLosey_956", "lang": "en", "friends-count": 90, "statuses-count": 6, "name": "Zack Losey", "followers-count": 116 }, "sender-location": point("26.53,80.6"), "send-time": datetime("2005-01-11T10:10:00.000Z"), "referred-topics": {{ "verizon", "platform" }}, "message-text": " like verizon its platform is amazing", "countA": 102, "countB": 190 }
+{ "tweetid": 103i64, "user": { "screen-name": "BrionyLafortune$483", "lang": "en", "friends-count": 87, "statuses-count": 496, "name": "Briony Lafortune", "followers-count": 4 }, "sender-location": point("42.2,73.96"), "send-time": datetime("2010-12-05T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " can't stand samsung its voice-command is OMG", "countA": 103, "countB": 27 }
+{ "tweetid": 104i64, "user": { "screen-name": "SaraGraham@726", "lang": "en", "friends-count": 38, "statuses-count": 398, "name": "Sara Graham", "followers-count": 68 }, "sender-location": point("32.83,81.29"), "send-time": datetime("2009-10-25T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " like iphone its platform is awesome", "countA": 104, "countB": 55 }
+{ "tweetid": 105i64, "user": { "screen-name": "EvanBarnes_217", "lang": "en", "friends-count": 42, "statuses-count": 239, "name": "Evan Barnes", "followers-count": 108 }, "sender-location": point("44.7,90.98"), "send-time": datetime("2008-07-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " like sprint its reachability is mind-blowing", "countA": 105, "countB": 3 }
+{ "tweetid": 106i64, "user": { "screen-name": "JulianeNorthey#34", "lang": "en", "friends-count": 69, "statuses-count": 94, "name": "Juliane Northey", "followers-count": 187 }, "sender-location": point("26.32,67.64"), "send-time": datetime("2007-06-26T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " can't stand sprint the voice-clarity is terrible", "countA": 106, "countB": 127 }
+{ "tweetid": 107i64, "user": { "screen-name": "ShannahBailey$196", "lang": "en", "friends-count": 47, "statuses-count": 215, "name": "Shannah Bailey", "followers-count": 67 }, "sender-location": point("40.85,87.01"), "send-time": datetime("2009-10-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " love verizon the reachability is awesome:)", "countA": 107, "countB": 31 }
+{ "tweetid": 108i64, "user": { "screen-name": "GranvilleKnisely$497", "lang": "en", "friends-count": 57, "statuses-count": 117, "name": "Granville Knisely", "followers-count": 52 }, "sender-location": point("35.46,78.27"), "send-time": datetime("2006-06-06T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " hate samsung its customization is horrible", "countA": 108, "countB": 148 }
+{ "tweetid": 109i64, "user": { "screen-name": "LeonardoJardine@763", "lang": "en", "friends-count": 48, "statuses-count": 415, "name": "Leonardo Jardine", "followers-count": 96 }, "sender-location": point("27.7,92.32"), "send-time": datetime("2010-12-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " dislike verizon the signal is bad", "countA": 109, "countB": 29 }
+{ "tweetid": 110i64, "user": { "screen-name": "AuroraMcelroy@927", "lang": "en", "friends-count": 79, "statuses-count": 297, "name": "Aurora Mcelroy", "followers-count": 119 }, "sender-location": point("48.56,85.12"), "send-time": datetime("2005-06-14T10:10:00.000Z"), "referred-topics": {{ "motorola", "shortcut-menu" }}, "message-text": " dislike motorola the shortcut-menu is bad:(", "countA": 110, "countB": 157 }
+{ "tweetid": 111i64, "user": { "screen-name": "NoleneLeslie#166", "lang": "en", "friends-count": 30, "statuses-count": 3, "name": "Nolene Leslie", "followers-count": 18 }, "sender-location": point("31.07,78.53"), "send-time": datetime("2005-10-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "platform" }}, "message-text": " like t-mobile its platform is good", "countA": 111, "countB": 50 }
+{ "tweetid": 112i64, "user": { "screen-name": "EusebioBeedell@329", "lang": "en", "friends-count": 94, "statuses-count": 341, "name": "Eusebio Beedell", "followers-count": 89 }, "sender-location": point("32.75,68.79"), "send-time": datetime("2007-08-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " can't stand verizon its network is terrible:(", "countA": 112, "countB": 30 }
+{ "tweetid": 113i64, "user": { "screen-name": "WoodySaltser$873", "lang": "en", "friends-count": 68, "statuses-count": 365, "name": "Woody Saltser", "followers-count": 132 }, "sender-location": point("37.57,88.05"), "send-time": datetime("2012-01-21T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "voice-clarity" }}, "message-text": " love t-mobile its voice-clarity is awesome", "countA": 113, "countB": 177 }
+{ "tweetid": 114i64, "user": { "screen-name": "ReannaSeelig#553", "lang": "en", "friends-count": 31, "statuses-count": 291, "name": "Reanna Seelig", "followers-count": 175 }, "sender-location": point("42.87,72.38"), "send-time": datetime("2006-03-20T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " dislike motorola the signal is OMG:(", "countA": 114, "countB": 59 }
+{ "tweetid": 115i64, "user": { "screen-name": "AllannahNapier@336", "lang": "en", "friends-count": 34, "statuses-count": 359, "name": "Allannah Napier", "followers-count": 50 }, "sender-location": point("31.29,88.73"), "send-time": datetime("2010-07-28T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " hate samsung the wireless is bad:(", "countA": 115, "countB": 175 }
+{ "tweetid": 116i64, "user": { "screen-name": "AlaynaOsteen_327", "lang": "en", "friends-count": 59, "statuses-count": 237, "name": "Alayna Osteen", "followers-count": 12 }, "sender-location": point("30.6,71.97"), "send-time": datetime("2007-07-25T10:10:00.000Z"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is amazing:)", "countA": 116, "countB": 70 }
+{ "tweetid": 117i64, "user": { "screen-name": "LeticiaMillard#139", "lang": "en", "friends-count": 95, "statuses-count": 46, "name": "Leticia Millard", "followers-count": 72 }, "sender-location": point("26.53,73.37"), "send-time": datetime("2005-06-22T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " dislike iphone its 3G is horrible", "countA": 117, "countB": 168 }
+{ "tweetid": 118i64, "user": { "screen-name": "WinifredMckee_639", "lang": "en", "friends-count": 48, "statuses-count": 442, "name": "Winifred Mckee", "followers-count": 199 }, "sender-location": point("27.51,76.65"), "send-time": datetime("2012-06-19T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " can't stand sprint the reachability is bad", "countA": 118, "countB": 70 }
+{ "tweetid": 119i64, "user": { "screen-name": "SungShea#585", "lang": "en", "friends-count": 38, "statuses-count": 193, "name": "Sung Shea", "followers-count": 149 }, "sender-location": point("28.86,83.73"), "send-time": datetime("2009-04-22T10:10:00.000Z"), "referred-topics": {{ "samsung", "customer-service" }}, "message-text": " can't stand samsung its customer-service is horrible:(", "countA": 119, "countB": 155 }
+{ "tweetid": 120i64, "user": { "screen-name": "BernadineSutton@199", "lang": "en", "friends-count": 72, "statuses-count": 46, "name": "Bernadine Sutton", "followers-count": 105 }, "sender-location": point("40.19,77.94"), "send-time": datetime("2007-11-12T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " like sprint the voice-clarity is good:)", "countA": 120, "countB": 63 }
+{ "tweetid": 121i64, "user": { "screen-name": "DeedeeJerome#182", "lang": "en", "friends-count": 74, "statuses-count": 342, "name": "Deedee Jerome", "followers-count": 170 }, "sender-location": point("42.66,73.84"), "send-time": datetime("2012-07-03T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " can't stand sprint its reachability is horrible", "countA": 121, "countB": 163 }
+{ "tweetid": 122i64, "user": { "screen-name": "NigelPrechtl$759", "lang": "en", "friends-count": 10, "statuses-count": 133, "name": "Nigel Prechtl", "followers-count": 137 }, "sender-location": point("37.22,80.92"), "send-time": datetime("2012-01-05T10:10:00.000Z"), "referred-topics": {{ "at&t", "speed" }}, "message-text": " dislike at&t its speed is terrible", "countA": 122, "countB": 23 }
+{ "tweetid": 123i64, "user": { "screen-name": "KimmyWynne$198", "lang": "en", "friends-count": 16, "statuses-count": 40, "name": "Kimmy Wynne", "followers-count": 6 }, "sender-location": point("26.49,70.55"), "send-time": datetime("2008-10-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " dislike t-mobile the signal is terrible", "countA": 123, "countB": 44 }
+{ "tweetid": 124i64, "user": { "screen-name": "ByronHarshman$352", "lang": "en", "friends-count": 26, "statuses-count": 133, "name": "Byron Harshman", "followers-count": 144 }, "sender-location": point("26.4,88.43"), "send-time": datetime("2012-03-21T10:10:00.000Z"), "referred-topics": {{ "samsung", "touch-screen" }}, "message-text": " dislike samsung the touch-screen is terrible", "countA": 124, "countB": 44 }
+{ "tweetid": 125i64, "user": { "screen-name": "PlacidPrevatt#865", "lang": "en", "friends-count": 3, "statuses-count": 493, "name": "Placid Prevatt", "followers-count": 10 }, "sender-location": point("43.09,84.0"), "send-time": datetime("2010-07-07T10:10:00.000Z"), "referred-topics": {{ "motorola", "voicemail-service" }}, "message-text": " love motorola its voicemail-service is amazing:)", "countA": 125, "countB": 7 }
+{ "tweetid": 126i64, "user": { "screen-name": "TranterGarneis_456", "lang": "en", "friends-count": 89, "statuses-count": 151, "name": "Tranter Garneis", "followers-count": 166 }, "sender-location": point("41.6,93.6"), "send-time": datetime("2007-08-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone its shortcut-menu is amazing:)", "countA": 126, "countB": 76 }
+{ "tweetid": 127i64, "user": { "screen-name": "DonyaWilliamson$23", "lang": "en", "friends-count": 62, "statuses-count": 325, "name": "Donya Williamson", "followers-count": 101 }, "sender-location": point("27.75,66.01"), "send-time": datetime("2005-11-02T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " can't stand verizon its reachability is OMG", "countA": 127, "countB": 184 }
+{ "tweetid": 128i64, "user": { "screen-name": "GalinaJoghs$90", "lang": "en", "friends-count": 61, "statuses-count": 86, "name": "Galina Joghs", "followers-count": 169 }, "sender-location": point("30.95,71.04"), "send-time": datetime("2010-06-01T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " can't stand motorola its signal is horrible", "countA": 128, "countB": 24 }
+{ "tweetid": 129i64, "user": { "screen-name": "SamsonWerner#683", "lang": "en", "friends-count": 92, "statuses-count": 171, "name": "Samson Werner", "followers-count": 108 }, "sender-location": point("36.53,92.04"), "send-time": datetime("2009-08-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " hate motorola its network is terrible:(", "countA": 129, "countB": 80 }
+{ "tweetid": 130i64, "user": { "screen-name": "GabrielleMang#424", "lang": "en", "friends-count": 66, "statuses-count": 8, "name": "Gabrielle Mang", "followers-count": 80 }, "sender-location": point("36.74,96.64"), "send-time": datetime("2006-04-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " love sprint its plan is amazing:)", "countA": 130, "countB": 157 }
+{ "tweetid": 131i64, "user": { "screen-name": "ZachariasBaldwin#74", "lang": "en", "friends-count": 5, "statuses-count": 205, "name": "Zacharias Baldwin", "followers-count": 87 }, "sender-location": point("25.44,72.7"), "send-time": datetime("2006-01-21T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " like sprint the platform is amazing", "countA": 131, "countB": 192 }
+{ "tweetid": 132i64, "user": { "screen-name": "FanniePoorbaugh@315", "lang": "en", "friends-count": 59, "statuses-count": 441, "name": "Fannie Poorbaugh", "followers-count": 114 }, "sender-location": point("48.89,68.65"), "send-time": datetime("2005-05-07T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " hate verizon the reachability is terrible", "countA": 132, "countB": 142 }
+{ "tweetid": 133i64, "user": { "screen-name": "SandraTeagarden$747", "lang": "en", "friends-count": 60, "statuses-count": 353, "name": "Sandra Teagarden", "followers-count": 141 }, "sender-location": point("44.37,76.54"), "send-time": datetime("2008-05-25T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " can't stand at&t its platform is bad", "countA": 133, "countB": 17 }
+{ "tweetid": 134i64, "user": { "screen-name": "SteveMayers_702", "lang": "en", "friends-count": 70, "statuses-count": 196, "name": "Steve Mayers", "followers-count": 22 }, "sender-location": point("41.52,91.39"), "send-time": datetime("2012-08-11T10:10:00.000Z"), "referred-topics": {{ "iphone", "speed" }}, "message-text": " love iphone its speed is amazing", "countA": 134, "countB": 86 }
+{ "tweetid": 135i64, "user": { "screen-name": "RosalynPullman@789", "lang": "en", "friends-count": 17, "statuses-count": 470, "name": "Rosalyn Pullman", "followers-count": 123 }, "sender-location": point("48.33,86.41"), "send-time": datetime("2009-12-08T10:10:00.000Z"), "referred-topics": {{ "motorola", "customization" }}, "message-text": " hate motorola its customization is bad:(", "countA": 135, "countB": 171 }
+{ "tweetid": 136i64, "user": { "screen-name": "LamarChauvin$832", "lang": "en", "friends-count": 21, "statuses-count": 234, "name": "Lamar Chauvin", "followers-count": 184 }, "sender-location": point("36.83,89.48"), "send-time": datetime("2011-06-26T10:10:00.000Z"), "referred-topics": {{ "samsung", "reachability" }}, "message-text": " love samsung its reachability is awesome:)", "countA": 136, "countB": 77 }
+{ "tweetid": 137i64, "user": { "screen-name": "EleaseReade#477", "lang": "en", "friends-count": 24, "statuses-count": 299, "name": "Elease Reade", "followers-count": 24 }, "sender-location": point("45.55,93.09"), "send-time": datetime("2012-07-19T10:10:00.000Z"), "referred-topics": {{ "at&t", "network" }}, "message-text": " hate at&t the network is bad:(", "countA": 137, "countB": 25 }
+{ "tweetid": 138i64, "user": { "screen-name": "LeviPhilbrick$328", "lang": "en", "friends-count": 73, "statuses-count": 77, "name": "Levi Philbrick", "followers-count": 179 }, "sender-location": point("33.51,68.88"), "send-time": datetime("2009-08-28T10:10:00.000Z"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " dislike verizon its shortcut-menu is horrible", "countA": 138, "countB": 5 }
+{ "tweetid": 139i64, "user": { "screen-name": "RaeburnNickolson_295", "lang": "en", "friends-count": 72, "statuses-count": 176, "name": "Raeburn Nickolson", "followers-count": 103 }, "sender-location": point("38.42,74.16"), "send-time": datetime("2008-05-28T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " like at&t its voicemail-service is good:)", "countA": 139, "countB": 175 }
+{ "tweetid": 140i64, "user": { "screen-name": "NerissaBallou@177", "lang": "en", "friends-count": 17, "statuses-count": 447, "name": "Nerissa Ballou", "followers-count": 183 }, "sender-location": point("34.65,81.44"), "send-time": datetime("2008-07-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "plan" }}, "message-text": " hate at&t its plan is OMG", "countA": 140, "countB": 192 }
+{ "tweetid": 141i64, "user": { "screen-name": "DanyelWalker#602", "lang": "en", "friends-count": 22, "statuses-count": 397, "name": "Danyel Walker", "followers-count": 154 }, "sender-location": point("40.38,78.39"), "send-time": datetime("2012-01-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "3G" }}, "message-text": " hate t-mobile its 3G is OMG", "countA": 141, "countB": 9 }
+{ "tweetid": 142i64, "user": { "screen-name": "YaronLeichter_45", "lang": "en", "friends-count": 80, "statuses-count": 103, "name": "Yaron Leichter", "followers-count": 115 }, "sender-location": point("42.59,71.72"), "send-time": datetime("2010-04-24T10:10:00.000Z"), "referred-topics": {{ "verizon", "signal" }}, "message-text": " like verizon the signal is mind-blowing:)", "countA": 142, "countB": 130 }
+{ "tweetid": 143i64, "user": { "screen-name": "DeemerCable_599", "lang": "en", "friends-count": 26, "statuses-count": 371, "name": "Deemer Cable", "followers-count": 34 }, "sender-location": point("29.89,70.29"), "send-time": datetime("2012-08-18T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " like motorola the wireless is awesome", "countA": 143, "countB": 50 }
+{ "tweetid": 144i64, "user": { "screen-name": "AliaHay_860", "lang": "en", "friends-count": 34, "statuses-count": 470, "name": "Alia Hay", "followers-count": 111 }, "sender-location": point("46.39,96.22"), "send-time": datetime("2010-12-18T10:10:00.000Z"), "referred-topics": {{ "at&t", "shortcut-menu" }}, "message-text": " like at&t its shortcut-menu is amazing:)", "countA": 144, "countB": 184 }
+{ "tweetid": 145i64, "user": { "screen-name": "AdamMoore$384", "lang": "en", "friends-count": 42, "statuses-count": 198, "name": "Adam Moore", "followers-count": 17 }, "sender-location": point("27.99,84.55"), "send-time": datetime("2007-03-16T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " love sprint its network is good", "countA": 145, "countB": 162 }
+{ "tweetid": 146i64, "user": { "screen-name": "AileenSouthern$868", "lang": "en", "friends-count": 62, "statuses-count": 388, "name": "Aileen Southern", "followers-count": 131 }, "sender-location": point("30.85,66.62"), "send-time": datetime("2005-07-14T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " like t-mobile the signal is amazing", "countA": 146, "countB": 102 }
+{ "tweetid": 147i64, "user": { "screen-name": "JewellWise_154", "lang": "en", "friends-count": 2, "statuses-count": 279, "name": "Jewell Wise", "followers-count": 107 }, "sender-location": point("46.72,83.98"), "send-time": datetime("2006-01-09T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " hate motorola the plan is terrible:(", "countA": 147, "countB": 174 }
+{ "tweetid": 148i64, "user": { "screen-name": "DanielJowers#519", "lang": "en", "friends-count": 23, "statuses-count": 22, "name": "Daniel Jowers", "followers-count": 131 }, "sender-location": point("34.26,72.22"), "send-time": datetime("2008-07-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-clarity" }}, "message-text": " like motorola its voice-clarity is mind-blowing:)", "countA": 148, "countB": 6 }
+{ "tweetid": 149i64, "user": { "screen-name": "DillonWilliams_557", "lang": "en", "friends-count": 18, "statuses-count": 136, "name": "Dillon Williams", "followers-count": 35 }, "sender-location": point("46.63,97.38"), "send-time": datetime("2011-05-09T10:10:00.000Z"), "referred-topics": {{ "motorola", "network" }}, "message-text": " love motorola the network is good", "countA": 149, "countB": 20 }
+{ "tweetid": 150i64, "user": { "screen-name": "DerrickBullard$202", "lang": "en", "friends-count": 16, "statuses-count": 9, "name": "Derrick Bullard", "followers-count": 100 }, "sender-location": point("41.89,90.62"), "send-time": datetime("2012-08-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " like t-mobile its touch-screen is mind-blowing", "countA": 150, "countB": 145 }
+{ "tweetid": 151i64, "user": { "screen-name": "LuigiMcfall_976", "lang": "en", "friends-count": 31, "statuses-count": 215, "name": "Luigi Mcfall", "followers-count": 79 }, "sender-location": point("45.38,70.52"), "send-time": datetime("2005-11-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " hate motorola its reachability is OMG:(", "countA": 151, "countB": 43 }
+{ "tweetid": 152i64, "user": { "screen-name": "MartinPinney_858", "lang": "en", "friends-count": 21, "statuses-count": 465, "name": "Martin Pinney", "followers-count": 16 }, "sender-location": point("32.87,75.66"), "send-time": datetime("2007-10-12T10:10:00.000Z"), "referred-topics": {{ "at&t", "reachability" }}, "message-text": " hate at&t its reachability is terrible", "countA": 152, "countB": 56 }
+{ "tweetid": 153i64, "user": { "screen-name": "JackieAft_623", "lang": "en", "friends-count": 70, "statuses-count": 413, "name": "Jackie Aft", "followers-count": 138 }, "sender-location": point("29.9,73.29"), "send-time": datetime("2010-07-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " like verizon the voice-clarity is amazing", "countA": 153, "countB": 154 }
+{ "tweetid": 154i64, "user": { "screen-name": "SherriWickes#118", "lang": "en", "friends-count": 20, "statuses-count": 31, "name": "Sherri Wickes", "followers-count": 59 }, "sender-location": point("39.2,79.2"), "send-time": datetime("2006-07-13T10:10:00.000Z"), "referred-topics": {{ "sprint", "network" }}, "message-text": " hate sprint the network is OMG:(", "countA": 154, "countB": 124 }
+{ "tweetid": 155i64, "user": { "screen-name": "CarlieCowher@103", "lang": "en", "friends-count": 81, "statuses-count": 127, "name": "Carlie Cowher", "followers-count": 184 }, "sender-location": point("30.3,76.43"), "send-time": datetime("2010-10-04T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-clarity" }}, "message-text": " can't stand sprint the voice-clarity is bad", "countA": 155, "countB": 39 }
+{ "tweetid": 156i64, "user": { "screen-name": "AndraWardle@74", "lang": "en", "friends-count": 41, "statuses-count": 35, "name": "Andra Wardle", "followers-count": 168 }, "sender-location": point("45.49,93.97"), "send-time": datetime("2009-02-18T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " love t-mobile its wireless is amazing:)", "countA": 156, "countB": 23 }
+{ "tweetid": 157i64, "user": { "screen-name": "KanishaPinney@150", "lang": "en", "friends-count": 89, "statuses-count": 315, "name": "Kanisha Pinney", "followers-count": 173 }, "sender-location": point("24.72,77.36"), "send-time": datetime("2005-06-10T10:10:00.000Z"), "referred-topics": {{ "verizon", "touch-screen" }}, "message-text": " like verizon the touch-screen is amazing", "countA": 157, "countB": 153 }
+{ "tweetid": 158i64, "user": { "screen-name": "GlyndaSchere@104", "lang": "en", "friends-count": 6, "statuses-count": 111, "name": "Glynda Schere", "followers-count": 120 }, "sender-location": point("33.86,67.49"), "send-time": datetime("2010-11-26T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " like verizon its reachability is amazing:)", "countA": 158, "countB": 96 }
+{ "tweetid": 159i64, "user": { "screen-name": "JenelleNehling@461", "lang": "en", "friends-count": 4, "statuses-count": 384, "name": "Jenelle Nehling", "followers-count": 57 }, "sender-location": point("32.65,89.38"), "send-time": datetime("2010-12-06T10:10:00.000Z"), "referred-topics": {{ "iphone", "shortcut-menu" }}, "message-text": " like iphone its shortcut-menu is good", "countA": 159, "countB": 113 }
+{ "tweetid": 160i64, "user": { "screen-name": "DelWheeler@286", "lang": "en", "friends-count": 23, "statuses-count": 403, "name": "Del Wheeler", "followers-count": 177 }, "sender-location": point("39.39,78.05"), "send-time": datetime("2011-07-23T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " love samsung the platform is mind-blowing", "countA": 160, "countB": 163 }
+{ "tweetid": 161i64, "user": { "screen-name": "TrinityCowart@360", "lang": "en", "friends-count": 85, "statuses-count": 204, "name": "Trinity Cowart", "followers-count": 145 }, "sender-location": point("32.11,76.64"), "send-time": datetime("2010-10-03T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "wireless" }}, "message-text": " like t-mobile its wireless is amazing:)", "countA": 161, "countB": 62 }
+{ "tweetid": 162i64, "user": { "screen-name": "HudsonBasmanoff_348", "lang": "en", "friends-count": 3, "statuses-count": 394, "name": "Hudson Basmanoff", "followers-count": 114 }, "sender-location": point("39.24,78.13"), "send-time": datetime("2007-09-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " can't stand motorola its wireless is OMG:(", "countA": 162, "countB": 57 }
+{ "tweetid": 163i64, "user": { "screen-name": "MatthewPowers_801", "lang": "en", "friends-count": 2, "statuses-count": 203, "name": "Matthew Powers", "followers-count": 199 }, "sender-location": point("33.79,69.57"), "send-time": datetime("2012-04-05T10:10:00.000Z"), "referred-topics": {{ "at&t", "customer-service" }}, "message-text": " love at&t its customer-service is awesome", "countA": 163, "countB": 115 }
+{ "tweetid": 164i64, "user": { "screen-name": "TitaniaKern$100", "lang": "en", "friends-count": 98, "statuses-count": 300, "name": "Titania Kern", "followers-count": 118 }, "sender-location": point("45.86,67.64"), "send-time": datetime("2005-11-04T10:10:00.000Z"), "referred-topics": {{ "at&t", "shortcut-menu" }}, "message-text": " love at&t the shortcut-menu is amazing", "countA": 164, "countB": 148 }
+{ "tweetid": 165i64, "user": { "screen-name": "EhtelCrissman#778", "lang": "en", "friends-count": 33, "statuses-count": 286, "name": "Ehtel Crissman", "followers-count": 63 }, "sender-location": point("32.98,82.49"), "send-time": datetime("2011-08-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is good:)", "countA": 165, "countB": 127 }
+{ "tweetid": 166i64, "user": { "screen-name": "WilletteLeslie@682", "lang": "en", "friends-count": 38, "statuses-count": 491, "name": "Willette Leslie", "followers-count": 75 }, "sender-location": point("32.57,84.97"), "send-time": datetime("2012-04-18T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " like verizon its reachability is mind-blowing", "countA": 166, "countB": 162 }
+{ "tweetid": 167i64, "user": { "screen-name": "DarellHincken_722", "lang": "en", "friends-count": 33, "statuses-count": 111, "name": "Darell Hincken", "followers-count": 74 }, "sender-location": point("48.57,77.77"), "send-time": datetime("2008-06-27T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "speed" }}, "message-text": " love t-mobile its speed is good:)", "countA": 167, "countB": 93 }
+{ "tweetid": 168i64, "user": { "screen-name": "DuaneKing@956", "lang": "en", "friends-count": 5, "statuses-count": 44, "name": "Duane King", "followers-count": 169 }, "sender-location": point("31.26,68.61"), "send-time": datetime("2011-03-24T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " like sprint its touch-screen is mind-blowing:)", "countA": 168, "countB": 174 }
+{ "tweetid": 169i64, "user": { "screen-name": "AmbroseKeilbach$300", "lang": "en", "friends-count": 76, "statuses-count": 278, "name": "Ambrose Keilbach", "followers-count": 54 }, "sender-location": point("29.75,71.35"), "send-time": datetime("2012-02-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "signal" }}, "message-text": " can't stand sprint its signal is horrible", "countA": 169, "countB": 129 }
+{ "tweetid": 170i64, "user": { "screen-name": "KarlBrooks#97", "lang": "en", "friends-count": 76, "statuses-count": 150, "name": "Karl Brooks", "followers-count": 117 }, "sender-location": point("30.77,85.78"), "send-time": datetime("2006-09-16T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " hate motorola the plan is horrible", "countA": 170, "countB": 174 }
+{ "tweetid": 171i64, "user": { "screen-name": "ShainaMayers$261", "lang": "en", "friends-count": 76, "statuses-count": 240, "name": "Shaina Mayers", "followers-count": 194 }, "sender-location": point("26.11,78.33"), "send-time": datetime("2005-06-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "voicemail-service" }}, "message-text": " love sprint its voicemail-service is mind-blowing:)", "countA": 171, "countB": 69 }
+{ "tweetid": 172i64, "user": { "screen-name": "LakeshaPery_35", "lang": "en", "friends-count": 58, "statuses-count": 300, "name": "Lakesha Pery", "followers-count": 51 }, "sender-location": point("38.45,75.31"), "send-time": datetime("2009-12-20T10:10:00.000Z"), "referred-topics": {{ "motorola", "3G" }}, "message-text": " like motorola its 3G is good:)", "countA": 172, "countB": 127 }
+{ "tweetid": 173i64, "user": { "screen-name": "DoranMingle#901", "lang": "en", "friends-count": 3, "statuses-count": 302, "name": "Doran Mingle", "followers-count": 152 }, "sender-location": point("47.76,91.28"), "send-time": datetime("2009-06-07T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " dislike at&t the customization is OMG", "countA": 173, "countB": 41 }
+{ "tweetid": 174i64, "user": { "screen-name": "AmadaHatcher#710", "lang": "en", "friends-count": 3, "statuses-count": 12, "name": "Amada Hatcher", "followers-count": 193 }, "sender-location": point("28.64,89.42"), "send-time": datetime("2011-09-16T10:10:00.000Z"), "referred-topics": {{ "samsung", "network" }}, "message-text": " hate samsung the network is OMG", "countA": 174, "countB": 0 }
+{ "tweetid": 175i64, "user": { "screen-name": "CorianderMoon@658", "lang": "en", "friends-count": 28, "statuses-count": 117, "name": "Coriander Moon", "followers-count": 77 }, "sender-location": point("43.82,87.23"), "send-time": datetime("2011-03-26T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " dislike samsung the customization is bad:(", "countA": 175, "countB": 5 }
+{ "tweetid": 176i64, "user": { "screen-name": "PhilomenaEiford#608", "lang": "en", "friends-count": 53, "statuses-count": 467, "name": "Philomena Eiford", "followers-count": 80 }, "sender-location": point("30.06,71.08"), "send-time": datetime("2006-05-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "signal" }}, "message-text": " like motorola the signal is amazing", "countA": 176, "countB": 89 }
+{ "tweetid": 177i64, "user": { "screen-name": "JacobCongdon$162", "lang": "en", "friends-count": 61, "statuses-count": 161, "name": "Jacob Congdon", "followers-count": 19 }, "sender-location": point("36.36,78.43"), "send-time": datetime("2006-12-10T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " dislike samsung its platform is horrible", "countA": 177, "countB": 15 }
+{ "tweetid": 178i64, "user": { "screen-name": "GeorgeRichards$777", "lang": "en", "friends-count": 12, "statuses-count": 213, "name": "George Richards", "followers-count": 72 }, "sender-location": point("44.78,90.69"), "send-time": datetime("2006-03-17T10:10:00.000Z"), "referred-topics": {{ "verizon", "speed" }}, "message-text": " can't stand verizon its speed is horrible", "countA": 178, "countB": 64 }
+{ "tweetid": 179i64, "user": { "screen-name": "BridgerHamilton@431", "lang": "en", "friends-count": 51, "statuses-count": 396, "name": "Bridger Hamilton", "followers-count": 110 }, "sender-location": point("32.82,81.54"), "send-time": datetime("2008-12-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " love verizon the plan is awesome", "countA": 179, "countB": 141 }
+{ "tweetid": 180i64, "user": { "screen-name": "FemieLucy@34", "lang": "en", "friends-count": 68, "statuses-count": 221, "name": "Femie Lucy", "followers-count": 11 }, "sender-location": point("31.58,82.78"), "send-time": datetime("2010-02-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " hate iphone its customization is OMG", "countA": 180, "countB": 191 }
+{ "tweetid": 181i64, "user": { "screen-name": "JodiNapier@338", "lang": "en", "friends-count": 96, "statuses-count": 467, "name": "Jodi Napier", "followers-count": 69 }, "sender-location": point("26.14,78.5"), "send-time": datetime("2007-01-06T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " love verizon the plan is mind-blowing", "countA": 181, "countB": 13 }
+{ "tweetid": 182i64, "user": { "screen-name": "MitsueRawls_424", "lang": "en", "friends-count": 91, "statuses-count": 70, "name": "Mitsue Rawls", "followers-count": 193 }, "sender-location": point("47.77,70.41"), "send-time": datetime("2007-04-12T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " dislike iphone its customer-service is OMG", "countA": 182, "countB": 59 }
+{ "tweetid": 183i64, "user": { "screen-name": "DeshawnAultman_690", "lang": "en", "friends-count": 49, "statuses-count": 330, "name": "Deshawn Aultman", "followers-count": 39 }, "sender-location": point("40.65,79.37"), "send-time": datetime("2005-08-11T10:10:00.000Z"), "referred-topics": {{ "sprint", "shortcut-menu" }}, "message-text": " like sprint the shortcut-menu is amazing:)", "countA": 183, "countB": 199 }
+{ "tweetid": 184i64, "user": { "screen-name": "BradfordEiford#127", "lang": "en", "friends-count": 90, "statuses-count": 425, "name": "Bradford Eiford", "followers-count": 10 }, "sender-location": point("40.85,91.17"), "send-time": datetime("2005-09-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "voicemail-service" }}, "message-text": " dislike iphone its voicemail-service is horrible:(", "countA": 184, "countB": 141 }
+{ "tweetid": 185i64, "user": { "screen-name": "MadelynGaskins_356", "lang": "en", "friends-count": 48, "statuses-count": 455, "name": "Madelyn Gaskins", "followers-count": 66 }, "sender-location": point("33.81,88.32"), "send-time": datetime("2011-10-22T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " love at&t its voicemail-service is good:)", "countA": 185, "countB": 189 }
+{ "tweetid": 186i64, "user": { "screen-name": "SophiaMang@768", "lang": "en", "friends-count": 28, "statuses-count": 86, "name": "Sophia Mang", "followers-count": 125 }, "sender-location": point("37.45,68.47"), "send-time": datetime("2010-08-16T10:10:00.000Z"), "referred-topics": {{ "iphone", "touch-screen" }}, "message-text": " love iphone its touch-screen is awesome:)", "countA": 186, "countB": 149 }
+{ "tweetid": 187i64, "user": { "screen-name": "VernonKnisely#170", "lang": "en", "friends-count": 63, "statuses-count": 406, "name": "Vernon Knisely", "followers-count": 31 }, "sender-location": point("32.93,94.65"), "send-time": datetime("2006-03-22T10:10:00.000Z"), "referred-topics": {{ "sprint", "reachability" }}, "message-text": " love sprint the reachability is awesome", "countA": 187, "countB": 98 }
+{ "tweetid": 188i64, "user": { "screen-name": "AmyEndsley@85", "lang": "en", "friends-count": 83, "statuses-count": 43, "name": "Amy Endsley", "followers-count": 7 }, "sender-location": point("27.33,82.34"), "send-time": datetime("2005-01-11T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " love at&t its customization is good", "countA": 188, "countB": 48 }
+{ "tweetid": 189i64, "user": { "screen-name": "AntonChristner#166", "lang": "en", "friends-count": 12, "statuses-count": 10, "name": "Anton Christner", "followers-count": 66 }, "sender-location": point("25.29,89.55"), "send-time": datetime("2006-11-22T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " can't stand motorola its wireless is terrible:(", "countA": 189, "countB": 160 }
+{ "tweetid": 190i64, "user": { "screen-name": "DeshawnHarris#34", "lang": "en", "friends-count": 32, "statuses-count": 488, "name": "Deshawn Harris", "followers-count": 178 }, "sender-location": point("45.46,76.04"), "send-time": datetime("2007-05-13T10:10:00.000Z"), "referred-topics": {{ "samsung", "customization" }}, "message-text": " like samsung the customization is awesome:)", "countA": 190, "countB": 86 }
+{ "tweetid": 191i64, "user": { "screen-name": "MarioHolts_870", "lang": "en", "friends-count": 20, "statuses-count": 192, "name": "Mario Holts", "followers-count": 71 }, "sender-location": point("29.69,71.42"), "send-time": datetime("2005-09-15T10:10:00.000Z"), "referred-topics": {{ "iphone", "speed" }}, "message-text": " hate iphone the speed is horrible:(", "countA": 191, "countB": 150 }
+{ "tweetid": 192i64, "user": { "screen-name": "DeanHall#220", "lang": "en", "friends-count": 55, "statuses-count": 236, "name": "Dean Hall", "followers-count": 68 }, "sender-location": point("48.12,72.0"), "send-time": datetime("2006-11-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " love sprint the platform is awesome", "countA": 192, "countB": 199 }
+{ "tweetid": 193i64, "user": { "screen-name": "LoanClarke_206", "lang": "en", "friends-count": 32, "statuses-count": 173, "name": "Loan Clarke", "followers-count": 186 }, "sender-location": point("45.39,96.01"), "send-time": datetime("2009-05-15T10:10:00.000Z"), "referred-topics": {{ "verizon", "customization" }}, "message-text": " can't stand verizon its customization is horrible:(", "countA": 193, "countB": 147 }
+{ "tweetid": 194i64, "user": { "screen-name": "DonnetteGoodman@627", "lang": "en", "friends-count": 61, "statuses-count": 202, "name": "Donnette Goodman", "followers-count": 106 }, "sender-location": point("44.72,73.13"), "send-time": datetime("2005-04-27T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "plan" }}, "message-text": " love t-mobile its plan is good", "countA": 194, "countB": 142 }
+{ "tweetid": 195i64, "user": { "screen-name": "TamekaPorter#315", "lang": "en", "friends-count": 79, "statuses-count": 63, "name": "Tameka Porter", "followers-count": 7 }, "sender-location": point("37.68,81.78"), "send-time": datetime("2006-06-24T10:10:00.000Z"), "referred-topics": {{ "samsung", "wireless" }}, "message-text": " like samsung its wireless is good", "countA": 195, "countB": 126 }
+{ "tweetid": 196i64, "user": { "screen-name": "GarlandClark@425", "lang": "en", "friends-count": 15, "statuses-count": 375, "name": "Garland Clark", "followers-count": 24 }, "sender-location": point("44.9,70.1"), "send-time": datetime("2006-01-08T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint the touch-screen is OMG:(", "countA": 196, "countB": 197 }
+{ "tweetid": 197i64, "user": { "screen-name": "RupertSanner$868", "lang": "en", "friends-count": 49, "statuses-count": 414, "name": "Rupert Sanner", "followers-count": 189 }, "sender-location": point("40.45,94.94"), "send-time": datetime("2012-06-20T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " can't stand iphone the customer-service is terrible:(", "countA": 197, "countB": 195 }
+{ "tweetid": 198i64, "user": { "screen-name": "JenZoucks#841", "lang": "en", "friends-count": 99, "statuses-count": 73, "name": "Jen Zoucks", "followers-count": 71 }, "sender-location": point("38.77,70.33"), "send-time": datetime("2010-07-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "signal" }}, "message-text": " like iphone its signal is good", "countA": 198, "countB": 94 }
+{ "tweetid": 199i64, "user": { "screen-name": "IsmaelLlora@529", "lang": "en", "friends-count": 63, "statuses-count": 239, "name": "Ismael Llora", "followers-count": 135 }, "sender-location": point("35.31,82.71"), "send-time": datetime("2007-03-19T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is amazing:)", "countA": 199, "countB": 172 }
+{ "tweetid": 200i64, "user": { "screen-name": "JenniferHoltzer_459", "lang": "en", "friends-count": 93, "statuses-count": 172, "name": "Jennifer Holtzer", "followers-count": 51 }, "sender-location": point("30.44,81.57"), "send-time": datetime("2007-11-24T10:10:00.000Z"), "referred-topics": {{ "at&t", "wireless" }}, "message-text": " love at&t the wireless is amazing", "countA": 200, "countB": 20 }
+{ "tweetid": 201i64, "user": { "screen-name": "EvanJardine$78", "lang": "en", "friends-count": 80, "statuses-count": 349, "name": "Evan Jardine", "followers-count": 155 }, "sender-location": point("39.12,68.0"), "send-time": datetime("2010-01-13T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " like t-mobile its signal is awesome:)", "countA": 201, "countB": 65 }
+{ "tweetid": 202i64, "user": { "screen-name": "KerenBard_268", "lang": "en", "friends-count": 28, "statuses-count": 348, "name": "Keren Bard", "followers-count": 158 }, "sender-location": point("32.72,77.86"), "send-time": datetime("2008-07-18T10:10:00.000Z"), "referred-topics": {{ "sprint", "platform" }}, "message-text": " love sprint its platform is awesome:)", "countA": 202, "countB": 117 }
+{ "tweetid": 203i64, "user": { "screen-name": "GilbertLosey#710", "lang": "en", "friends-count": 67, "statuses-count": 141, "name": "Gilbert Losey", "followers-count": 192 }, "sender-location": point("44.66,84.32"), "send-time": datetime("2011-09-27T10:10:00.000Z"), "referred-topics": {{ "at&t", "signal" }}, "message-text": " dislike at&t its signal is horrible", "countA": 203, "countB": 182 }
+{ "tweetid": 204i64, "user": { "screen-name": "TawnyGraham_816", "lang": "en", "friends-count": 63, "statuses-count": 13, "name": "Tawny Graham", "followers-count": 155 }, "sender-location": point("43.08,86.89"), "send-time": datetime("2012-05-10T10:10:00.000Z"), "referred-topics": {{ "verizon", "touch-screen" }}, "message-text": " dislike verizon its touch-screen is terrible", "countA": 204, "countB": 104 }
+{ "tweetid": 205i64, "user": { "screen-name": "LamontWilkins_977", "lang": "en", "friends-count": 19, "statuses-count": 102, "name": "Lamont Wilkins", "followers-count": 127 }, "sender-location": point("31.77,87.68"), "send-time": datetime("2005-04-12T10:10:00.000Z"), "referred-topics": {{ "samsung", "3G" }}, "message-text": " dislike samsung its 3G is bad", "countA": 205, "countB": 57 }
+{ "tweetid": 206i64, "user": { "screen-name": "VincentRiggle$122", "lang": "en", "friends-count": 99, "statuses-count": 105, "name": "Vincent Riggle", "followers-count": 173 }, "sender-location": point("34.28,74.71"), "send-time": datetime("2007-05-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " like motorola the wireless is amazing:)", "countA": 206, "countB": 3 }
+{ "tweetid": 207i64, "user": { "screen-name": "AdelaJones_400", "lang": "en", "friends-count": 74, "statuses-count": 121, "name": "Adela Jones", "followers-count": 161 }, "sender-location": point("28.5,94.52"), "send-time": datetime("2010-01-20T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "platform" }}, "message-text": " like t-mobile the platform is good:)", "countA": 207, "countB": 110 }
+{ "tweetid": 208i64, "user": { "screen-name": "PrestonLittle$360", "lang": "en", "friends-count": 61, "statuses-count": 430, "name": "Preston Little", "followers-count": 115 }, "sender-location": point("35.79,78.52"), "send-time": datetime("2010-03-01T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " dislike sprint the speed is terrible:(", "countA": 208, "countB": 149 }
+{ "tweetid": 209i64, "user": { "screen-name": "TaraAnderson#214", "lang": "en", "friends-count": 60, "statuses-count": 219, "name": "Tara Anderson", "followers-count": 110 }, "sender-location": point("34.04,79.5"), "send-time": datetime("2006-12-05T10:10:00.000Z"), "referred-topics": {{ "motorola", "plan" }}, "message-text": " dislike motorola its plan is OMG", "countA": 209, "countB": 164 }
+{ "tweetid": 210i64, "user": { "screen-name": "DeshawnWallace$305", "lang": "en", "friends-count": 29, "statuses-count": 279, "name": "Deshawn Wallace", "followers-count": 27 }, "sender-location": point("39.49,76.58"), "send-time": datetime("2011-01-05T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " dislike t-mobile the customization is OMG", "countA": 210, "countB": 70 }
+{ "tweetid": 211i64, "user": { "screen-name": "OliFisher$694", "lang": "en", "friends-count": 39, "statuses-count": 131, "name": "Oli Fisher", "followers-count": 101 }, "sender-location": point("44.91,94.72"), "send-time": datetime("2010-04-10T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " love iphone the 3G is mind-blowing", "countA": 211, "countB": 95 }
+{ "tweetid": 212i64, "user": { "screen-name": "LizaMathews#376", "lang": "en", "friends-count": 40, "statuses-count": 107, "name": "Liza Mathews", "followers-count": 70 }, "sender-location": point("46.01,77.85"), "send-time": datetime("2009-07-20T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " like verizon the plan is mind-blowing:)", "countA": 212, "countB": 189 }
+{ "tweetid": 213i64, "user": { "screen-name": "MylesRahl#433", "lang": "en", "friends-count": 51, "statuses-count": 144, "name": "Myles Rahl", "followers-count": 90 }, "sender-location": point("45.41,95.69"), "send-time": datetime("2006-04-03T10:10:00.000Z"), "referred-topics": {{ "verizon", "3G" }}, "message-text": " love verizon its 3G is mind-blowing:)", "countA": 213, "countB": 190 }
+{ "tweetid": 214i64, "user": { "screen-name": "BertEve@968", "lang": "en", "friends-count": 84, "statuses-count": 110, "name": "Bert Eve", "followers-count": 122 }, "sender-location": point("43.25,87.82"), "send-time": datetime("2009-11-07T10:10:00.000Z"), "referred-topics": {{ "at&t", "voice-command" }}, "message-text": " dislike at&t the voice-command is terrible:(", "countA": 214, "countB": 142 }
+{ "tweetid": 215i64, "user": { "screen-name": "MelissaLaurenzi_383", "lang": "en", "friends-count": 78, "statuses-count": 318, "name": "Melissa Laurenzi", "followers-count": 19 }, "sender-location": point("37.82,86.96"), "send-time": datetime("2010-05-14T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " like iphone its customer-service is mind-blowing", "countA": 215, "countB": 67 }
+{ "tweetid": 216i64, "user": { "screen-name": "ReneaPennington#175", "lang": "en", "friends-count": 42, "statuses-count": 16, "name": "Renea Pennington", "followers-count": 52 }, "sender-location": point("39.06,78.62"), "send-time": datetime("2006-05-23T10:10:00.000Z"), "referred-topics": {{ "iphone", "3G" }}, "message-text": " dislike iphone its 3G is bad:(", "countA": 216, "countB": 50 }
+{ "tweetid": 217i64, "user": { "screen-name": "UrsulaMitchell@26", "lang": "en", "friends-count": 50, "statuses-count": 150, "name": "Ursula Mitchell", "followers-count": 191 }, "sender-location": point("34.02,71.3"), "send-time": datetime("2005-03-26T10:10:00.000Z"), "referred-topics": {{ "iphone", "customer-service" }}, "message-text": " like iphone its customer-service is awesome", "countA": 217, "countB": 154 }
+{ "tweetid": 218i64, "user": { "screen-name": "WillyLambert_669", "lang": "en", "friends-count": 6, "statuses-count": 44, "name": "Willy Lambert", "followers-count": 60 }, "sender-location": point("32.84,73.25"), "send-time": datetime("2011-10-04T10:10:00.000Z"), "referred-topics": {{ "verizon", "customization" }}, "message-text": " like verizon the customization is amazing", "countA": 218, "countB": 135 }
+{ "tweetid": 219i64, "user": { "screen-name": "ChristineLaurence_912", "lang": "en", "friends-count": 77, "statuses-count": 458, "name": "Christine Laurence", "followers-count": 166 }, "sender-location": point("35.39,90.54"), "send-time": datetime("2009-12-01T10:10:00.000Z"), "referred-topics": {{ "at&t", "customization" }}, "message-text": " like at&t the customization is awesome:)", "countA": 219, "countB": 170 }
+{ "tweetid": 220i64, "user": { "screen-name": "JedFiddler_540", "lang": "en", "friends-count": 11, "statuses-count": 264, "name": "Jed Fiddler", "followers-count": 48 }, "sender-location": point("44.11,91.45"), "send-time": datetime("2009-10-10T10:10:00.000Z"), "referred-topics": {{ "at&t", "voicemail-service" }}, "message-text": " like at&t its voicemail-service is awesome:)", "countA": 220, "countB": 188 }
+{ "tweetid": 221i64, "user": { "screen-name": "RubinMueller_263", "lang": "en", "friends-count": 7, "statuses-count": 374, "name": "Rubin Mueller", "followers-count": 77 }, "sender-location": point("25.65,78.68"), "send-time": datetime("2006-09-20T10:10:00.000Z"), "referred-topics": {{ "sprint", "signal" }}, "message-text": " love sprint its signal is amazing", "countA": 221, "countB": 185 }
+{ "tweetid": 222i64, "user": { "screen-name": "JeniWeldi#255", "lang": "en", "friends-count": 71, "statuses-count": 259, "name": "Jeni Weldi", "followers-count": 63 }, "sender-location": point("27.21,67.74"), "send-time": datetime("2009-11-24T10:10:00.000Z"), "referred-topics": {{ "iphone", "customization" }}, "message-text": " like iphone its customization is mind-blowing:)", "countA": 222, "countB": 51 }
+{ "tweetid": 223i64, "user": { "screen-name": "RochelleSaline@265", "lang": "en", "friends-count": 18, "statuses-count": 319, "name": "Rochelle Saline", "followers-count": 11 }, "sender-location": point("33.44,67.08"), "send-time": datetime("2007-10-23T10:10:00.000Z"), "referred-topics": {{ "iphone", "voice-command" }}, "message-text": " can't stand iphone its voice-command is OMG", "countA": 223, "countB": 84 }
+{ "tweetid": 224i64, "user": { "screen-name": "TabbyEckhardstein@204", "lang": "en", "friends-count": 5, "statuses-count": 207, "name": "Tabby Eckhardstein", "followers-count": 5 }, "sender-location": point("24.33,84.9"), "send-time": datetime("2005-02-01T10:10:00.000Z"), "referred-topics": {{ "samsung", "speed" }}, "message-text": " hate samsung its speed is horrible", "countA": 224, "countB": 192 }
+{ "tweetid": 225i64, "user": { "screen-name": "FranklynBurns$23", "lang": "en", "friends-count": 73, "statuses-count": 475, "name": "Franklyn Burns", "followers-count": 131 }, "sender-location": point("26.5,94.14"), "send-time": datetime("2007-08-19T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " love motorola its reachability is good", "countA": 225, "countB": 65 }
+{ "tweetid": 226i64, "user": { "screen-name": "CorianderFischer#204", "lang": "en", "friends-count": 4, "statuses-count": 205, "name": "Coriander Fischer", "followers-count": 30 }, "sender-location": point("46.59,73.39"), "send-time": datetime("2010-03-13T10:10:00.000Z"), "referred-topics": {{ "motorola", "reachability" }}, "message-text": " can't stand motorola its reachability is horrible", "countA": 226, "countB": 73 }
+{ "tweetid": 227i64, "user": { "screen-name": "EmikoBarth$856", "lang": "en", "friends-count": 28, "statuses-count": 7, "name": "Emiko Barth", "followers-count": 98 }, "sender-location": point("42.43,97.13"), "send-time": datetime("2005-03-23T10:10:00.000Z"), "referred-topics": {{ "verizon", "reachability" }}, "message-text": " can't stand verizon the reachability is bad:(", "countA": 227, "countB": 184 }
+{ "tweetid": 228i64, "user": { "screen-name": "LavoneStroh$831", "lang": "en", "friends-count": 53, "statuses-count": 2, "name": "Lavone Stroh", "followers-count": 44 }, "sender-location": point("32.04,66.17"), "send-time": datetime("2011-04-25T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " love t-mobile its touch-screen is awesome:)", "countA": 228, "countB": 38 }
+{ "tweetid": 229i64, "user": { "screen-name": "ShantelLaurence@745", "lang": "en", "friends-count": 78, "statuses-count": 195, "name": "Shantel Laurence", "followers-count": 124 }, "sender-location": point("46.58,97.58"), "send-time": datetime("2007-12-02T10:10:00.000Z"), "referred-topics": {{ "iphone", "wireless" }}, "message-text": " love iphone its wireless is awesome:)", "countA": 229, "countB": 26 }
+{ "tweetid": 230i64, "user": { "screen-name": "PiedadMosser@971", "lang": "en", "friends-count": 84, "statuses-count": 80, "name": "Piedad Mosser", "followers-count": 198 }, "sender-location": point("40.44,70.59"), "send-time": datetime("2005-06-27T10:10:00.000Z"), "referred-topics": {{ "motorola", "wireless" }}, "message-text": " hate motorola the wireless is bad", "countA": 230, "countB": 46 }
+{ "tweetid": 231i64, "user": { "screen-name": "CorrieHindman$963", "lang": "en", "friends-count": 54, "statuses-count": 78, "name": "Corrie Hindman", "followers-count": 180 }, "sender-location": point("26.04,77.36"), "send-time": datetime("2012-03-07T10:10:00.000Z"), "referred-topics": {{ "sprint", "wireless" }}, "message-text": " like sprint its wireless is mind-blowing:)", "countA": 231, "countB": 77 }
+{ "tweetid": 232i64, "user": { "screen-name": "DignaPorter_932", "lang": "en", "friends-count": 63, "statuses-count": 278, "name": "Digna Porter", "followers-count": 139 }, "sender-location": point("41.76,97.51"), "send-time": datetime("2009-12-01T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is mind-blowing:)", "countA": 232, "countB": 85 }
+{ "tweetid": 233i64, "user": { "screen-name": "TerrellStoddard@17", "lang": "en", "friends-count": 74, "statuses-count": 3, "name": "Terrell Stoddard", "followers-count": 35 }, "sender-location": point("34.89,73.83"), "send-time": datetime("2007-01-28T10:10:00.000Z"), "referred-topics": {{ "at&t", "touch-screen" }}, "message-text": " can't stand at&t the touch-screen is terrible", "countA": 233, "countB": 28 }
+{ "tweetid": 234i64, "user": { "screen-name": "EzraField#602", "lang": "en", "friends-count": 7, "statuses-count": 41, "name": "Ezra Field", "followers-count": 169 }, "sender-location": point("26.51,74.01"), "send-time": datetime("2011-03-06T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " love motorola the speed is good", "countA": 234, "countB": 94 }
+{ "tweetid": 235i64, "user": { "screen-name": "NathanielWentzel$505", "lang": "en", "friends-count": 40, "statuses-count": 178, "name": "Nathaniel Wentzel", "followers-count": 83 }, "sender-location": point("47.03,84.5"), "send-time": datetime("2007-07-02T10:10:00.000Z"), "referred-topics": {{ "motorola", "voice-clarity" }}, "message-text": " hate motorola its voice-clarity is bad", "countA": 235, "countB": 53 }
+{ "tweetid": 236i64, "user": { "screen-name": "AlanePycroft$112", "lang": "en", "friends-count": 61, "statuses-count": 222, "name": "Alane Pycroft", "followers-count": 125 }, "sender-location": point("31.82,81.86"), "send-time": datetime("2010-09-09T10:10:00.000Z"), "referred-topics": {{ "at&t", "platform" }}, "message-text": " like at&t its platform is amazing:)", "countA": 236, "countB": 133 }
+{ "tweetid": 237i64, "user": { "screen-name": "EliseoMunson$584", "lang": "en", "friends-count": 21, "statuses-count": 353, "name": "Eliseo Munson", "followers-count": 9 }, "sender-location": point("39.12,74.86"), "send-time": datetime("2011-05-02T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " hate sprint its speed is terrible", "countA": 237, "countB": 100 }
+{ "tweetid": 238i64, "user": { "screen-name": "LupeCram@152", "lang": "en", "friends-count": 89, "statuses-count": 57, "name": "Lupe Cram", "followers-count": 51 }, "sender-location": point("40.34,71.58"), "send-time": datetime("2010-12-13T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is amazing", "countA": 238, "countB": 47 }
+{ "tweetid": 239i64, "user": { "screen-name": "AuroraChristman#544", "lang": "en", "friends-count": 96, "statuses-count": 435, "name": "Aurora Christman", "followers-count": 88 }, "sender-location": point("35.44,67.79"), "send-time": datetime("2009-08-11T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "signal" }}, "message-text": " love t-mobile the signal is mind-blowing", "countA": 239, "countB": 114 }
+{ "tweetid": 240i64, "user": { "screen-name": "JermainePotter_329", "lang": "en", "friends-count": 20, "statuses-count": 85, "name": "Jermaine Potter", "followers-count": 177 }, "sender-location": point("35.26,90.47"), "send-time": datetime("2012-04-17T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "network" }}, "message-text": " like t-mobile its network is mind-blowing:)", "countA": 240, "countB": 104 }
+{ "tweetid": 241i64, "user": { "screen-name": "KaetyHall$972", "lang": "en", "friends-count": 23, "statuses-count": 416, "name": "Kaety Hall", "followers-count": 87 }, "sender-location": point("25.06,93.28"), "send-time": datetime("2011-02-13T10:10:00.000Z"), "referred-topics": {{ "verizon", "network" }}, "message-text": " can't stand verizon its network is bad:(", "countA": 241, "countB": 45 }
+{ "tweetid": 242i64, "user": { "screen-name": "LaurenChristopher#195", "lang": "en", "friends-count": 12, "statuses-count": 149, "name": "Lauren Christopher", "followers-count": 183 }, "sender-location": point("24.03,79.65"), "send-time": datetime("2012-05-22T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "touch-screen" }}, "message-text": " love t-mobile the touch-screen is amazing", "countA": 242, "countB": 179 }
+{ "tweetid": 243i64, "user": { "screen-name": "KoreyBonner$296", "lang": "en", "friends-count": 56, "statuses-count": 412, "name": "Korey Bonner", "followers-count": 190 }, "sender-location": point("42.54,97.81"), "send-time": datetime("2006-10-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "touch-screen" }}, "message-text": " like iphone its touch-screen is amazing:)", "countA": 243, "countB": 176 }
+{ "tweetid": 244i64, "user": { "screen-name": "LaetitiaWise@568", "lang": "en", "friends-count": 87, "statuses-count": 363, "name": "Laetitia Wise", "followers-count": 49 }, "sender-location": point("39.34,70.46"), "send-time": datetime("2010-11-25T10:10:00.000Z"), "referred-topics": {{ "iphone", "voicemail-service" }}, "message-text": " hate iphone its voicemail-service is terrible", "countA": 244, "countB": 150 }
+{ "tweetid": 245i64, "user": { "screen-name": "PatricaKellogg@123", "lang": "en", "friends-count": 4, "statuses-count": 80, "name": "Patrica Kellogg", "followers-count": 47 }, "sender-location": point("34.06,84.02"), "send-time": datetime("2005-05-27T10:10:00.000Z"), "referred-topics": {{ "sprint", "touch-screen" }}, "message-text": " hate sprint its touch-screen is bad:(", "countA": 245, "countB": 83 }
+{ "tweetid": 246i64, "user": { "screen-name": "AdelleSwink$50", "lang": "en", "friends-count": 64, "statuses-count": 411, "name": "Adelle Swink", "followers-count": 165 }, "sender-location": point("28.66,71.42"), "send-time": datetime("2007-04-11T10:10:00.000Z"), "referred-topics": {{ "sprint", "plan" }}, "message-text": " can't stand sprint the plan is horrible", "countA": 246, "countB": 128 }
+{ "tweetid": 247i64, "user": { "screen-name": "MckennaAlbright$534", "lang": "en", "friends-count": 94, "statuses-count": 374, "name": "Mckenna Albright", "followers-count": 61 }, "sender-location": point("45.26,93.62"), "send-time": datetime("2011-11-12T10:10:00.000Z"), "referred-topics": {{ "sprint", "speed" }}, "message-text": " can't stand sprint the speed is OMG", "countA": 247, "countB": 194 }
+{ "tweetid": 248i64, "user": { "screen-name": "MerryReade$689", "lang": "en", "friends-count": 41, "statuses-count": 172, "name": "Merry Reade", "followers-count": 105 }, "sender-location": point("42.21,81.81"), "send-time": datetime("2009-08-18T10:10:00.000Z"), "referred-topics": {{ "verizon", "wireless" }}, "message-text": " like verizon its wireless is amazing", "countA": 248, "countB": 123 }
+{ "tweetid": 249i64, "user": { "screen-name": "CoryGoldvogel$187", "lang": "en", "friends-count": 91, "statuses-count": 359, "name": "Cory Goldvogel", "followers-count": 20 }, "sender-location": point("39.94,83.42"), "send-time": datetime("2005-09-11T10:10:00.000Z"), "referred-topics": {{ "verizon", "plan" }}, "message-text": " dislike verizon its plan is bad:(", "countA": 249, "countB": 163 }
+{ "tweetid": 250i64, "user": { "screen-name": "RoystonRummel@500", "lang": "en", "friends-count": 34, "statuses-count": 154, "name": "Royston Rummel", "followers-count": 146 }, "sender-location": point("40.73,72.93"), "send-time": datetime("2005-11-20T10:10:00.000Z"), "referred-topics": {{ "samsung", "touch-screen" }}, "message-text": " love samsung its touch-screen is amazing:)", "countA": 250, "countB": 125 }
\ No newline at end of file

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
index 50adb57..523c59c 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/http/servlet/UpdateAPIServlet.java

@@ -31,7 +31,7 @@
 
     protected List<Statement.Kind> getAllowedStatements() {
         Kind[] statementsArray = { Kind.DATAVERSE_DECL, Kind.DELETE, Kind.INSERT, Kind.UPDATE, Kind.DML_CMD_LIST,
-                Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT };
+                Kind.LOAD, Kind.CONNECT_FEED, Kind.DISCONNECT_FEED, Kind.SET, Kind.COMPACT, Kind.EXTERNAL_DATASET_REFRESH };
         return Arrays.asList(statementsArray);
     }
 

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 35518a6..ef823ff 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java

@@ -19,8 +19,10 @@
 import java.nio.ByteBuffer;
 import java.rmi.RemoteException;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -61,12 +63,16 @@
 import edu.uci.ics.asterix.aql.expression.NodeGroupDropStatement;
 import edu.uci.ics.asterix.aql.expression.NodegroupDecl;
 import edu.uci.ics.asterix.aql.expression.Query;
+import edu.uci.ics.asterix.aql.expression.RefreshExternalDatasetStatement;
 import edu.uci.ics.asterix.aql.expression.SetStatement;
 import edu.uci.ics.asterix.aql.expression.TypeDecl;
 import edu.uci.ics.asterix.aql.expression.TypeDropStatement;
 import edu.uci.ics.asterix.aql.expression.WriteStatement;
 import edu.uci.ics.asterix.aql.util.FunctionUtils;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.common.config.GlobalConfig;
 import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
 import edu.uci.ics.asterix.common.exceptions.ACIDException;
@@ -75,6 +81,7 @@
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.file.DatasetOperations;
 import edu.uci.ics.asterix.file.DataverseOperations;
+import edu.uci.ics.asterix.file.ExternalIndexingOperations;
 import edu.uci.ics.asterix.file.FeedOperations;
 import edu.uci.ics.asterix.file.IndexOperations;
 import edu.uci.ics.asterix.formats.base.IDataFormat;
@@ -92,6 +99,7 @@
 import edu.uci.ics.asterix.metadata.entities.Datatype;
 import edu.uci.ics.asterix.metadata.entities.Dataverse;
 import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
 import edu.uci.ics.asterix.metadata.entities.Feed;
 import edu.uci.ics.asterix.metadata.entities.FeedActivity;
 import edu.uci.ics.asterix.metadata.entities.FeedPolicy;
@@ -101,6 +109,7 @@
 import edu.uci.ics.asterix.metadata.entities.NodeGroup;
 import edu.uci.ics.asterix.metadata.feeds.BuiltinFeedPolicies;
 import edu.uci.ics.asterix.metadata.feeds.FeedUtil;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -308,6 +317,11 @@
                     break;
                 }
 
+                case EXTERNAL_DATASET_REFRESH: {
+                    handleExternalDatasetRefreshStatement(metadataProvider, stmt, hcc);
+                    break;
+                }
+
                 case WRITE: {
                     Pair<IAWriterFactory, FileSplit> result = handleWriteStatement(metadataProvider, stmt);
                     if (result.first != null) {
@@ -480,7 +494,20 @@
                 case EXTERNAL: {
                     String adapter = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getAdapter();
                     Map<String, String> properties = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getProperties();
-                    datasetDetails = new ExternalDatasetDetails(adapter, properties);
+                    Identifier ngNameId = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getNodegroupName();
+                    String ngName = ngNameId != null ? ngNameId.getValue() : configureNodegroupForDataset(dd,
+                            dataverseName, mdTxnCtx);
+                    String compactionPolicy = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getCompactionPolicy();
+                    Map<String, String> compactionPolicyProperties = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl())
+                            .getCompactionPolicyProperties();
+                    if (compactionPolicy == null) {
+                        compactionPolicy = GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME;
+                        compactionPolicyProperties = GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES;
+                    } else {
+                        validateCompactionPolicy(compactionPolicy, compactionPolicyProperties, mdTxnCtx);
+                    }
+                    datasetDetails = new ExternalDatasetDetails(adapter, properties, ngName, new Date(),
+                            ExternalDatasetTransactionState.COMMIT, compactionPolicy, compactionPolicyProperties);
                     break;
                 }
 
@@ -625,7 +652,6 @@
 
     private void handleCreateIndexStatement(AqlMetadataProvider metadataProvider, Statement stmt,
             IHyracksClientConnection hcc) throws Exception {
-
         ProgressState progress = ProgressState.NO_PROGRESS;
         MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
         boolean bActiveTxn = true;
@@ -637,6 +663,12 @@
         String indexName = null;
         JobSpecification spec = null;
         Dataset ds = null;
+        // For external datasets
+        ArrayList<ExternalFile> externalFilesSnapshot = null;
+        boolean firstExternalDatasetIndex = false;
+        boolean filesIndexReplicated = false;
+        Index filesIndex = null;
+        boolean datasetLocked = false;
         try {
             CreateIndexStatement stmtCreateIndex = (CreateIndexStatement) stmt;
             dataverseName = getActiveDataverseName(stmtCreateIndex.getDataverseName());
@@ -669,17 +701,61 @@
                 }
             }
 
-            List<FeedActivity> feedActivities = MetadataManager.INSTANCE.getActiveFeeds(mdTxnCtx, dataverseName,
-                    datasetName);
-            if (feedActivities != null && !feedActivities.isEmpty()) {
-                StringBuilder builder = new StringBuilder();
+            if (ds.getDatasetType() == DatasetType.INTERNAL) {
+                List<FeedActivity> feedActivities = MetadataManager.INSTANCE.getActiveFeeds(mdTxnCtx, dataverseName,
+                        datasetName);
+                if (feedActivities != null && !feedActivities.isEmpty()) {
+                    StringBuilder builder = new StringBuilder();
 
-                for (FeedActivity fa : feedActivities) {
-                    builder.append(fa + "\n");
+                    for (FeedActivity fa : feedActivities) {
+                        builder.append(fa + "\n");
+                    }
+                    throw new AsterixException("Dataset" + datasetName
+                            + " is currently being fed into by the following feeds " + "." + builder.toString()
+                            + "\nOperation not supported.");
                 }
-                throw new AsterixException("Dataset" + datasetName
-                        + " is currently being fed into by the following feeds " + "." + builder.toString()
-                        + "\nOperation not supported.");
+
+            } else {
+                // External dataset
+                // Check if the dataset is indexible
+                if (!ExternalIndexingOperations.isIndexible((ExternalDatasetDetails) ds.getDatasetDetails())) {
+                    throw new AlgebricksException("dataset using "
+                            + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
+                            + " Adapter can't be indexed");
+                }
+                // check if the name of the index is valid
+                if (!ExternalIndexingOperations.isValidIndexName(datasetName, indexName)) {
+                    throw new AlgebricksException("external dataset index name is invalid");
+                }
+                // lock external dataset
+                ExternalDatasetsRegistry.INSTANCE.buildIndexBegin(ds);
+                datasetLocked = true;
+                // Check if the files index exist
+                filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                        datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+                firstExternalDatasetIndex = (filesIndex == null);
+                if (firstExternalDatasetIndex) {
+                    // Get snapshot from External File System
+                    externalFilesSnapshot = ExternalIndexingOperations.getSnapshotFromExternalFileSystem(ds);
+                    // Add an entry for the files index
+                    filesIndex = new Index(dataverseName, datasetName,
+                            ExternalIndexingOperations.getFilesIndexName(datasetName), IndexType.BTREE,
+                            ExternalIndexingOperations.FILE_INDEX_FIELDS, false, IMetadataEntity.PENDING_ADD_OP);
+                    MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
+                    // Add files to the external files index
+                    for (ExternalFile file : externalFilesSnapshot) {
+                        MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+                    }
+                    // This is the first index for the external dataset, replicate the files index
+                    spec = ExternalIndexingOperations.buildFilesIndexReplicationJobSpec(ds, externalFilesSnapshot,
+                            metadataProvider, true);
+                    if (spec == null) {
+                        throw new AsterixException(
+                                "Failed to create job spec for replicating Files Index For external dataset");
+                    }
+                    filesIndexReplicated = true;
+                    runJob(hcc, spec, true);
+                }
             }
 
             //#. add a new index with PendingAddOp
@@ -726,12 +802,41 @@
                     indexName);
             index.setPendingOp(IMetadataEntity.PENDING_NO_OP);
             MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
+            // add another new files index with PendingNoOp after deleting the index with PendingAddOp
+            if (firstExternalDatasetIndex) {
+                MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                        datasetName, filesIndex.getIndexName());
+                filesIndex.setPendingOp(IMetadataEntity.PENDING_NO_OP);
+                MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
+                // update transaction timestamp
+                ((ExternalDatasetDetails) ds.getDatasetDetails()).setRefreshTimestamp(new Date());
+                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
+            }
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
 
         } catch (Exception e) {
             if (bActiveTxn) {
                 abort(e, e, mdTxnCtx);
             }
+            // If files index was replicated for external dataset, it should be cleaned up on NC side
+            if (filesIndexReplicated) {
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                        ExternalIndexingOperations.getFilesIndexName(datasetName));
+                try {
+                    JobSpecification jobSpec = ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+                            metadataProvider, ds);
+                    MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    bActiveTxn = false;
+                    runJob(hcc, jobSpec, true);
+                } catch (Exception e2) {
+                    e.addSuppressed(e2);
+                    if (bActiveTxn) {
+                        abort(e, e2, mdTxnCtx);
+                    }
+                }
+            }
 
             if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
                 //#. execute compensation operations
@@ -754,7 +859,35 @@
                     }
                 }
 
-                //   remove the record from the metadata.
+                if (firstExternalDatasetIndex) {
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                    try {
+                        // Drop External Files from metadata
+                        MetadataManager.INSTANCE.dropDatasetExternalFiles(mdTxnCtx, ds);
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    } catch (Exception e2) {
+                        e.addSuppressed(e2);
+                        abort(e, e2, mdTxnCtx);
+                        throw new IllegalStateException("System is inconsistent state: pending files for("
+                                + dataverseName + "." + datasetName + ") couldn't be removed from the metadata", e);
+                    }
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                    try {
+                        // Drop the files index from metadata
+                        MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                                datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    } catch (Exception e2) {
+                        e.addSuppressed(e2);
+                        abort(e, e2, mdTxnCtx);
+                        throw new IllegalStateException("System is inconsistent state: pending index(" + dataverseName
+                                + "." + datasetName + "." + ExternalIndexingOperations.getFilesIndexName(datasetName)
+                                + ") couldn't be removed from the metadata", e);
+                    }
+                }
+                // remove the record from the metadata.
                 mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
                 metadataProvider.setMetadataTxnContext(mdTxnCtx);
                 try {
@@ -764,13 +897,16 @@
                 } catch (Exception e2) {
                     e.addSuppressed(e2);
                     abort(e, e2, mdTxnCtx);
-                    throw new IllegalStateException("System is inconsistent state: pending index(" + dataverseName
+                    throw new IllegalStateException("System is in inconsistent state: pending index(" + dataverseName
                             + "." + datasetName + "." + indexName + ") couldn't be removed from the metadata", e);
                 }
             }
             throw e;
         } finally {
             releaseWriteLatch();
+            if (datasetLocked) {
+                ExternalDatasetsRegistry.INSTANCE.buildIndexEnd(ds);
+            }
         }
     }
 
@@ -864,7 +1000,6 @@
                 String datasetName = datasets.get(j).getDatasetName();
                 DatasetType dsType = datasets.get(j).getDatasetType();
                 if (dsType == DatasetType.INTERNAL) {
-
                     List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
                             datasetName);
                     for (int k = 0; k < indexes.size(); k++) {
@@ -878,6 +1013,23 @@
 
                     CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
                     jobsToExecute.add(DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider));
+                } else {
+                    // External dataset
+                    List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
+                            datasetName);
+                    for (int k = 0; k < indexes.size(); k++) {
+                        if (ExternalIndexingOperations.isFileIndex(indexes.get(k))) {
+                            CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                    indexes.get(k).getIndexName());
+                            jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+                                    metadataProvider, datasets.get(j)));
+                        } else {
+                            CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                    indexes.get(k).getIndexName());
+                            jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider,
+                                    datasets.get(j)));
+                        }
+                    }
                 }
             }
             jobsToExecute.add(DataverseOperations.createDropDataverseJobSpec(dv, metadataProvider));
@@ -906,7 +1058,7 @@
             if (activeDefaultDataverse != null && activeDefaultDataverse.getDataverseName() == dataverseName) {
                 activeDefaultDataverse = null;
             }
-
+            ExternalDatasetsRegistry.INSTANCE.removeDataverse(dv);
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
         } catch (Exception e) {
             if (bActiveTxn) {
@@ -1029,16 +1181,52 @@
                 mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
                 bActiveTxn = true;
                 metadataProvider.setMetadataTxnContext(mdTxnCtx);
+            } else {
+                // External dataset
+                //#. prepare jobs to drop the datatset and the indexes in NC
+                List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
+                for (int j = 0; j < indexes.size(); j++) {
+                    if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
+                        CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                indexes.get(j).getIndexName());
+                        jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
+                    } else {
+                        CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                indexes.get(j).getIndexName());
+                        jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds, metadataProvider,
+                                ds));
+                    }
+                }
+
+                //#. mark the existing dataset as PendingDropOp
+                MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
+                MetadataManager.INSTANCE.addDataset(
+                        mdTxnCtx,
+                        new Dataset(dataverseName, datasetName, ds.getItemTypeName(), ds.getDatasetDetails(), ds
+                                .getHints(), ds.getDatasetType(), ds.getDatasetId(), IMetadataEntity.PENDING_DROP_OP));
+
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                bActiveTxn = false;
+                progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+
+                //#. run the jobs
+                for (JobSpecification jobSpec : jobsToExecute) {
+                    runJob(hcc, jobSpec, true);
+                }
+                if (indexes.size() > 0) {
+                    ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
+                }
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                metadataProvider.setMetadataTxnContext(mdTxnCtx);
             }
 
             //#. finally, delete the dataset.
             MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
             // Drop the associated nodegroup
-            if (ds.getDatasetType() == DatasetType.INTERNAL) {
-                String nodegroup = ((InternalDatasetDetails) ds.getDatasetDetails()).getNodeGroupName();
-                if (!nodegroup.equalsIgnoreCase(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)) {
-                    MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, dataverseName + ":" + datasetName);
-                }
+            String nodegroup = ds.getDatasetDetails().getNodeGroupName();
+            if (!nodegroup.equalsIgnoreCase(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)) {
+                MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, dataverseName + ":" + datasetName);
             }
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1092,6 +1280,7 @@
         String dataverseName = null;
         String datasetName = null;
         String indexName = null;
+        boolean dropFilesIndex = false;
         List<JobSpecification> jobsToExecute = new ArrayList<JobSpecification>();
         try {
             IndexDropStatement stmtIndexDrop = (IndexDropStatement) stmt;
@@ -1155,8 +1344,74 @@
                 //#. finally, delete the existing index
                 MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
             } else {
-                throw new AlgebricksException(datasetName
-                        + " is an external dataset. Indexes are not maintained for external datasets.");
+                // External dataset
+                indexName = stmtIndexDrop.getIndexName().getValue();
+                Index index = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, indexName);
+                if (index == null) {
+                    if (stmtIndexDrop.getIfExists()) {
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                        return;
+                    } else {
+                        throw new AlgebricksException("There is no index with this name " + indexName + ".");
+                    }
+                } else if (ExternalIndexingOperations.isFileIndex(index)) {
+                    throw new AlgebricksException("Dropping a dataset's files index is not allowed.");
+                }
+                //#. prepare a job to drop the index in NC.
+                CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName, indexName);
+                jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
+                List<Index> datasetIndexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
+                        datasetName);
+                if (datasetIndexes.size() == 2) {
+                    dropFilesIndex = true;
+                    // only one index + the files index, we need to delete both of the indexes
+                    for (Index externalIndex : datasetIndexes) {
+                        if (ExternalIndexingOperations.isFileIndex(externalIndex)) {
+                            cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+                                    externalIndex.getIndexName());
+                            jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+                                    metadataProvider, ds));
+                            //#. mark PendingDropOp on the existing files index
+                            MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName,
+                                    externalIndex.getIndexName());
+                            MetadataManager.INSTANCE.addIndex(
+                                    mdTxnCtx,
+                                    new Index(dataverseName, datasetName, externalIndex.getIndexName(), externalIndex
+                                            .getIndexType(), externalIndex.getKeyFieldNames(), externalIndex
+                                            .isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
+                        }
+                    }
+                }
+
+                //#. mark PendingDropOp on the existing index
+                MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
+                MetadataManager.INSTANCE.addIndex(mdTxnCtx,
+                        new Index(dataverseName, datasetName, indexName, index.getIndexType(),
+                                index.getKeyFieldNames(), index.isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
+
+                //#. commit the existing transaction before calling runJob. 
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                bActiveTxn = false;
+                progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+
+                for (JobSpecification jobSpec : jobsToExecute) {
+                    runJob(hcc, jobSpec, true);
+                }
+
+                //#. begin a new transaction
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+                //#. finally, delete the existing index
+                MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
+                if (dropFilesIndex) {
+                    // delete the files index too
+                    MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName,
+                            ExternalIndexingOperations.getFilesIndexName(datasetName));
+                    MetadataManager.INSTANCE.dropDatasetExternalFiles(mdTxnCtx, ds);
+                    ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
+                }
             }
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
 
@@ -1183,6 +1438,10 @@
                 try {
                     MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
                             datasetName, indexName);
+                    if (dropFilesIndex) {
+                        MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+                                datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+                    }
                     MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
                 } catch (Exception e2) {
                     e.addSuppressed(e2);
@@ -1673,29 +1932,41 @@
             CompactStatement compactStatement = (CompactStatement) stmt;
             dataverseName = getActiveDataverseName(compactStatement.getDataverseName());
             datasetName = compactStatement.getDatasetName().getValue();
-
             Dataset ds = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
             if (ds == null) {
                 throw new AlgebricksException("There is no dataset with this name " + datasetName + " in dataverse "
                         + dataverseName + ".");
-            } else if (ds.getDatasetType() != DatasetType.INTERNAL) {
-                throw new AlgebricksException("Cannot compact the extrenal dataset " + datasetName + ".");
             }
-
-            // Prepare jobs to compact the datatset and its indexes
             List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
-            for (int j = 0; j < indexes.size(); j++) {
-                if (indexes.get(j).isSecondaryIndex()) {
-                    CompiledIndexCompactStatement cics = new CompiledIndexCompactStatement(dataverseName, datasetName,
-                            indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes.get(j)
-                                    .getGramLength(), indexes.get(j).getIndexType());
-                    jobsToExecute.add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, metadataProvider, ds));
-                }
+            if (indexes.size() == 0) {
+                throw new AlgebricksException("Cannot compact the extrenal dataset " + datasetName
+                        + " because it has no indexes");
             }
-            Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
-                    dataverseName);
-            jobsToExecute.add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider));
-
+            if (ds.getDatasetType() == DatasetType.INTERNAL) {
+                for (int j = 0; j < indexes.size(); j++) {
+                    if (indexes.get(j).isSecondaryIndex()) {
+                        CompiledIndexCompactStatement cics = new CompiledIndexCompactStatement(dataverseName,
+                                datasetName, indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes
+                                        .get(j).getGramLength(), indexes.get(j).getIndexType());
+                        jobsToExecute
+                                .add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, metadataProvider, ds));
+                    }
+                }
+                Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
+                        dataverseName);
+                jobsToExecute.add(DatasetOperations.compactDatasetJobSpec(dataverse, datasetName, metadataProvider));
+            } else {
+                for (int j = 0; j < indexes.size(); j++) {
+                    if (!ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
+                        CompiledIndexCompactStatement cics = new CompiledIndexCompactStatement(dataverseName,
+                                datasetName, indexes.get(j).getIndexName(), indexes.get(j).getKeyFieldNames(), indexes
+                                        .get(j).getGramLength(), indexes.get(j).getIndexType());
+                        jobsToExecute
+                                .add(IndexOperations.buildSecondaryIndexCompactJobSpec(cics, metadataProvider, ds));
+                    }
+                }
+                jobsToExecute.add(ExternalIndexingOperations.compactFilesIndexJobSpec(ds, metadataProvider));
+            }
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
 
@@ -1720,9 +1991,9 @@
         boolean bActiveTxn = true;
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
         acquireReadLatch();
-
+        JobSpecification compiled = null;
         try {
-            JobSpecification compiled = rewriteCompileQuery(metadataProvider, query, null);
+            compiled = rewriteCompileQuery(metadataProvider, query, null);
 
             QueryResult queryResult = new QueryResult(query, metadataProvider.getResultSetId());
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1786,6 +2057,8 @@
             throw e;
         } finally {
             releaseReadLatch();
+            // release locks aquired during compilation of the query
+            ExternalDatasetsRegistry.INSTANCE.releaseAcquiredLocks(metadataProvider);
         }
     }
 
@@ -1819,6 +2092,232 @@
         }
     }
 
+    private void handleExternalDatasetRefreshStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+            IHyracksClientConnection hcc) throws Exception {
+        RefreshExternalDatasetStatement stmtRefresh = (RefreshExternalDatasetStatement) stmt;
+        ExternalDatasetTransactionState transactionState = ExternalDatasetTransactionState.COMMIT;
+        MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+        acquireWriteLatch();
+        boolean bActiveTxn = true;
+        metadataProvider.setMetadataTxnContext(mdTxnCtx);
+        String dataverseName = null;
+        String datasetName = null;
+        JobSpecification spec = null;
+        Dataset ds = null;
+        List<ExternalFile> metadataFiles = null;
+        List<ExternalFile> deletedFiles = null;
+        List<ExternalFile> addedFiles = null;
+        List<ExternalFile> appendedFiles = null;
+        List<Index> indexes = null;
+        Dataset transactionDataset = null;
+        boolean lockAquired = false;
+        boolean success = false;
+        try {
+            dataverseName = getActiveDataverseName(stmtRefresh.getDataverseName());
+            datasetName = stmtRefresh.getDatasetName().getValue();
+            ds = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
+                    datasetName);
+
+            // Dataset exists ?
+            if (ds == null) {
+                throw new AlgebricksException("There is no dataset with this name " + datasetName + " in dataverse "
+                        + dataverseName);
+            }
+            // Dataset external ?
+            if (ds.getDatasetType() != DatasetType.EXTERNAL) {
+                throw new AlgebricksException("dataset " + datasetName + " in dataverse " + dataverseName
+                        + " is not an external dataset");
+            }
+            // Dataset has indexes ?
+            indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
+            if (indexes.size() == 0) {
+                throw new AlgebricksException("External dataset " + datasetName + " in dataverse " + dataverseName
+                        + " doesn't have any index");
+            }
+
+            // Record transaction time
+            Date txnTime = new Date();
+
+            // refresh lock here
+            ExternalDatasetsRegistry.INSTANCE.refreshBegin(ds);
+            lockAquired = true;
+
+            // Get internal files
+            metadataFiles = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, ds);
+            deletedFiles = new ArrayList<ExternalFile>();
+            addedFiles = new ArrayList<ExternalFile>();
+            appendedFiles = new ArrayList<ExternalFile>();
+
+            // Compute delta
+            // Now we compare snapshot with external file system
+            if (ExternalIndexingOperations
+                    .isDatasetUptodate(ds, metadataFiles, addedFiles, deletedFiles, appendedFiles)) {
+                ((ExternalDatasetDetails) ds.getDatasetDetails()).setRefreshTimestamp(txnTime);
+                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                // latch will be released in the finally clause
+                return;
+            }
+
+            // At this point, we know data has changed in the external file system, record transaction in metadata and start            
+            transactionDataset = ExternalIndexingOperations.createTransactionDataset(ds);
+            /*
+             * Remove old dataset record and replace it with a new one
+             */
+            MetadataManager.INSTANCE.updateDataset(mdTxnCtx, transactionDataset);
+
+            // Add delta files to the metadata
+            for (ExternalFile file : addedFiles) {
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+            for (ExternalFile file : appendedFiles) {
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+            for (ExternalFile file : deletedFiles) {
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+
+            // Create the files index update job
+            spec = ExternalIndexingOperations.buildFilesIndexUpdateOp(ds, metadataFiles, deletedFiles, addedFiles,
+                    appendedFiles, metadataProvider);
+
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            bActiveTxn = false;
+            transactionState = ExternalDatasetTransactionState.BEGIN;
+
+            //run the files update job
+            runJob(hcc, spec, true);
+
+            for (Index index : indexes) {
+                if (!ExternalIndexingOperations.isFileIndex(index)) {
+                    spec = ExternalIndexingOperations.buildIndexUpdateOp(ds, index, metadataFiles, deletedFiles,
+                            addedFiles, appendedFiles, metadataProvider);
+                    //run the files update job
+                    runJob(hcc, spec, true);
+                }
+            }
+
+            // all index updates has completed successfully, record transaction state
+            spec = ExternalIndexingOperations.buildCommitJob(ds, indexes, metadataProvider);
+
+            // Aquire write latch again -> start a transaction and record the decision to commit
+            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+            metadataProvider.setMetadataTxnContext(mdTxnCtx);
+            bActiveTxn = true;
+            ((ExternalDatasetDetails) transactionDataset.getDatasetDetails())
+                    .setState(ExternalDatasetTransactionState.READY_TO_COMMIT);
+            ((ExternalDatasetDetails) transactionDataset.getDatasetDetails()).setRefreshTimestamp(txnTime);
+            MetadataManager.INSTANCE.updateDataset(mdTxnCtx, transactionDataset);
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            bActiveTxn = false;
+            transactionState = ExternalDatasetTransactionState.READY_TO_COMMIT;
+            // We don't release the latch since this job is expected to be quick
+            runJob(hcc, spec, true);
+            // Start a new metadata transaction to record the final state of the transaction
+            mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+            metadataProvider.setMetadataTxnContext(mdTxnCtx);
+            bActiveTxn = true;
+
+            for (ExternalFile file : metadataFiles) {
+                if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP) {
+                    MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP) {
+                    Iterator<ExternalFile> iterator = appendedFiles.iterator();
+                    while (iterator.hasNext()) {
+                        ExternalFile appendedFile = iterator.next();
+                        if (file.getFileName().equals(appendedFile.getFileName())) {
+                            // delete existing file
+                            MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                            // delete existing appended file
+                            MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, appendedFile);
+                            // add the original file with appended information
+                            appendedFile.setFileNumber(file.getFileNumber());
+                            appendedFile.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                            MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, appendedFile);
+                            iterator.remove();
+                        }
+                    }
+                }
+            }
+
+            // remove the deleted files delta
+            for (ExternalFile file : deletedFiles) {
+                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+            }
+
+            // insert new files
+            for (ExternalFile file : addedFiles) {
+                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                file.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+            }
+
+            // mark the transaction as complete
+            ((ExternalDatasetDetails) transactionDataset.getDatasetDetails())
+                    .setState(ExternalDatasetTransactionState.COMMIT);
+            MetadataManager.INSTANCE.updateDataset(mdTxnCtx, transactionDataset);
+
+            // commit metadata transaction
+            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+            success = true;
+        } catch (Exception e) {
+            if (bActiveTxn) {
+                abort(e, e, mdTxnCtx);
+            }
+            if (transactionState == ExternalDatasetTransactionState.READY_TO_COMMIT) {
+                throw new IllegalStateException("System is inconsistent state: commit of (" + dataverseName + "."
+                        + datasetName + ") refresh couldn't carry out the commit phase", e);
+            }
+            if (transactionState == ExternalDatasetTransactionState.COMMIT) {
+                // Nothing to do , everything should be clean
+                throw e;
+            }
+            if (transactionState == ExternalDatasetTransactionState.BEGIN) {
+                // transaction failed, need to do the following
+                // clean NCs removing transaction components
+                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                bActiveTxn = true;
+                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                spec = ExternalIndexingOperations.buildAbortOp(ds, indexes, metadataProvider);
+                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                bActiveTxn = false;
+                try {
+                    runJob(hcc, spec, true);
+                } catch (Exception e2) {
+                    // This should never happen -- fix throw illegal
+                    e.addSuppressed(e2);
+                    throw new IllegalStateException("System is in inconsistent state. Failed to abort refresh", e);
+                }
+                // remove the delta of files
+                // return the state of the dataset to committed
+                try {
+                    mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                    for (ExternalFile file : deletedFiles) {
+                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                    }
+                    for (ExternalFile file : addedFiles) {
+                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                    }
+                    for (ExternalFile file : appendedFiles) {
+                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                    }
+                    MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
+                    // commit metadata transaction
+                    MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                } catch (Exception e2) {
+                    abort(e, e2, mdTxnCtx);
+                    e.addSuppressed(e2);
+                    throw new IllegalStateException("System is in inconsistent state. Failed to drop delta files", e);
+                }
+            }
+        } finally {
+            releaseWriteLatch();
+            if (lockAquired) {
+                ExternalDatasetsRegistry.INSTANCE.refreshEnd(ds, success);
+            }
+        }
+    }
+
     private JobId runJob(IHyracksClientConnection hcc, JobSpecification spec, boolean waitForCompletion)
             throws Exception {
         spec.setFrameSize(OptimizationConfUtil.getPhysicalOptimizationConfig().getFrameSize());

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index ac0b919..19283ff 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java

@@ -26,6 +26,7 @@
 import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
 import edu.uci.ics.asterix.common.config.GlobalConfig;
 import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -123,7 +124,7 @@
         JobSpecification specPrimary = JobSpecificationUtils.createJobSpecification();
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataset.getDataverseName(), datasetName,
+                .splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), datasetName,
                         datasetName);
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
@@ -164,7 +165,7 @@
         int[] blooFilterKeyFields = DatasetUtils.createBloomFilterKeyFields(dataset);
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
         FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
         StringBuilder sb = new StringBuilder();
         for (int i = 0; i < fs.length; i++) {
@@ -225,10 +226,10 @@
         int[] blooFilterKeyFields = DatasetUtils.createBloomFilterKeyFields(dataset);
 
         ExternalDatasetDetails externalDatasetDetails = new ExternalDatasetDetails(loadStmt.getAdapter(),
-                loadStmt.getProperties());
+                loadStmt.getProperties(), null, null, ExternalDatasetTransactionState.COMMIT, null, null);
 
         Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = metadataProvider.buildExternalDataScannerRuntime(
-                spec, itemType, externalDatasetDetails, format);
+                spec, itemType, externalDatasetDetails, format, dataset);
         IOperatorDescriptor scanner = p.first;
         AlgebricksPartitionConstraint scannerPc = p.second;
         RecordDescriptor recDesc = computePayloadKeyRecordDescriptor(dataset, itemType, payloadSerde, format);
@@ -253,7 +254,7 @@
         fieldPermutation[numKeys] = 0;
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
 
         FileSplit[] fs = splitsAndConstraint.first.getFileSplits();
         StringBuilder sb = new StringBuilder();
@@ -389,7 +390,7 @@
         int[] blooFilterKeyFields = DatasetUtils.createBloomFilterKeyFields(dataset);
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
 
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
 

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java
new file mode 100644
index 0000000..bcd9ff9
--- /dev/null
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/ExternalIndexingOperations.java

@@ -0,0 +1,750 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.file;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMRTreeIOOperationCallbackFactory;
+import edu.uci.ics.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
+import edu.uci.ics.asterix.external.adapter.factory.HDFSAdapterFactory;
+import edu.uci.ics.asterix.external.adapter.factory.HDFSIndexingAdapterFactory;
+import edu.uci.ics.asterix.external.adapter.factory.HiveAdapterFactory;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalDatasetIndexesAbortOperatorDescriptor;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalDatasetIndexesCommitOperatorDescriptor;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalDatasetIndexesRecoverOperatorDescriptor;
+import edu.uci.ics.asterix.external.indexing.operators.IndexInfoOperatorDescriptor;
+import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
+import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.external.FilesIndexDescription;
+import edu.uci.ics.asterix.metadata.external.IndexingConstants;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
+import edu.uci.ics.asterix.tools.external.data.ExternalFilesIndexOperatorDescriptor;
+import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
+import edu.uci.ics.asterix.transaction.management.resource.ExternalBTreeLocalResourceMetadata;
+import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
+import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
+import edu.uci.ics.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
+import edu.uci.ics.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.jobgen.impl.ConnectorPolicyAssignmentPolicy;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import edu.uci.ics.hyracks.storage.common.file.LocalResource;
+
+public class ExternalIndexingOperations {
+
+    public static final ArrayList<String> FILE_INDEX_FIELDS = new ArrayList<String>();
+    static {
+        FILE_INDEX_FIELDS.add("");
+    }
+
+    public static boolean isIndexible(ExternalDatasetDetails ds) {
+        String adapter = ds.getAdapter();
+        if (adapter.equalsIgnoreCase("hdfs") || adapter.equalsIgnoreCase("hive")
+                || adapter.equalsIgnoreCase("edu.uci.ics.asterix.external.dataset.adapter.HDFSAdapter")
+                || adapter.equalsIgnoreCase("edu.uci.ics.asterix.external.dataset.adapter.HIVEAdapter")) {
+            return true;
+        }
+        return false;
+    }
+
+    public static boolean isRefereshActive(ExternalDatasetDetails ds) {
+        return ds.getState() != ExternalDatasetTransactionState.COMMIT;
+    }
+
+    public static boolean datasetUsesHiveAdapter(ExternalDatasetDetails ds) {
+        String adapter = ds.getAdapter();
+        return (adapter.equalsIgnoreCase("hive") || adapter
+                .equalsIgnoreCase("edu.uci.ics.asterix.external.dataset.adapter.HIVEAdapter"));
+    }
+
+    public static boolean isValidIndexName(String datasetName, String indexName) {
+        return (!datasetName.concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX).equals(indexName));
+    }
+
+    public static String getFilesIndexName(String datasetName) {
+        return datasetName.concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX);
+    }
+
+    public static int getRIDSize(Dataset dataset) {
+        ExternalDatasetDetails dsd = ((ExternalDatasetDetails) dataset.getDatasetDetails());
+        return IndexingConstants.getRIDSize(dsd.getProperties().get(IndexingConstants.KEY_INPUT_FORMAT));
+    }
+
+    public static IBinaryComparatorFactory[] getComparatorFactories(Dataset dataset) {
+        ExternalDatasetDetails dsd = ((ExternalDatasetDetails) dataset.getDatasetDetails());
+        return IndexingConstants.getComparatorFactories((dsd.getProperties().get(IndexingConstants.KEY_INPUT_FORMAT)));
+    }
+
+    public static IBinaryComparatorFactory[] getBuddyBtreeComparatorFactories() {
+        return IndexingConstants.getBuddyBtreeComparatorFactories();
+    }
+
+    public static ArrayList<ExternalFile> getSnapshotFromExternalFileSystem(Dataset dataset) throws AlgebricksException {
+        ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
+        ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
+        try {
+            // Create the file system object
+            FileSystem fs = getFileSystemObject(datasetDetails.getProperties());
+            // If dataset uses hive adapter, add path to the dataset properties
+            if (datasetUsesHiveAdapter(datasetDetails)) {
+                HiveAdapterFactory.populateConfiguration(datasetDetails.getProperties());
+            }
+            // Get paths of dataset
+            String path = datasetDetails.getProperties().get(HDFSAdapterFactory.KEY_PATH);
+            String[] paths = path.split(",");
+
+            // Add fileStatuses to files
+            for (String aPath : paths) {
+                FileStatus[] fileStatuses = fs.listStatus(new Path(aPath));
+                for (int i = 0; i < fileStatuses.length; i++) {
+                    int nextFileNumber = files.size();
+                    if (fileStatuses[i].isDir()) {
+                        listSubFiles(dataset, fs, fileStatuses[i], files);
+                    } else {
+                        files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(),
+                                nextFileNumber, fileStatuses[i].getPath().toUri().getPath(), new Date(fileStatuses[i]
+                                        .getModificationTime()), fileStatuses[i].getLen(),
+                                ExternalFilePendingOp.PENDING_NO_OP));
+                    }
+                }
+            }
+            // Close file system
+            fs.close();
+            if (files.size() == 0) {
+                throw new AlgebricksException("File Snapshot retrieved from external file system is empty");
+            }
+            return files;
+        } catch (Exception e) {
+            e.printStackTrace();
+            throw new AlgebricksException("Unable to get list of HDFS files " + e);
+        }
+    }
+
+    /* list all files under the directory
+     * src is expected to be a folder
+     */
+    private static void listSubFiles(Dataset dataset, FileSystem srcFs, FileStatus src, ArrayList<ExternalFile> files)
+            throws IOException {
+        Path path = src.getPath();
+        FileStatus[] fileStatuses = srcFs.listStatus(path);
+        for (int i = 0; i < fileStatuses.length; i++) {
+            int nextFileNumber = files.size();
+            if (fileStatuses[i].isDir()) {
+                listSubFiles(dataset, srcFs, fileStatuses[i], files);
+            } else {
+                files.add(new ExternalFile(dataset.getDataverseName(), dataset.getDatasetName(), nextFileNumber,
+                        fileStatuses[i].getPath().toUri().getPath(), new Date(fileStatuses[i].getModificationTime()),
+                        fileStatuses[i].getLen(), ExternalFilePendingOp.PENDING_NO_OP));
+            }
+        }
+    }
+
+    public static FileSystem getFileSystemObject(Map<String, String> map) throws IOException {
+        Configuration conf = new Configuration();
+        conf.set("fs.default.name", ((String) map.get(HDFSAdapterFactory.KEY_HDFS_URL)).trim());
+        conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
+        return FileSystem.get(conf);
+    }
+
+    public static JobSpecification buildFilesIndexReplicationJobSpec(Dataset dataset,
+            ArrayList<ExternalFile> externalFilesSnapshot, AqlMetadataProvider metadataProvider, boolean createIndex)
+            throws MetadataException, AlgebricksException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(),
+                        dataset.getDatasetName(), getFilesIndexName(dataset.getDatasetName()), true);
+        IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
+        ILocalResourceMetadata localResourceMetadata = new ExternalBTreeLocalResourceMetadata(
+                FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
+                FilesIndexDescription.FILES_INDEX_COMP_FACTORIES, new int[] { 0 }, false, dataset.getDatasetId(),
+                mergePolicyFactory, mergePolicyFactoryProperties);
+        PersistentLocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
+                localResourceMetadata, LocalResource.ExternalBTreeResource);
+        ExternalBTreeDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeDataflowHelperFactory(
+                mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                        dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        ExternalFilesIndexOperatorDescriptor externalFilesOp = new ExternalFilesIndexOperatorDescriptor(spec,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                secondaryFileSplitProvider, indexDataflowHelperFactory, localResourceFactoryProvider,
+                externalFilesSnapshot, createIndex);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, externalFilesOp,
+                secondarySplitsAndConstraint.second);
+        spec.addRoot(externalFilesOp);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+
+    /**
+     * This method create an indexing operator that index records in HDFS
+     * 
+     * @param jobSpec
+     * @param itemType
+     * @param dataset
+     * @param format
+     * @param files
+     * @param indexerDesc
+     * @return
+     * @throws Exception
+     */
+    private static Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> getExternalDataIndexingOperator(
+            JobSpecification jobSpec, IAType itemType, Dataset dataset, List<ExternalFile> files,
+            RecordDescriptor indexerDesc, AqlMetadataProvider metadataProvider) throws Exception {
+        HDFSIndexingAdapterFactory adapterFactory = new HDFSIndexingAdapterFactory();
+        adapterFactory.setFiles(files);
+        adapterFactory.configure(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties(),
+                (ARecordType) itemType);
+        return new Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint>(
+                new ExternalDataScanOperatorDescriptor(jobSpec, indexerDesc, adapterFactory),
+                adapterFactory.getPartitionConstraint());
+    }
+
+    public static Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> createExternalIndexingOp(
+            JobSpecification spec, AqlMetadataProvider metadataProvider, Dataset dataset, ARecordType itemType,
+            RecordDescriptor indexerDesc, List<ExternalFile> files) throws Exception {
+        if (files == null) {
+            files = MetadataManager.INSTANCE.getDatasetExternalFiles(metadataProvider.getMetadataTxnContext(), dataset);
+        }
+        return getExternalDataIndexingOperator(spec, itemType, dataset, files, indexerDesc, metadataProvider);
+    }
+
+    /**
+     * At the end of this method, we expect to have 4 sets as follows:
+     * metadataFiles should contain only the files that are appended in their original state
+     * addedFiles should contain new files that has number assigned starting after the max original file number
+     * deleteedFiles should contain files that are no longer there in the file system
+     * appendedFiles should have the new file information of existing files
+     * The method should return false in case of zero delta
+     * 
+     * @param dataset
+     * @param metadataFiles
+     * @param addedFiles
+     * @param deletedFiles
+     * @param appendedFiles
+     * @return
+     * @throws MetadataException
+     * @throws AlgebricksException
+     */
+    public static boolean isDatasetUptodate(Dataset dataset, List<ExternalFile> metadataFiles,
+            List<ExternalFile> addedFiles, List<ExternalFile> deletedFiles, List<ExternalFile> appendedFiles)
+            throws MetadataException, AlgebricksException {
+        boolean uptodate = true;
+        int newFileNumber = metadataFiles.get(metadataFiles.size() - 1).getFileNumber() + 1;
+
+        ArrayList<ExternalFile> fileSystemFiles = getSnapshotFromExternalFileSystem(dataset);
+
+        // Loop over file system files < taking care of added files >
+        for (ExternalFile fileSystemFile : fileSystemFiles) {
+            boolean fileFound = false;
+            Iterator<ExternalFile> mdFilesIterator = metadataFiles.iterator();
+            while (mdFilesIterator.hasNext()) {
+                ExternalFile metadataFile = mdFilesIterator.next();
+                if (fileSystemFile.getFileName().equals(metadataFile.getFileName())) {
+                    // Same file name
+                    if (fileSystemFile.getLastModefiedTime().equals(metadataFile.getLastModefiedTime())) {
+                        // Same timestamp
+                        if (fileSystemFile.getSize() == metadataFile.getSize()) {
+                            // Same size -> no op
+                            mdFilesIterator.remove();
+                            fileFound = true;
+                        } else {
+                            // Different size -> append op
+                            metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_APPEND_OP);
+                            fileSystemFile.setPendingOp(ExternalFilePendingOp.PENDING_APPEND_OP);
+                            appendedFiles.add(fileSystemFile);
+                            fileFound = true;
+                            uptodate = false;
+                        }
+                    } else {
+                        // Same file name, Different file mod date -> delete and add
+                        metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_DROP_OP);
+                        deletedFiles.add(new ExternalFile(metadataFile.getDataverseName(), metadataFile
+                                .getDatasetName(), 0, metadataFile.getFileName(), metadataFile.getLastModefiedTime(),
+                                metadataFile.getSize(), ExternalFilePendingOp.PENDING_DROP_OP));
+                        fileSystemFile.setPendingOp(ExternalFilePendingOp.PENDING_ADD_OP);
+                        fileSystemFile.setFileNumber(newFileNumber);
+                        addedFiles.add(fileSystemFile);
+                        newFileNumber++;
+                        fileFound = true;
+                        uptodate = false;
+                    }
+                }
+                if (fileFound)
+                    break;
+            }
+            if (!fileFound) {
+                // File not stored previously in metadata -> pending add op
+                fileSystemFile.setPendingOp(ExternalFilePendingOp.PENDING_ADD_OP);
+                fileSystemFile.setFileNumber(newFileNumber);
+                addedFiles.add(fileSystemFile);
+                newFileNumber++;
+                uptodate = false;
+            }
+        }
+
+        // Done with files from external file system -> metadata files now contain both deleted files and appended ones
+        // first, correct number assignment to deleted and updated files
+        for (ExternalFile deletedFile : deletedFiles) {
+            deletedFile.setFileNumber(newFileNumber);
+            newFileNumber++;
+        }
+        for (ExternalFile appendedFile : appendedFiles) {
+            appendedFile.setFileNumber(newFileNumber);
+            newFileNumber++;
+        }
+
+        // include the remaining deleted files
+        Iterator<ExternalFile> mdFilesIterator = metadataFiles.iterator();
+        while (mdFilesIterator.hasNext()) {
+            ExternalFile metadataFile = mdFilesIterator.next();
+            if (metadataFile.getPendingOp() == ExternalFilePendingOp.PENDING_NO_OP) {
+                metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_DROP_OP);
+                deletedFiles.add(new ExternalFile(metadataFile.getDataverseName(), metadataFile.getDatasetName(),
+                        newFileNumber, metadataFile.getFileName(), metadataFile.getLastModefiedTime(), metadataFile
+                                .getSize(), metadataFile.getPendingOp()));
+                newFileNumber++;
+                uptodate = false;
+            }
+        }
+        return uptodate;
+    }
+
+    public static Dataset createTransactionDataset(Dataset dataset) {
+        ExternalDatasetDetails originalDsd = (ExternalDatasetDetails) dataset.getDatasetDetails();
+        ExternalDatasetDetails dsd = new ExternalDatasetDetails(originalDsd.getAdapter(), originalDsd.getProperties(),
+                originalDsd.getNodeGroupName(), originalDsd.getTimestamp(), ExternalDatasetTransactionState.BEGIN,
+                originalDsd.getCompactionPolicy(), originalDsd.getCompactionPolicyProperties());
+        Dataset transactionDatset = new Dataset(dataset.getDataverseName(), dataset.getDatasetName(),
+                dataset.getItemTypeName(), dsd, dataset.getHints(), DatasetType.EXTERNAL, dataset.getDatasetId(),
+                dataset.getPendingOp());
+        return transactionDatset;
+    }
+
+    public static boolean isFileIndex(Index index) {
+        return (index.getIndexName().equals(getFilesIndexName(index.getDatasetName())));
+    }
+
+    public static JobSpecification buildDropFilesIndexJobSpec(CompiledIndexDropStatement indexDropStmt,
+            AqlMetadataProvider metadataProvider, Dataset dataset) throws AlgebricksException, MetadataException {
+        String dataverseName = indexDropStmt.getDataverseName() == null ? metadataProvider.getDefaultDataverseName()
+                : indexDropStmt.getDataverseName();
+        String datasetName = indexDropStmt.getDatasetName();
+        String indexName = indexDropStmt.getIndexName();
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForFilesIndex(dataverseName, datasetName, indexName, true);
+        AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
+                metadataProvider.getMetadataTxnContext());
+        IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
+                        dataset.getDatasetId()), compactionInfo.first, compactionInfo.second,
+                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+                        storageProperties.getBloomFilterFalsePositiveRate(), false));
+        AlgebricksPartitionConstraintHelper
+                .setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
+        spec.addRoot(btreeDrop);
+
+        return spec;
+    }
+
+    public static JobSpecification buildFilesIndexUpdateOp(Dataset ds, List<ExternalFile> metadataFiles,
+            List<ExternalFile> deletedFiles, List<ExternalFile> addedFiles, List<ExternalFile> appendedFiles,
+            AqlMetadataProvider metadataProvider) throws MetadataException, AlgebricksException {
+        ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
+        for (ExternalFile file : metadataFiles) {
+            if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP)
+                files.add(file);
+            else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_APPEND_OP) {
+                for (ExternalFile appendedFile : appendedFiles) {
+                    if (appendedFile.getFileName().equals(file.getFileName())) {
+                        files.add(new ExternalFile(file.getDataverseName(), file.getDatasetName(),
+                                file.getFileNumber(), file.getFileName(), file.getLastModefiedTime(), appendedFile
+                                        .getSize(), ExternalFilePendingOp.PENDING_NO_OP));
+                    }
+                }
+            }
+        }
+        for (ExternalFile file : addedFiles) {
+            files.add(file);
+        }
+        Collections.sort(files);
+        return buildFilesIndexReplicationJobSpec(ds, files, metadataProvider, false);
+    }
+
+    public static JobSpecification buildIndexUpdateOp(Dataset ds, Index index, List<ExternalFile> metadataFiles,
+            List<ExternalFile> deletedFiles, List<ExternalFile> addedFiles, List<ExternalFile> appendedFiles,
+            AqlMetadataProvider metadataProvider) throws AsterixException, AlgebricksException {
+        // Create files list
+        ArrayList<ExternalFile> files = new ArrayList<ExternalFile>();
+
+        for (ExternalFile metadataFile : metadataFiles) {
+            if (metadataFile.getPendingOp() != ExternalFilePendingOp.PENDING_APPEND_OP) {
+                files.add(metadataFile);
+            } else {
+                metadataFile.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                files.add(metadataFile);
+            }
+        }
+        // add new files
+        for (ExternalFile file : addedFiles) {
+            files.add(file);
+        }
+        // add appended files
+        for (ExternalFile file : appendedFiles) {
+            files.add(file);
+        }
+
+        CompiledCreateIndexStatement ccis = new CompiledCreateIndexStatement(index.getIndexName(),
+                index.getDataverseName(), index.getDatasetName(), index.getKeyFieldNames(), index.getGramLength(),
+                index.getIndexType());
+        return IndexOperations.buildSecondaryIndexLoadingJobSpec(ccis, metadataProvider, files);
+    }
+
+    public static JobSpecification buildCommitJob(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
+            throws AlgebricksException, AsterixException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                        getFilesIndexName(ds.getDatasetName()));
+        IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
+                mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
+        IndexInfoOperatorDescriptor filesIndexInfo = new IndexInfoOperatorDescriptor(filesIndexSplitProvider,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
+
+        ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory> btreeDataflowHelperFactories = new ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> btreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+        ArrayList<ExternalRTreeDataflowHelperFactory> rtreeDataflowHelperFactories = new ArrayList<ExternalRTreeDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> rtreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+
+        for (Index index : indexes) {
+            if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
+                Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
+                        .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                                index.getIndexName());
+                if (index.getIndexType() == IndexType.BTREE) {
+                    btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, spec));
+                    btreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                } else if (index.getIndexType() == IndexType.RTREE) {
+                    rtreeDataflowHelperFactories.add(getRTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, metadataProvider, spec));
+                    rtreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                }
+            }
+        }
+
+        ExternalDatasetIndexesCommitOperatorDescriptor op = new ExternalDatasetIndexesCommitOperatorDescriptor(spec,
+                filesIndexDataflowHelperFactory, filesIndexInfo, btreeDataflowHelperFactories, btreeInfos,
+                rtreeDataflowHelperFactories, rtreeInfos);
+
+        spec.addRoot(op);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op,
+                filesIndexSplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+
+    private static ExternalBTreeDataflowHelperFactory getFilesIndexDataflowHelperFactory(Dataset ds,
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
+            AsterixStorageProperties storageProperties, JobSpecification spec) {
+        return new ExternalBTreeDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
+                new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+                storageProperties.getBloomFilterFalsePositiveRate(),
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+    }
+
+    private static ExternalBTreeWithBuddyDataflowHelperFactory getBTreeDataflowHelperFactory(Dataset ds, Index index,
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
+            AsterixStorageProperties storageProperties, JobSpecification spec) {
+        return new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
+                new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+                storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+    }
+
+    @SuppressWarnings("rawtypes")
+    private static ExternalRTreeDataflowHelperFactory getRTreeDataflowHelperFactory(Dataset ds, Index index,
+            ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyFactoryProperties,
+            AsterixStorageProperties storageProperties, AqlMetadataProvider metadataProvider, JobSpecification spec)
+            throws AlgebricksException, AsterixException {
+        int numPrimaryKeys = getRIDSize(ds);
+        List<String> secondaryKeyFields = index.getKeyFieldNames();
+        secondaryKeyFields.size();
+        ARecordType itemType = (ARecordType) metadataProvider.findType(ds.getDataverseName(), ds.getItemTypeName());
+        Pair<IAType, Boolean> spatialTypePair = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0), itemType);
+        IAType spatialType = spatialTypePair.first;
+        if (spatialType == null) {
+            throw new AsterixException("Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
+        }
+        int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
+        int numNestedSecondaryKeyFields = numDimensions * 2;
+        IPrimitiveValueProviderFactory[] valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
+        IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
+
+        ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys
+                + numNestedSecondaryKeyFields];
+        ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numNestedSecondaryKeyFields + numPrimaryKeys];
+        IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(spatialType.getTypeTag());
+        ATypeTag keyType = nestedKeyType.getTypeTag();
+
+        keyType = nestedKeyType.getTypeTag();
+        for (int i = 0; i < numNestedSecondaryKeyFields; i++) {
+            ISerializerDeserializer keySerde = AqlSerializerDeserializerProvider.INSTANCE
+                    .getSerializerDeserializer(nestedKeyType);
+            secondaryRecFields[i] = keySerde;
+
+            secondaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(
+                    nestedKeyType, true);
+            secondaryTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
+            valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
+        }
+        // Add serializers and comparators for primary index fields.
+        for (int i = 0; i < numPrimaryKeys; i++) {
+            secondaryRecFields[numNestedSecondaryKeyFields + i] = IndexingConstants.getSerializerDeserializer(i);
+            secondaryTypeTraits[numNestedSecondaryKeyFields + i] = IndexingConstants.getTypeTraits(i);
+        }
+        int[] primaryKeyFields = new int[numPrimaryKeys];
+        for (int i = 0; i < primaryKeyFields.length; i++) {
+            primaryKeyFields[i] = i + numNestedSecondaryKeyFields;
+        }
+
+        return new ExternalRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                getBuddyBtreeComparatorFactories(), mergePolicyFactory, mergePolicyFactoryProperties,
+                new SecondaryIndexOperationTrackerProvider(ds.getDatasetId()),
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
+                AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+                storageProperties.getBloomFilterFalsePositiveRate(), new int[] { index.getKeyFieldNames().size() },
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(ds));
+    }
+
+    public static JobSpecification buildAbortOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
+            throws AlgebricksException, AsterixException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                        getFilesIndexName(ds.getDatasetName()));
+        IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
+                mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
+        IndexInfoOperatorDescriptor filesIndexInfo = new IndexInfoOperatorDescriptor(filesIndexSplitProvider,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
+
+        ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory> btreeDataflowHelperFactories = new ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> btreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+        ArrayList<ExternalRTreeDataflowHelperFactory> rtreeDataflowHelperFactories = new ArrayList<ExternalRTreeDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> rtreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+
+        for (Index index : indexes) {
+            if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
+                Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
+                        .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                                index.getIndexName());
+                if (index.getIndexType() == IndexType.BTREE) {
+                    btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, spec));
+                    btreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                } else if (index.getIndexType() == IndexType.RTREE) {
+                    rtreeDataflowHelperFactories.add(getRTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, metadataProvider, spec));
+                    rtreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                }
+            }
+        }
+
+        ExternalDatasetIndexesAbortOperatorDescriptor op = new ExternalDatasetIndexesAbortOperatorDescriptor(spec,
+                filesIndexDataflowHelperFactory, filesIndexInfo, btreeDataflowHelperFactories, btreeInfos,
+                rtreeDataflowHelperFactories, rtreeInfos);
+
+        spec.addRoot(op);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op,
+                filesIndexSplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+
+    }
+
+    public static JobSpecification buildRecoverOp(Dataset ds, List<Index> indexes, AqlMetadataProvider metadataProvider)
+            throws AlgebricksException, AsterixException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(ds,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> filesIndexSplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                        getFilesIndexName(ds.getDatasetName()));
+        IFileSplitProvider filesIndexSplitProvider = filesIndexSplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory = getFilesIndexDataflowHelperFactory(ds,
+                mergePolicyFactory, mergePolicyFactoryProperties, storageProperties, spec);
+        IndexInfoOperatorDescriptor filesIndexInfo = new IndexInfoOperatorDescriptor(filesIndexSplitProvider,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
+
+        ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory> btreeDataflowHelperFactories = new ArrayList<ExternalBTreeWithBuddyDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> btreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+        ArrayList<ExternalRTreeDataflowHelperFactory> rtreeDataflowHelperFactories = new ArrayList<ExternalRTreeDataflowHelperFactory>();
+        ArrayList<IndexInfoOperatorDescriptor> rtreeInfos = new ArrayList<IndexInfoOperatorDescriptor>();
+
+        for (Index index : indexes) {
+            if (isValidIndexName(index.getDatasetName(), index.getIndexName())) {
+                Pair<IFileSplitProvider, AlgebricksPartitionConstraint> indexSplitsAndConstraint = metadataProvider
+                        .splitProviderAndPartitionConstraintsForDataset(ds.getDataverseName(), ds.getDatasetName(),
+                                index.getIndexName());
+                if (index.getIndexType() == IndexType.BTREE) {
+                    btreeDataflowHelperFactories.add(getBTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, spec));
+                    btreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                } else if (index.getIndexType() == IndexType.RTREE) {
+                    rtreeDataflowHelperFactories.add(getRTreeDataflowHelperFactory(ds, index, mergePolicyFactory,
+                            mergePolicyFactoryProperties, storageProperties, metadataProvider, spec));
+                    rtreeInfos.add(new IndexInfoOperatorDescriptor(indexSplitsAndConstraint.first,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER));
+                }
+            }
+        }
+
+        ExternalDatasetIndexesRecoverOperatorDescriptor op = new ExternalDatasetIndexesRecoverOperatorDescriptor(spec,
+                filesIndexDataflowHelperFactory, filesIndexInfo, btreeDataflowHelperFactories, btreeInfos,
+                rtreeDataflowHelperFactories, rtreeInfos);
+
+        spec.addRoot(op);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, op,
+                filesIndexSplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+
+    public static JobSpecification compactFilesIndexJobSpec(Dataset dataset, AqlMetadataProvider metadataProvider)
+            throws MetadataException, AlgebricksException {
+        JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
+        AsterixStorageProperties storageProperties = asterixPropertiesProvider.getStorageProperties();
+        Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
+                metadataProvider.getMetadataTxnContext());
+        ILSMMergePolicyFactory mergePolicyFactory = compactionInfo.first;
+        Map<String, String> mergePolicyFactoryProperties = compactionInfo.second;
+        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
+                .splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(),
+                        dataset.getDatasetName(), getFilesIndexName(dataset.getDatasetName()), true);
+        IFileSplitProvider secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
+        ExternalBTreeDataflowHelperFactory indexDataflowHelperFactory = new ExternalBTreeDataflowHelperFactory(
+                mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                        dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties.getBloomFilterFalsePositiveRate(),
+                ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                secondaryFileSplitProvider, FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS,
+                FilesIndexDescription.FILES_INDEX_COMP_FACTORIES, new int[] { 0 }, indexDataflowHelperFactory,
+                NoOpOperationCallbackFactory.INSTANCE);
+        spec.addRoot(compactOp);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
+                secondarySplitsAndConstraint.second);
+        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+        return spec;
+    }
+}

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index 5e3f7de..859e39c 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java

@@ -14,6 +14,7 @@
  */
 package edu.uci.ics.asterix.file;
 
+import java.util.List;
 import java.util.Map;
 
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
@@ -24,6 +25,7 @@
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
@@ -66,6 +68,17 @@
                         physicalOptimizationConfig);
         return secondaryIndexHelper.buildLoadingJobSpec();
     }
+    
+    public static JobSpecification buildSecondaryIndexLoadingJobSpec(CompiledCreateIndexStatement createIndexStmt,
+            AqlMetadataProvider metadataProvider, List<ExternalFile> files) throws AsterixException, AlgebricksException {
+        SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper
+                .createIndexOperationsHelper(createIndexStmt.getIndexType(), createIndexStmt.getDataverseName(),
+                        createIndexStmt.getDatasetName(), createIndexStmt.getIndexName(),
+                        createIndexStmt.getKeyFields(), createIndexStmt.getGramLength(), metadataProvider,
+                        physicalOptimizationConfig);
+        secondaryIndexHelper.setExternalFiles(files);
+        return secondaryIndexHelper.buildLoadingJobSpec();
+    }
 
     public static JobSpecification buildDropSecondaryIndexJobSpec(CompiledIndexDropStatement indexDropStmt,
             AqlMetadataProvider metadataProvider, Dataset dataset) throws AlgebricksException, MetadataException {
@@ -76,7 +89,7 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, indexName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName);
         AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
                 metadataProvider.getMetadataTxnContext());

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
index 47b2567..597e835 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeOperationsHelper.java

@@ -16,12 +16,17 @@
 
 import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
 import edu.uci.ics.asterix.common.exceptions.AsterixException;
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
+import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeWithBuddyIOOperationCallbackFactory;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
 import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
 import edu.uci.ics.asterix.transaction.management.resource.LSMBTreeLocalResourceMetadata;
+import edu.uci.ics.asterix.transaction.management.resource.ExternalBTreeWithBuddyLocalResourceMetadata;
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
 import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
@@ -35,9 +40,12 @@
 import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
+import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
@@ -55,21 +63,39 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
-        ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
-                secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields, true,
-                dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties);
-        ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
-                localResourceMetadata, LocalResource.LSMBTreeResource);
-
+        ILocalResourceFactoryProvider localResourceFactoryProvider;
+        IIndexDataflowHelperFactory indexDataflowHelperFactory;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
+            ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    true, dataset.getDatasetId(), mergePolicyFactory, mergePolicyFactoryProperties);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.LSMBTreeResource);
+            indexDataflowHelperFactory = new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
+                    dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
+                    new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
+                    storageProperties.getBloomFilterFalsePositiveRate(), false);
+        } else {
+            // External dataset local resource and dataflow helper
+            int[] buddyBreeFields = new int[] { numSecondaryKeys };
+            ILocalResourceMetadata localResourceMetadata = new ExternalBTreeWithBuddyLocalResourceMetadata(
+                    dataset.getDatasetId(), secondaryComparatorFactories, secondaryRecDesc.getTypeTraits(),
+                    mergePolicyFactory, mergePolicyFactoryProperties, buddyBreeFields);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.ExternalBTreeWithBuddyResource);
+            indexDataflowHelperFactory = new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory,
+                    mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+                    storageProperties.getBloomFilterFalsePositiveRate(), buddyBreeFields,
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        }
         TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
-                secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
-                        dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), false), localResourceFactoryProvider,
+                secondaryBloomFilterKeyFields, indexDataflowHelperFactory, localResourceFactoryProvider,
                 NoOpOperationCallbackFactory.INSTANCE);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
                 secondaryPartitionConstraint);
@@ -81,49 +107,100 @@
     @Override
     public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+            /*
+             * In case of external data, this method is used to build loading jobs for both initial load on index creation
+             * and transaction load on dataset referesh
+             */
+            // Create external indexing scan operator
+            ExternalDataScanOperatorDescriptor primaryScanOp = createExternalIndexingOp(spec);
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec, numSecondaryKeys);
 
-        // Create dummy key provider for feeding the primary index scan. 
-        AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+            }
 
-        // Create primary index scan op.
-        BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
 
-        // Assign op.
-        AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp, numSecondaryKeys);
-
-        // If any of the secondary fields are nullable, then add a select op that filters nulls.
-        AlgebricksMetaOperatorDescriptor selectOp = null;
-        if (anySecondaryKeyIsNullable) {
-            selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
-        }
-
-        // Sort by secondary keys.
-        ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
-
-        AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        // Create secondary BTree bulk load op.
-        TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
-                spec,
-                numSecondaryKeys,
-                new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
-                        mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
-                                dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                        LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
-                                .getBloomFilterFalsePositiveRate(), false), BTree.DEFAULT_FILL_FACTOR);
-
-        // Connect the operators.
-        spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
-        spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
-        if (anySecondaryKeyIsNullable) {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
-            spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+            // Create secondary BTree bulk load op.
+            AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
+            ExternalBTreeWithBuddyDataflowHelperFactory dataflowHelperFactory = new ExternalBTreeWithBuddyDataflowHelperFactory(
+                    mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                            dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+                    storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+            if (externalFiles != null) {
+                // Transaction load
+                secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, numSecondaryKeys, dataflowHelperFactory,
+                        BTree.DEFAULT_FILL_FACTOR);
+            } else {
+                // Initial load
+                secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numSecondaryKeys, dataflowHelperFactory,
+                        BTree.DEFAULT_FILL_FACTOR);
+            }
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+            return spec;
         } else {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            // Create dummy key provider for feeding the primary index scan. 
+            AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
+
+            // Create primary index scan op.
+            BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
+
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp, numSecondaryKeys);
+
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+            }
+
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+            // Create secondary BTree bulk load op.
+            TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
+                    spec,
+                    numSecondaryKeys,
+                    new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
+                                    .getBloomFilterFalsePositiveRate(), false), BTree.DEFAULT_FILL_FACTOR);
+
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
+            return spec;
         }
-        spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
-        spec.addRoot(secondaryBulkLoadOp);
-        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
-        return spec;
     }
 
     @Override
@@ -131,15 +208,32 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
-                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
-                secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(
-                        dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), false),
-                NoOpOperationCallbackFactory.INSTANCE);
+        LSMTreeIndexCompactOperatorDescriptor compactOp;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
+                                    .getBloomFilterFalsePositiveRate(), false), NoOpOperationCallbackFactory.INSTANCE);
+        } else {
+            // External dataset
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new ExternalBTreeWithBuddyDataflowHelperFactory(mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE, storageProperties
+                                    .getBloomFilterFalsePositiveRate(), new int[] { numSecondaryKeys },
+                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+                    NoOpOperationCallbackFactory.INSTANCE);
+        }
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
                 secondaryPartitionConstraint);
         spec.addRoot(compactOp);

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
index ca049a6..851de4a 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexOperationsHelper.java

@@ -22,6 +22,7 @@
 
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
 import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
 import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -31,6 +32,7 @@
 import edu.uci.ics.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
 import edu.uci.ics.asterix.common.transactions.IRecoveryManager.ResourceType;
 import edu.uci.ics.asterix.common.transactions.JobId;
+import edu.uci.ics.asterix.external.indexing.operators.ExternalIndexBulkModifyOperatorDescriptor;
 import edu.uci.ics.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
 import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
 import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
@@ -38,7 +40,10 @@
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
 import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.external.IndexingConstants;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
 import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.IAType;
@@ -123,6 +128,8 @@
     protected ILSMMergePolicyFactory mergePolicyFactory;
     protected Map<String, String> mergePolicyFactoryProperties;
 
+    protected List<ExternalFile> externalFiles;
+
     // Prevent public construction. Should be created via createIndexCreator().
     protected SecondaryIndexOperationsHelper(PhysicalOptimizationConfig physOptConf,
             IAsterixPropertiesProvider propertiesProvider) {
@@ -178,24 +185,25 @@
         if (dataset == null) {
             throw new AsterixException("Unknown dataset " + datasetName);
         }
-        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
-            throw new AsterixException("Cannot index an external dataset (" + datasetName + ").");
-        }
+
         itemType = (ARecordType) metadataProvider.findType(dataset.getDataverseName(), dataset.getItemTypeName());
         payloadSerde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
-        numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
         numSecondaryKeys = secondaryKeyFields.size();
-        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, datasetName);
-        primaryFileSplitProvider = primarySplitsAndConstraint.first;
-        primaryPartitionConstraint = primarySplitsAndConstraint.second;
         Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
-                .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName,
-                        secondaryIndexName);
+                .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, secondaryIndexName);
         secondaryFileSplitProvider = secondarySplitsAndConstraint.first;
         secondaryPartitionConstraint = secondarySplitsAndConstraint.second;
-        // Must be called in this order.
-        setPrimaryRecDescAndComparators();
+
+        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+            numPrimaryKeys = ExternalIndexingOperations.getRIDSize(dataset);
+        } else {
+            numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
+            Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider
+                    .splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, datasetName);
+            primaryFileSplitProvider = primarySplitsAndConstraint.first;
+            primaryPartitionConstraint = primarySplitsAndConstraint.second;
+            setPrimaryRecDescAndComparators();
+        }
         setSecondaryRecDescAndComparators(indexType, secondaryKeyFields, gramLength, metadataProvider);
         numElementsHint = metadataProvider.getCardinalityPerPartitionHint(dataset);
         Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(dataset,
@@ -245,9 +253,11 @@
         ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
         IBinaryComparatorFactoryProvider comparatorFactoryProvider = metadataProvider.getFormat()
                 .getBinaryComparatorFactoryProvider();
+        // Record column is 0 for external datasets, numPrimaryKeys for internal ones
+        int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
         for (int i = 0; i < numSecondaryKeys; i++) {
             secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(
-                    itemType, secondaryKeyFields.get(i), numPrimaryKeys);
+                    itemType, secondaryKeyFields.get(i), recordColumn);
             Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(i), itemType);
             IAType keyType = keyTypePair.first;
             anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
@@ -257,12 +267,23 @@
             secondaryTypeTraits[i] = typeTraitProvider.getTypeTrait(keyType);
             secondaryBloomFilterKeyFields[i] = i;
         }
-        // Add serializers and comparators for primary index fields.
-        for (int i = 0; i < numPrimaryKeys; i++) {
-            secondaryRecFields[numSecondaryKeys + i] = primaryRecDesc.getFields()[i];
-            secondaryTypeTraits[numSecondaryKeys + i] = primaryRecDesc.getTypeTraits()[i];
-            if (indexType != IndexType.RTREE) {
-                secondaryComparatorFactories[numSecondaryKeys + i] = primaryComparatorFactories[i];
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            // Add serializers and comparators for primary index fields.
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numSecondaryKeys + i] = primaryRecDesc.getFields()[i];
+                secondaryTypeTraits[numSecondaryKeys + i] = primaryRecDesc.getTypeTraits()[i];
+                if (indexType != IndexType.RTREE) {
+                    secondaryComparatorFactories[numSecondaryKeys + i] = primaryComparatorFactories[i];
+                }
+            }
+        } else {
+            // Add serializers and comparators for RID fields.
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numSecondaryKeys + i] = IndexingConstants.getSerializerDeserializer(i);
+                secondaryTypeTraits[numSecondaryKeys + i] = IndexingConstants.getTypeTraits(i);
+                if (indexType != IndexType.RTREE) {
+                    secondaryComparatorFactories[numSecondaryKeys + i] = IndexingConstants.getComparatorFactory(i);
+                }
             }
         }
         secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
@@ -314,8 +335,9 @@
                 new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
                         mergePolicyFactory, mergePolicyFactoryProperties, new PrimaryIndexOperationTrackerProvider(
                                 dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                        LSMBTreeIOOperationCallbackFactory.INSTANCE,
-                        storageProperties.getBloomFilterFalsePositiveRate(), true), false, false, null, searchCallbackFactory);
+                        LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
+                                .getBloomFilterFalsePositiveRate(), true), false, false, null,
+                searchCallbackFactory);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
                 primaryPartitionConstraint);
         return primarySearchOp;
@@ -409,4 +431,100 @@
                 primaryPartitionConstraint);
         return asterixSelectOp;
     }
+
+    // This method creates a source indexing operator for external data
+    protected ExternalDataScanOperatorDescriptor createExternalIndexingOp(JobSpecification spec)
+            throws AlgebricksException, AsterixException {
+        // A record + primary keys
+        ISerializerDeserializer[] serdes = new ISerializerDeserializer[1 + numPrimaryKeys];
+        ITypeTraits[] typeTraits = new ITypeTraits[1 + numPrimaryKeys];
+        // payload serde and type traits for the record slot
+        serdes[0] = payloadSerde;
+        typeTraits[0] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(itemType);
+        //  serdes and type traits for rid fields
+        for (int i = 1; i < serdes.length; i++) {
+            serdes[i] = IndexingConstants.getSerializerDeserializer(i - 1);
+            typeTraits[i] = IndexingConstants.getTypeTraits(i - 1);
+        }
+        // output record desc
+        RecordDescriptor indexerDesc = new RecordDescriptor(serdes, typeTraits);
+
+        // Create the operator and its partition constraits
+        Pair<ExternalDataScanOperatorDescriptor, AlgebricksPartitionConstraint> indexingOpAndConstraints;
+        try {
+            indexingOpAndConstraints = ExternalIndexingOperations.createExternalIndexingOp(spec, metadataProvider,
+                    dataset, itemType, indexerDesc, externalFiles);
+        } catch (Exception e) {
+            throw new AlgebricksException(e);
+        }
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexingOpAndConstraints.first,
+                indexingOpAndConstraints.second);
+
+        // Set the primary partition constraints to this partition constraints
+        primaryPartitionConstraint = indexingOpAndConstraints.second;
+        return indexingOpAndConstraints.first;
+    }
+
+    protected AlgebricksMetaOperatorDescriptor createExternalAssignOp(JobSpecification spec, int numSecondaryKeys) throws AlgebricksException {
+        int[] outColumns = new int[numSecondaryKeys];
+        int[] projectionList = new int[numSecondaryKeys + numPrimaryKeys];
+        for (int i = 0; i < numSecondaryKeys; i++) {
+            outColumns[i] = i + numPrimaryKeys + 1;
+            projectionList[i] = i + numPrimaryKeys + 1;
+        }
+
+        IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[secondaryFieldAccessEvalFactories.length];
+        for (int i = 0; i < secondaryFieldAccessEvalFactories.length; ++i) {
+            sefs[i] = new LogicalExpressionJobGenToExpressionRuntimeProviderAdapter.ScalarEvaluatorFactoryAdapter(
+                    secondaryFieldAccessEvalFactories[i]);
+        }
+        //add External RIDs to the projection list
+        for (int i = 0; i < numPrimaryKeys; i++) {
+            projectionList[numSecondaryKeys + i] = i + 1;
+        }
+
+        AssignRuntimeFactory assign = new AssignRuntimeFactory(outColumns, sefs, projectionList);
+        AlgebricksMetaOperatorDescriptor asterixAssignOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 1,
+                new IPushRuntimeFactory[] { assign }, new RecordDescriptor[] { secondaryRecDesc });
+        return asterixAssignOp;
+    }
+
+    protected ExternalIndexBulkModifyOperatorDescriptor createExternalIndexBulkModifyOp(JobSpecification spec,
+            int numSecondaryKeyFields, IIndexDataflowHelperFactory dataflowHelperFactory, float fillFactor)
+            throws MetadataException, AlgebricksException {
+        int[] fieldPermutation = new int[numSecondaryKeyFields + numPrimaryKeys];
+        for (int i = 0; i < numSecondaryKeyFields + numPrimaryKeys; i++) {
+            fieldPermutation[i] = i;
+        }
+        // create a list of file ids
+        int numOfDeletedFiles = 0;
+        for (ExternalFile file : externalFiles) {
+            if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP)
+                numOfDeletedFiles++;
+        }
+        int[] deletedFiles = new int[numOfDeletedFiles];
+        int i = 0;
+        for (ExternalFile file : externalFiles) {
+            if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP) {
+                deletedFiles[i] = file.getFileNumber();
+            }
+        }
+        ExternalIndexBulkModifyOperatorDescriptor treeIndexBulkLoadOp = new ExternalIndexBulkModifyOperatorDescriptor(
+                spec, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                dataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE, deletedFiles, fieldPermutation,
+                fillFactor, numElementsHint);
+        AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
+                secondaryPartitionConstraint);
+        return treeIndexBulkLoadOp;
+    }
+
+    public List<ExternalFile> getExternalFiles() {
+        return externalFiles;
+    }
+
+    public void setExternalFiles(List<ExternalFile> externalFiles) {
+        this.externalFiles = externalFiles;
+    }
 }

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
index 225ee53..37aa427 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java

@@ -18,6 +18,7 @@
 
 import edu.uci.ics.asterix.common.api.ILocalResourceMetadata;
 import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
 import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
 import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
 import edu.uci.ics.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -29,11 +30,15 @@
 import edu.uci.ics.asterix.formats.nontagged.AqlTypeTraitProvider;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.external.IndexingConstants;
+import edu.uci.ics.asterix.metadata.feeds.ExternalDataScanOperatorDescriptor;
+import edu.uci.ics.asterix.metadata.utils.ExternalDatasetsRegistry;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
 import edu.uci.ics.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerProvider;
 import edu.uci.ics.asterix.transaction.management.resource.LSMRTreeLocalResourceMetadata;
+import edu.uci.ics.asterix.transaction.management.resource.ExternalRTreeLocalResourceMetadata;
 import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceFactoryProvider;
 import edu.uci.ics.asterix.transaction.management.service.transaction.AsterixRuntimeComponentsProvider;
 import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
@@ -53,10 +58,13 @@
 import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
 import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexCreateOperatorDescriptor;
 import edu.uci.ics.hyracks.storage.am.common.impls.NoOpOperationCallbackFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexCompactOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.lsm.rtree.dataflow.LSMRTreeDataflowHelperFactory;
 import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreePolicyType;
 import edu.uci.ics.hyracks.storage.common.file.ILocalResourceFactoryProvider;
@@ -80,26 +88,50 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
-        ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
-                secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
-                valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
-                        secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
-                mergePolicyFactoryProperties, primaryKeyFields);
-        ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
-                localResourceMetadata, LocalResource.LSMRTreeResource);
+        IIndexDataflowHelperFactory indexDataflowHelperFactory;
+        ILocalResourceFactoryProvider localResourceFactoryProvider;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            //prepare a LocalResourceMetadata which will be stored in NC's local resource repository
+            ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
+                    valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
+                    mergePolicyFactoryProperties, primaryKeyFields);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.LSMRTreeResource);
+            indexDataflowHelperFactory = new LSMRTreeDataflowHelperFactory(valueProviderFactories,
+                    RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
+                            dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
+                    new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
+                    AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+                    storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields);
+        } else {
+            // External dataset
+            // Prepare a LocalResourceMetadata which will be stored in NC's local resource repository
+            ILocalResourceMetadata localResourceMetadata = new ExternalRTreeLocalResourceMetadata(
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
+                    ExternalIndexingOperations.getBuddyBtreeComparatorFactories(), valueProviderFactories,
+                    RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
+                    mergePolicyFactoryProperties, primaryKeyFields);
+            localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(localResourceMetadata,
+                    LocalResource.ExternalRTreeResource);
+
+            indexDataflowHelperFactory = new ExternalRTreeDataflowHelperFactory(valueProviderFactories,
+                    RTreePolicyType.RTREE, ExternalIndexingOperations.getBuddyBtreeComparatorFactories(),
+                    mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
+                            dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
+                    new int[] { numNestedSecondaryKeyFields },
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+        }
 
         TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
                 AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
                 secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, null,
-                new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
-                        primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
-                        mergePolicyFactory, mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(
-                                dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                        LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
-                                secondaryComparatorFactories.length), storageProperties
-                                .getBloomFilterFalsePositiveRate(), primaryKeyFields),
-                localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
+                indexDataflowHelperFactory, localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
                 secondaryPartitionConstraint);
         spec.addRoot(secondaryIndexCreateOp);
@@ -125,8 +157,9 @@
         }
         int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
         numNestedSecondaryKeyFields = numDimensions * 2;
+        int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
         secondaryFieldAccessEvalFactories = metadata.getFormat().createMBRFactory(itemType, secondaryKeyFields.get(0),
-                numPrimaryKeys, numDimensions);
+                recordColumn, numDimensions);
         secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
         valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
         ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys
@@ -144,9 +177,16 @@
             valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
         }
         // Add serializers and comparators for primary index fields.
-        for (int i = 0; i < numPrimaryKeys; i++) {
-            secondaryRecFields[numNestedSecondaryKeyFields + i] = primaryRecDesc.getFields()[i];
-            secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i];
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numNestedSecondaryKeyFields + i] = primaryRecDesc.getFields()[i];
+                secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i];
+            }
+        } else {
+            for (int i = 0; i < numPrimaryKeys; i++) {
+                secondaryRecFields[numNestedSecondaryKeyFields + i] = IndexingConstants.getSerializerDeserializer(i);
+                secondaryTypeTraits[numNestedSecondaryKeyFields + i] = IndexingConstants.getTypeTraits(i);
+            }
         }
         secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
         primaryKeyFields = new int[numPrimaryKeys];
@@ -158,51 +198,107 @@
     @Override
     public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            // Create dummy key provider for feeding the primary index scan. 
+            AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
 
-        // Create dummy key provider for feeding the primary index scan. 
-        AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
+            // Create primary index scan op.
+            BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
 
-        // Create primary index scan op.
-        BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp,
+                    numNestedSecondaryKeyFields);
 
-        // Assign op.
-        AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp,
-                numNestedSecondaryKeyFields);
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
+            }
 
-        // If any of the secondary fields are nullable, then add a select op that filters nulls.
-        AlgebricksMetaOperatorDescriptor selectOp = null;
-        if (anySecondaryKeyIsNullable) {
-            selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
-        }
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
+                    new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length) }, secondaryRecDesc);
 
-        // Sort by secondary keys.
-        ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
-                new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
-                        secondaryComparatorFactories.length) }, secondaryRecDesc);
-
-        AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        // Create secondary RTree bulk load op.
-        TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec,
-                numNestedSecondaryKeyFields, new LSMRTreeDataflowHelperFactory(valueProviderFactories,
-                        RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
-                                dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
-                        AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
-                        storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields),
-                BTree.DEFAULT_FILL_FACTOR);
-        // Connect the operators.
-        spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
-        spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
-        if (anySecondaryKeyIsNullable) {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
-            spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+            // Create secondary RTree bulk load op.
+            TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
+                    spec,
+                    numNestedSecondaryKeyFields,
+                    new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                            primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                                    secondaryComparatorFactories.length), storageProperties
+                                    .getBloomFilterFalsePositiveRate(), primaryKeyFields), BTree.DEFAULT_FILL_FACTOR);
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         } else {
-            spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            // External dataset
+            /*
+             * In case of external data, this method is used to build loading jobs for both initial load on index creation
+             * and transaction load on dataset referesh
+             */
+            // Create external indexing scan operator
+            ExternalDataScanOperatorDescriptor primaryScanOp = createExternalIndexingOp(spec);
+            // Assign op.
+            AlgebricksMetaOperatorDescriptor asterixAssignOp = createExternalAssignOp(spec, numNestedSecondaryKeyFields);
+
+            // If any of the secondary fields are nullable, then add a select op that filters nulls.
+            AlgebricksMetaOperatorDescriptor selectOp = null;
+            if (anySecondaryKeyIsNullable) {
+                selectOp = createFilterNullsSelectOp(spec, numSecondaryKeys);
+            }
+
+            // Sort by secondary keys.
+            ExternalSortOperatorDescriptor sortOp = createSortOp(spec,
+                    new IBinaryComparatorFactory[] { AqlMetadataProvider.proposeLinearizer(keyType,
+                            secondaryComparatorFactories.length) }, secondaryRecDesc);
+            AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
+
+            // Create the dataflow helper factory
+            ExternalRTreeDataflowHelperFactory dataflowHelperFactory = new ExternalRTreeDataflowHelperFactory(
+                    valueProviderFactories, RTreePolicyType.RTREE, primaryComparatorFactories, mergePolicyFactory,
+                    mergePolicyFactoryProperties, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
+                    AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
+                    storageProperties.getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
+                    ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset));
+            // Create secondary RTree bulk load op.
+            AbstractTreeIndexOperatorDescriptor secondaryBulkLoadOp;
+            if (externalFiles != null) {
+                // Transaction load
+                secondaryBulkLoadOp = createExternalIndexBulkModifyOp(spec, numNestedSecondaryKeyFields,
+                        dataflowHelperFactory, BTree.DEFAULT_FILL_FACTOR);
+            } else {
+                // Initial load
+                secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numNestedSecondaryKeyFields,
+                        dataflowHelperFactory, BTree.DEFAULT_FILL_FACTOR);
+            }
+            // Connect the operators.
+            spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
+            if (anySecondaryKeyIsNullable) {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
+                spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
+            } else {
+                spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
+            }
+            spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
+            spec.addRoot(secondaryBulkLoadOp);
+            spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         }
-        spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
-        spec.addRoot(secondaryBulkLoadOp);
-        spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
         return spec;
     }
 
@@ -211,17 +307,37 @@
         JobSpecification spec = JobSpecificationUtils.createJobSpecification();
 
         AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
-        LSMTreeIndexCompactOperatorDescriptor compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
-                AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
-                secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
-                secondaryBloomFilterKeyFields, new LSMRTreeDataflowHelperFactory(valueProviderFactories,
-                        RTreePolicyType.RTREE, primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(
-                                dataset.getDatasetId()), mergePolicyFactory, mergePolicyFactoryProperties,
-                        new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
-                        AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
-                        AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
-                        storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields),
-                NoOpOperationCallbackFactory.INSTANCE);
+        LSMTreeIndexCompactOperatorDescriptor compactOp;
+        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new LSMRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                            primaryComparatorFactories, new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
+                            mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                                    secondaryComparatorFactories.length), storageProperties
+                                    .getBloomFilterFalsePositiveRate(), primaryKeyFields),
+                    NoOpOperationCallbackFactory.INSTANCE);
+        } else {
+            // External dataset
+            compactOp = new LSMTreeIndexCompactOperatorDescriptor(spec,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                    AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, secondaryFileSplitProvider,
+                    secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields,
+                    new ExternalRTreeDataflowHelperFactory(valueProviderFactories, RTreePolicyType.RTREE,
+                            primaryComparatorFactories, mergePolicyFactory, mergePolicyFactoryProperties,
+                            new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+                            AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+                            LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
+                                    secondaryComparatorFactories.length), storageProperties
+                                    .getBloomFilterFalsePositiveRate(), new int[] { numNestedSecondaryKeyFields },
+                            ExternalDatasetsRegistry.INSTANCE.getDatasetVersion(dataset)),
+                    NoOpOperationCallbackFactory.INSTANCE);
+        }
         AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
                 secondaryPartitionConstraint);
         spec.addRoot(compactOp);

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java
new file mode 100644
index 0000000..0af6a7e
--- /dev/null
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/AsterixGlobalRecoveryManager.java

@@ -0,0 +1,212 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.hyracks.bootstrap;
+
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
+import edu.uci.ics.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import edu.uci.ics.asterix.file.ExternalIndexingOperations;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.api.IClusterEventsSubscriber;
+import edu.uci.ics.asterix.metadata.api.IClusterManagementWork;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
+import edu.uci.ics.asterix.metadata.cluster.IClusterManagementWorkResponse;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.asterix.metadata.entities.ExternalDatasetDetails;
+import edu.uci.ics.asterix.metadata.entities.ExternalFile;
+import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties.State;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class AsterixGlobalRecoveryManager implements IClusterEventsSubscriber {
+
+    private static State state;
+    private static final Logger LOGGER = Logger.getLogger(AsterixGlobalRecoveryManager.class.getName());
+    private HyracksConnection hcc;
+    public static AsterixGlobalRecoveryManager INSTANCE;
+
+    public AsterixGlobalRecoveryManager(HyracksConnection hcc) throws Exception {
+        state = AsterixClusterProperties.INSTANCE.getState();
+        this.hcc = hcc;
+    }
+
+    @Override
+    public Set<IClusterManagementWork> notifyNodeFailure(Set<String> deadNodeIds) {
+        state = AsterixClusterProperties.INSTANCE.getState();
+        AsterixClusterProperties.INSTANCE.setGlobalRecoveryCompleted(false);
+        return null;
+    }
+
+    @Override
+    public Set<IClusterManagementWork> notifyNodeJoin(String joinedNodeId) {
+        // perform global recovery if state changed to active
+        final State newState = AsterixClusterProperties.INSTANCE.getState();
+        boolean needToRecover = !newState.equals(state) && (newState == State.ACTIVE);
+        if (needToRecover) {
+            Thread recoveryThread = new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    LOGGER.info("Starting AsterixDB's Global Recovery");
+                    MetadataTransactionContext mdTxnCtx = null;
+                    try {
+                        Thread.sleep(4000);
+                        MetadataManager.INSTANCE.init();
+                        // Loop over datasets
+                        mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                        List<Dataverse> dataverses = MetadataManager.INSTANCE.getDataverses(mdTxnCtx);
+                        for (Dataverse dataverse : dataverses) {
+                            if (!dataverse.getDataverseName().equals(MetadataConstants.METADATA_DATAVERSE_NAME)) {
+                                AqlMetadataProvider metadataProvider = new AqlMetadataProvider(dataverse);
+                                List<Dataset> datasets = MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx,
+                                        dataverse.getDataverseName());
+                                for (Dataset dataset : datasets) {
+                                    if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+                                        // External dataset
+                                        // Get indexes
+                                        List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx,
+                                                dataset.getDataverseName(), dataset.getDatasetName());
+                                        if (indexes.size() > 0) {
+                                            // Get the state of the dataset
+                                            ExternalDatasetDetails dsd = (ExternalDatasetDetails) dataset
+                                                    .getDatasetDetails();
+                                            ExternalDatasetTransactionState datasetState = dsd.getState();
+                                            if (datasetState == ExternalDatasetTransactionState.BEGIN) {
+                                                List<ExternalFile> files = MetadataManager.INSTANCE
+                                                        .getDatasetExternalFiles(mdTxnCtx, dataset);
+                                                // if persumed abort, roll backward
+                                                // 1. delete all pending files
+                                                for (ExternalFile file : files) {
+                                                    if (file.getPendingOp() != ExternalFilePendingOp.PENDING_NO_OP) {
+                                                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                                                    }
+                                                }
+                                                // 2. clean artifacts in NCs
+                                                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                                                JobSpecification jobSpec = ExternalIndexingOperations.buildAbortOp(
+                                                        dataset, indexes, metadataProvider);
+                                                executeHyracksJob(jobSpec);
+                                                // 3. correct the dataset state
+                                                ((ExternalDatasetDetails) dataset.getDatasetDetails())
+                                                        .setState(ExternalDatasetTransactionState.COMMIT);
+                                                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, dataset);
+                                                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                                                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                                            } else if (datasetState == ExternalDatasetTransactionState.READY_TO_COMMIT) {
+                                                List<ExternalFile> files = MetadataManager.INSTANCE
+                                                        .getDatasetExternalFiles(mdTxnCtx, dataset);
+                                                // if ready to commit, roll forward
+                                                // 1. commit indexes in NCs
+                                                metadataProvider.setMetadataTxnContext(mdTxnCtx);
+                                                JobSpecification jobSpec = ExternalIndexingOperations.buildRecoverOp(
+                                                        dataset, indexes, metadataProvider);
+                                                executeHyracksJob(jobSpec);
+                                                // 2. add pending files in metadata
+                                                for (ExternalFile file : files) {
+                                                    if (file.getPendingOp() == ExternalFilePendingOp.PENDING_ADD_OP) {
+                                                        MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
+                                                        file.setPendingOp(ExternalFilePendingOp.PENDING_NO_OP);
+                                                        MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+                                                    } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_DROP_OP) {
+                                                        // find original file
+                                                        for (ExternalFile originalFile : files) {
+                                                            if (originalFile.getFileName().equals(file.getFileName())) {
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        file);
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        originalFile);
+                                                                break;
+                                                            }
+                                                        }
+                                                    } else if (file.getPendingOp() == ExternalFilePendingOp.PENDING_APPEND_OP) {
+                                                        // find original file
+                                                        for (ExternalFile originalFile : files) {
+                                                            if (originalFile.getFileName().equals(file.getFileName())) {
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        file);
+                                                                MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx,
+                                                                        originalFile);
+                                                                originalFile.setSize(file.getSize());
+                                                                MetadataManager.INSTANCE.addExternalFile(mdTxnCtx,
+                                                                        originalFile);
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                                // 3. correct the dataset state
+                                                ((ExternalDatasetDetails) dataset.getDatasetDetails())
+                                                        .setState(ExternalDatasetTransactionState.COMMIT);
+                                                MetadataManager.INSTANCE.updateDataset(mdTxnCtx, dataset);
+                                                MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                                                mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+                    } catch (Exception e) {
+                        // This needs to be fixed <-- Needs to shutdown the system -->
+                        /*
+                         * Note: Throwing this illegal state exception will terminate this thread
+                         * and feeds listeners will not be notified.
+                         */
+                        LOGGER.severe("Global recovery was not completed successfully" + e);
+                        try {
+                            MetadataManager.INSTANCE.abortTransaction(mdTxnCtx);
+                        } catch (Exception e1) {
+                            if (LOGGER.isLoggable(Level.SEVERE)) {
+                                LOGGER.severe("Exception in aborting" + e.getMessage());
+                            }
+                            throw new IllegalStateException(e1);
+                        }
+                    }
+                    AsterixClusterProperties.INSTANCE.setGlobalRecoveryCompleted(true);
+                    LOGGER.info("Global Recovery Completed");
+                }
+            });
+            state = newState;
+            recoveryThread.start();
+        }
+        return null;
+    }
+
+    private void executeHyracksJob(JobSpecification spec) throws Exception {
+        spec.setMaxReattempts(0);
+        JobId jobId = hcc.startJob(spec);
+        hcc.waitForCompletion(jobId);
+    }
+
+    @Override
+    public void notifyRequestCompletion(IClusterManagementWorkResponse response) {
+        // Do nothing
+    }
+
+    @Override
+    public void notifyStateChange(State previousState, State newState) {
+        // Do nothing?
+    }
+}

diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 6314d54..ce528bf 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java

@@ -37,6 +37,7 @@
 import edu.uci.ics.asterix.metadata.MetadataManager;
 import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
 import edu.uci.ics.asterix.metadata.bootstrap.AsterixStateProxy;
+import edu.uci.ics.asterix.metadata.cluster.ClusterManager;
 import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
 import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
 import edu.uci.ics.hyracks.api.application.ICCApplicationEntryPoint;
@@ -94,6 +95,9 @@
 		setupFeedServer(externalProperties);
 		feedServer.start();
 
+		AsterixGlobalRecoveryManager.INSTANCE = new AsterixGlobalRecoveryManager((HyracksConnection) getNewHyracksClientConnection());
+		ClusterManager.INSTANCE.registerSubscriber(AsterixGlobalRecoveryManager.INSTANCE);
+		
 		ccAppCtx.addClusterLifecycleListener(ClusterLifecycleListener.INSTANCE);
 	}
 

diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
index b9d7491..0750a58 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta16/meta16.1.adm

@@ -1,13 +1,14 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 13, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 2, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 8, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 3, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 1, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 10, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 11, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 12, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 7, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 4, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 9, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 5, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 13, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 2, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 8, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 3, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 1, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DataTypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "FileNumber" ], "PrimaryKey": [ "DataverseName", "DatasetName", "FileNumber" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 14, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 10, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 11, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 12, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 7, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 4, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 9, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 5, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm
index 66a5a68..4580e90 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta17/meta17.1.adm

@@ -1,70 +1,73 @@
-{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Mon Sep 23 00:04:06 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Thu Oct 24 01:40:50 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "FileNumber", "FieldType": "int32" }, { "FieldName": "FileName", "FieldType": "string" }, { "FieldName": "FileSize", "FieldType": "int64" }, { "FieldName": "FileModTime", "FieldType": "datetime" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "LastRefreshTime", "FieldType": "datetime" }, { "FieldName": "TransactionState", "FieldType": "int32" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm b/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm
index a8dfc95..c14e3f2 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/meta19/meta19.1.adm

@@ -1,16 +1,17 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "IndexName": "ExternalFile", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "FileNumber" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm b/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
index b9d7491..0750a58 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/metadata_dataset/metadata_dataset.1.adm

@@ -1,13 +1,14 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 13, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 2, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 8, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 3, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 1, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 10, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 11, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 12, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 7, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 4, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 9, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 5, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 13, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 2, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 8, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 3, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 1, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DataTypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "FileNumber" ], "PrimaryKey": [ "DataverseName", "DatasetName", "FileNumber" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 14, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 10, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "DataTypeName": "FeedActivityRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "PrimaryKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 11, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "DataTypeName": "FeedPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "PolicyName" ], "PrimaryKey": [ "DataverseName", "PolicyName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 12, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "DataTypeName": "FunctionRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name", "Arity" ], "PrimaryKey": [ "DataverseName", "Name", "Arity" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 7, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "DataTypeName": "IndexRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "IndexName" ], "PrimaryKey": [ "DataverseName", "DatasetName", "IndexName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 4, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "DataTypeName": "LibraryRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 9, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "DataTypeName": "NodeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "NodeName" ], "PrimaryKey": [ "NodeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 5, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "DataTypeName": "NodeGroupRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "GroupName" ], "PrimaryKey": [ "GroupName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 6, "PendingOp": 0 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm b/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
index 4081076..4580e90 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/metadata_datatype/metadata_datatype.1.adm

@@ -1,70 +1,73 @@
-{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Mon Sep 23 00:25:26 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Thu Oct 24 01:40:50 PDT 2013" }
-{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Thu Sep 26 03:23:51 PDT 2013" }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatatypeName": "CompactionPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "DataTypeName", "FieldType": "string" }, { "FieldName": "DatasetType", "FieldType": "string" }, { "FieldName": "InternalDetails", "FieldType": "Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "ExternalDetails", "FieldType": "Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "Hints", "FieldType": "Field_Hints_in_DatasetRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "DatasetId", "FieldType": "int32" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatasourceAdapterRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Classname", "FieldType": "string" }, { "FieldName": "Type", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatatypeName", "FieldType": "string" }, { "FieldName": "Derived", "FieldType": "Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "DataverseRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DataFormat", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "ExternalFileRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "FileNumber", "FieldType": "int32" }, { "FieldName": "FileName", "FieldType": "string" }, { "FieldName": "FileSize", "FieldType": "int64" }, { "FieldName": "FileModTime", "FieldType": "datetime" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedActivityRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "ActivityId", "FieldType": "int32" }, { "FieldName": "ActivityType", "FieldType": "string" }, { "FieldName": "Details", "FieldType": "Field_Details_in_FeedActivityRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedPolicyRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "PolicyName", "FieldType": "string" }, { "FieldName": "Description", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_FeedPolicyRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FeedRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "FeedName", "FieldType": "string" }, { "FieldName": "AdaptorName", "FieldType": "string" }, { "FieldName": "AdaptorConfiguration", "FieldType": "Field_AdaptorConfiguration_in_FeedRecordType" }, { "FieldName": "Function", "FieldType": "Field_Function_in_FeedRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_AdaptorConfiguration_in_FeedRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Details_in_FeedActivityRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Details_in_FeedActivityRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FieldName", "FieldType": "string" }, { "FieldName": "FieldType", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Function_in_FeedRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Hints_in_DatasetRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Hints_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_NodeNames_in_NodeGroupRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "string", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Params_in_FunctionRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType", "Derived": { "Tag": "UNORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": "Field_Properties_in_FeedPolicyRecordType_ItemType", "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_FeedPolicyRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType_ItemType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Value", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_SearchKey_in_IndexRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "UNION", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": [ "null", "string" ], "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "FunctionRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Arity", "FieldType": "string" }, { "FieldName": "Params", "FieldType": "Field_Params_in_FunctionRecordType" }, { "FieldName": "ReturnType", "FieldType": "string" }, { "FieldName": "Definition", "FieldType": "string" }, { "FieldName": "Language", "FieldType": "string" }, { "FieldName": "Kind", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "IndexRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "DatasetName", "FieldType": "string" }, { "FieldName": "IndexName", "FieldType": "string" }, { "FieldName": "IndexStructure", "FieldType": "string" }, { "FieldName": "SearchKey", "FieldType": "Field_SearchKey_in_IndexRecordType" }, { "FieldName": "IsPrimary", "FieldType": "boolean" }, { "FieldName": "Timestamp", "FieldType": "string" }, { "FieldName": "PendingOp", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "LibraryRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DataverseName", "FieldType": "string" }, { "FieldName": "Name", "FieldType": "string" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeGroupRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "NodeNames", "FieldType": "Field_NodeNames_in_NodeGroupRecordType" }, { "FieldName": "Timestamp", "FieldType": "string" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "NodeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": false, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "NodeName", "FieldType": "string" }, { "FieldName": "NumberOfCores", "FieldType": "int32" }, { "FieldName": "WorkingMemorySize", "FieldType": "int32" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "Tag", "FieldType": "string" }, { "FieldName": "IsAnonymous", "FieldType": "boolean" }, { "FieldName": "EnumValues", "FieldType": "Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Record", "FieldType": "Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "Union", "FieldType": "Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "UnorderedList", "FieldType": "Field_UnorderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" }, { "FieldName": "OrderedList", "FieldType": "Field_OrderedList_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_EnumValues_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "DatasourceAdapter", "FieldType": "string" }, { "FieldName": "Properties", "FieldType": "Field_Properties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "LastRefreshTime", "FieldType": "datetime" }, { "FieldName": "TransactionState", "FieldType": "int32" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_ExternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "FileStructure", "FieldType": "string" }, { "FieldName": "PartitioningStrategy", "FieldType": "string" }, { "FieldName": "PartitioningKey", "FieldType": "Field_PartitioningKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "PrimaryKey", "FieldType": "Field_PrimaryKey_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" }, { "FieldName": "GroupName", "FieldType": "string" }, { "FieldName": "Autogenerated", "FieldType": "boolean" }, { "FieldName": "CompactionPolicy", "FieldType": "string" }, { "FieldName": "CompactionPolicyProperties", "FieldType": "Field_CompactionPolicyProperties_in_Type_#1_UnionType_Field_InternalDetails_in_DatasetRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "EnumValues": null, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "IsOpen", "FieldType": "boolean" }, { "FieldName": "Fields", "FieldType": "Field_Fields_in_Type_#1_UnionType_Field_Record_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType" } ] }, "Union": null, "UnorderedList": null, "OrderedList": null }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "Type_#1_UnionType_Field_Union_in_Type_#1_UnionType_Field_Derived_in_DatatypeRecordType", "Derived": { "Tag": "ORDEREDLIST", "IsAnonymous": true, "EnumValues": null, "Record": null, "Union": null, "UnorderedList": null, "OrderedList": "string" }, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "boolean", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "circle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "date", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "datetime", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "day-time-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "double", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "float", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int16", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int32", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int64", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "int8", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "interval", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "line", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "null", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "point3d", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "polygon", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "rectangle", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "string", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "time", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "uuid", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
+{ "DataverseName": "Metadata", "DatatypeName": "year-month-duration", "Derived": null, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014" }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm b/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm
index a8dfc95..c14e3f2 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/metadata_index/metadata_index.1.adm

@@ -1,16 +1,17 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Thu Sep 26 02:39:06 PDT 2013", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "IndexName": "CompactionPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "CompactionPolicy" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "Dataset", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "IndexName": "GroupName", "IndexStructure": "BTREE", "SearchKey": [ "GroupName", "DataverseName", "DatasetName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "IndexName": "DatasourceAdapter", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "Datatype", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatatypeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "IndexName": "DatatypeName", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "NestedDatatypeName", "TopDatatypeName" ], "IsPrimary": false, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "IndexName": "Dataverse", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "IndexName": "ExternalFile", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "FileNumber" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Feed", "IndexName": "Feed", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedActivity", "IndexName": "FeedActivity", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "FeedName", "DatasetName", "ActivityId" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "FeedPolicy", "IndexName": "FeedPolicy", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "PolicyName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Function", "IndexName": "Function", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name", "Arity" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Index", "IndexName": "Index", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "DatasetName", "IndexName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Library", "IndexName": "Library", "IndexStructure": "BTREE", "SearchKey": [ "DataverseName", "Name" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Node", "IndexName": "Node", "IndexStructure": "BTREE", "SearchKey": [ "NodeName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Nodegroup", "IndexName": "Nodegroup", "IndexStructure": "BTREE", "SearchKey": [ "GroupName" ], "IsPrimary": true, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "PendingOp": 0 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.ddl.aql
new file mode 100644
index 0000000..55bda77f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.ddl.aql

@@ -0,0 +1,38 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+	screen-name: string,
+	lang: string,
+	friends-count: int32,
+	statuses-count: int32,
+	name: string,
+	followers-count: int32
+} 
+
+create type TweetMessageType as closed {
+	tweetid: int64,
+        user: TwitterUserType,
+        sender-location: point,
+	send-time: datetime,
+        referred-topics: {{ string }},
+	message-text: string,
+	countA: int32,
+	countB: int32
+}
+
+create external dataset TweetMessages(TweetMessageType) using hdfs(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/tw_for_indexleftouterjoin.adm"),("input-format"="text-input-format"),("format"="adm"));
+
+create index IdIx on TweetMessages(tweetid) type btree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;
+create index twmSndLocIx on TweetMessages(sender-location) type rtree;
+

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.2.update.aql


diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.3.query.aql
new file mode 100644
index 0000000..50e72e3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.3.query.aql

@@ -0,0 +1,21 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary rtree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+let $n :=  create-circle($t1.sender-location, 0.5)
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"loc1":$t1.sender-location,
+"nearby-message": for $t2 in dataset('TweetMessages')
+                             where spatial-intersect($t2.sender-location, $n) 
+                             order by $t2.tweetid 
+                             return {"tweetid2":$t2.tweetid, "loc2":$t2.sender-location}
+};
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.1.ddl.aql
new file mode 100644
index 0000000..ed033eb
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.1.ddl.aql

@@ -0,0 +1,36 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type TwitterUserType as closed {
+	screen-name: string,
+	lang: string,
+	friends-count: int32,
+	statuses-count: int32,
+	name: string,
+	followers-count: int32
+} 
+
+create type TweetMessageType as closed {
+	tweetid: int64,
+        user: TwitterUserType,
+        sender-location: point,
+	send-time: datetime,
+        referred-topics: {{ string }},
+	message-text: string,
+	countA: int32,
+	countB: int32
+}
+
+create external dataset TweetMessages(TweetMessageType) using hdfs(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/tw_for_indexleftouterjoin.adm"),("input-format"="text-input-format"),("format"="adm"));
+
+create index IdIx on TweetMessages(tweetid) type btree;
+create index msgCountAIx on TweetMessages(countA) type btree;
+create index msgCountBIx on TweetMessages(countB) type btree;

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.2.update.aql
new file mode 100644
index 0000000..16cbac3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.2.update.aql

@@ -0,0 +1,6 @@
+/*
+ * Description  : Test that left-outer-join may use two available indexes, one for primary index in prob subtree and another for secondary btree index in index subtree.
+ * Issue        : 730, 741                 
+ * Expected Res : Success
+ * Date         : 8th May 2014
+ */
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.3.query.aql
new file mode 100644
index 0000000..408a2e1
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/leftouterjoin/leftouterjoin.3.query.aql

@@ -0,0 +1,14 @@
+use dataverse test;
+
+for $t1 in dataset('TweetMessages')
+where $t1.tweetid < int64("10")
+order by $t1.tweetid
+return {
+"tweetid1": $t1.tweetid,
+"count1":$t1.countA,
+"t2info": for $t2 in dataset('TweetMessages') 
+          where $t1.countA /* +indexnl */= $t2.countB
+          order by $t2.tweetid 
+          return {"tweetid2": $t2.tweetid,
+                  "count2":$t2.countB}
+};
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.1.ddl.aql
new file mode 100644
index 0000000..4f95913
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.1.ddl.aql

@@ -0,0 +1,23 @@
+/*
+* Description  : Create an external dataset that contains records stored with rc hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type EmployeeType as closed {
+ id: int32,
+ name: string,
+ age: int32
+};
+
+create external dataset EmployeeDataset(EmployeeType)
+using hdfs
+(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.rc"),("input-format"="rc-input-format"),("format"="binary"),("parser"="hive-parser"),("hive-serde"="org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe"));
+
+create index EmployeeAgeIdx on EmployeeDataset(age);
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.2.update.aql
new file mode 100644
index 0000000..569c3e6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.2.update.aql

@@ -0,0 +1,7 @@
+/*
+* Description  : Create an external dataset that contains records stored with rc hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.3.query.aql
new file mode 100644
index 0000000..d46031c
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rc-format/rc-format.3.query.aql

@@ -0,0 +1,12 @@
+/*
+* Description  : Create an external dataset that contains records stored with rc hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+use dataverse test;
+
+for $emp in dataset EmployeeDataset
+where $emp.age = 22
+return $emp;
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.1.ddl.aql
new file mode 100644
index 0000000..8bd00c5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.1.ddl.aql

@@ -0,0 +1,19 @@
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type MyRecord as closed {
+  id: int32,
+  point: point,
+  kwds: string,
+  line1: line,
+  line2: line,
+  poly1: polygon,
+  poly2: polygon,
+  rec: rectangle,
+  circle: circle
+}
+
+create external dataset MyData(MyRecord) using hdfs(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/spatialData.json"),("input-format"="text-input-format"),("format"="adm"));
+
+create index rtree_index_point on MyData(point) type rtree;
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.2.update.aql


diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.3.query.aql
new file mode 100644
index 0000000..9986216
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/rtree-index/rtree-index.3.query.aql

@@ -0,0 +1,6 @@
+use dataverse test;
+
+for $o in dataset('MyData')
+where spatial-intersect($o.point, create-polygon([4.0,1.0,4.0,4.0,12.0,4.0,12.0,1.0]))
+order by $o.id
+return {"id":$o.id}
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.1.ddl.aql
new file mode 100644
index 0000000..f149f35
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.1.ddl.aql

@@ -0,0 +1,23 @@
+/*
+* Description  : Create an external dataset that contains records stored with sequence hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type EmployeeType as closed {
+ id: int32,
+ name: string,
+ age: int32
+};
+
+create external dataset EmployeeDataset(EmployeeType)
+using hdfs
+(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.seq"),("input-format"="sequence-input-format"),("format"="delimited-text"),("delimiter"="|"));
+
+create index EmployeeAgeIdx on EmployeeDataset(age);
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.2.update.aql
new file mode 100644
index 0000000..593a13b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.2.update.aql

@@ -0,0 +1,7 @@
+/*
+* Description  : Create an external dataset that contains records stored with sequence hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.3.query.aql
new file mode 100644
index 0000000..b714671
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/sequence-format/sequence-format.3.query.aql

@@ -0,0 +1,12 @@
+/*
+* Description  : Create an external dataset that contains records stored with sequence hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+use dataverse test;
+
+for $emp in dataset EmployeeDataset
+where $emp.age = 22
+return $emp;
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.1.ddl.aql
new file mode 100644
index 0000000..8109531
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.1.ddl.aql

@@ -0,0 +1,23 @@
+/*
+* Description  : Create an external dataset that contains records stored with text hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+drop dataverse test if exists;
+create dataverse test;
+
+use dataverse test;
+
+create type EmployeeType as closed {
+ id: int32,
+ name: string,
+ age: int32
+};
+
+create external dataset EmployeeDataset(EmployeeType)
+using hdfs
+(("hdfs"="hdfs://127.0.0.1:31888"),("path"="/asterix/external-indexing-test.txt"),("input-format"="text-input-format"),("format"="delimited-text"),("delimiter"="|"));
+
+create index EmployeeAgeIdx on EmployeeDataset(age);
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.2.update.aql
new file mode 100644
index 0000000..4fb3db0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.2.update.aql

@@ -0,0 +1,7 @@
+/*
+* Description  : Create an external dataset that contains records stored with text hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.3.query.aql
new file mode 100644
index 0000000..954e877
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/external-indexing/text-format/text-format.3.query.aql

@@ -0,0 +1,12 @@
+/*
+* Description  : Create an external dataset that contains records stored with text hdfs file format.
+                 Build an index over the external dataset age attribute
+                 Perform a query over the dataset using the index.
+* Expected Res : Success
+* Date         : 3rd Jan 2014
+*/
+use dataverse test;
+
+for $emp in dataset EmployeeDataset
+where $emp.age = 22
+return $emp;
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm b/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
index 62d8a6c..660e2a8 100644
--- a/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/cross-dataverse/cross-dv19/cross-dv19.1.adm

@@ -1,7 +1,7 @@
-{ "DataverseName": "test1", "DatasetName": "TwitterData", "DataTypeName": "Tweet", "DatasetType": "EXTERNAL", "InternalDetails": null, "ExternalDetails": { "DatasourceAdapter": "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter", "Properties": [ { "Name": "path", "Value": "nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ] }, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 119, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t1", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 113, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 116, "PendingOp": 0 }
-{ "DataverseName": "test1", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 117, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 114, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 115, "PendingOp": 0 }
-{ "DataverseName": "test2", "DatasetName": "t4", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Tue Sep 24 22:25:02 PDT 2013", "DatasetId": 118, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "test1", "DatasetName": "TwitterData", "DataTypeName": "Tweet", "DatasetType": "EXTERNAL", "InternalDetails": null, "ExternalDetails": { "DatasourceAdapter": "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter", "Properties": [ { "Name": "path", "Value": "nc1://data/twitter/extrasmalltweets.txt" }, { "Name": "format", "Value": "adm" } ], "GroupName": "DEFAULT_NG_ALL_NODES", "LastRefreshTime": datetime("2014-06-08T20:30:43.724Z"), "TransactionState": 0, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 107, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t1", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:42 PDT 2014", "DatasetId": 101, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 104, "PendingOp": 0 }
+{ "DataverseName": "test1", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 105, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t2", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:42 PDT 2014", "DatasetId": 102, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t3", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:42 PDT 2014", "DatasetId": 103, "PendingOp": 0 }
+{ "DataverseName": "test2", "DatasetName": "t4", "DataTypeName": "testtype", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "id" ], "PrimaryKey": [ "id" ], "GroupName": "DEFAULT_NG_ALL_NODES", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:30:43 PDT 2014", "DatasetId": 106, "PendingOp": 0 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.adm
new file mode 100644
index 0000000..b4337b3
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin-rtree/leftouterjoin-rtree.1.adm

@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "loc1": point("42.83,72.44"), "nearby-message": [ { "tweetid2": 1i64, "loc2": point("42.83,72.44") }, { "tweetid2": 55i64, "loc2": point("42.77,72.16") }, { "tweetid2": 114i64, "loc2": point("42.87,72.38") } ] }
+{ "tweetid1": 2i64, "loc1": point("34.81,72.44"), "nearby-message": [ { "tweetid2": 2i64, "loc2": point("34.81,72.44") } ] }
+{ "tweetid1": 3i64, "loc1": point("24.54,82.66"), "nearby-message": [ { "tweetid2": 3i64, "loc2": point("24.54,82.66") } ] }
+{ "tweetid1": 4i64, "loc1": point("38.14,68.1"), "nearby-message": [ { "tweetid2": 4i64, "loc2": point("38.14,68.1") } ] }
+{ "tweetid1": 5i64, "loc1": point("35.4,68.89"), "nearby-message": [ { "tweetid2": 5i64, "loc2": point("35.4,68.89") } ] }
+{ "tweetid1": 6i64, "loc1": point("42.75,78.5"), "nearby-message": [ { "tweetid2": 6i64, "loc2": point("42.75,78.5") } ] }
+{ "tweetid1": 7i64, "loc1": point("48.16,71.59"), "nearby-message": [ { "tweetid2": 7i64, "loc2": point("48.16,71.59") }, { "tweetid2": 42i64, "loc2": point("47.86,71.93") }, { "tweetid2": 192i64, "loc2": point("48.12,72.0") } ] }
+{ "tweetid1": 8i64, "loc1": point("36.17,72.56"), "nearby-message": [ { "tweetid2": 8i64, "loc2": point("36.17,72.56") } ] }
+{ "tweetid1": 9i64, "loc1": point("38.02,70.38"), "nearby-message": [ { "tweetid2": 9i64, "loc2": point("38.02,70.38") }, { "tweetid2": 51i64, "loc2": point("37.65,70.54") } ] }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin/leftouterjoin.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin/leftouterjoin.1.adm
new file mode 100644
index 0000000..1907bca
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/leftouterjoin/leftouterjoin.1.adm

@@ -0,0 +1,9 @@
+{ "tweetid1": 1i64, "count1": 1, "t2info": [  ] }
+{ "tweetid1": 2i64, "count1": 2, "t2info": [ { "tweetid2": 60i64, "count2": 2 } ] }
+{ "tweetid1": 3i64, "count1": 3, "t2info": [ { "tweetid2": 105i64, "count2": 3 }, { "tweetid2": 206i64, "count2": 3 } ] }
+{ "tweetid1": 4i64, "count1": 4, "t2info": [  ] }
+{ "tweetid1": 5i64, "count1": 5, "t2info": [ { "tweetid2": 138i64, "count2": 5 }, { "tweetid2": 175i64, "count2": 5 } ] }
+{ "tweetid1": 6i64, "count1": 6, "t2info": [ { "tweetid2": 148i64, "count2": 6 } ] }
+{ "tweetid1": 7i64, "count1": 7, "t2info": [ { "tweetid2": 125i64, "count2": 7 } ] }
+{ "tweetid1": 8i64, "count1": 8, "t2info": [  ] }
+{ "tweetid1": 9i64, "count1": 9, "t2info": [ { "tweetid2": 141i64, "count2": 9 } ] }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/rc-format/rc-format.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/rc-format/rc-format.1.adm
new file mode 100644
index 0000000..3ee4a1e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/rc-format/rc-format.1.adm

@@ -0,0 +1,2 @@
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/rtree-index/rtree-index.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/rtree-index/rtree-index.1.adm
new file mode 100644
index 0000000..d22217a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/rtree-index/rtree-index.1.adm

@@ -0,0 +1,2 @@
+{ "id": 12 }
+{ "id": 20 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/sequence-format/sequence-format.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/sequence-format/sequence-format.1.adm
new file mode 100644
index 0000000..3ee4a1e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/sequence-format/sequence-format.1.adm

@@ -0,0 +1,2 @@
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/external-indexing/text-format/text-format.1.adm b/asterix-app/src/test/resources/runtimets/results/external-indexing/text-format/text-format.1.adm
new file mode 100644
index 0000000..3ee4a1e
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/external-indexing/text-format/text-format.1.adm

@@ -0,0 +1,2 @@
+{ "id": 3, "name": "Samuel", "age": 22 }
+{ "id": 10, "name": "David", "age": 22 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm b/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
index f3636a4..1b4a1f6 100644
--- a/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/user-defined-functions/udf23/udf23.1.adm

@@ -1,6 +1,6 @@
-{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 13, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 2, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 8, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 3, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 1, "PendingOp": 0 }
-{ "DataverseName": "Metadata", "DatasetName": "Feed", "DataTypeName": "FeedRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "FeedName" ], "PrimaryKey": [ "DataverseName", "FeedName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Thu Oct 24 01:49:04 PDT 2013", "DatasetId": 10, "PendingOp": 0 }
\ No newline at end of file
+{ "DataverseName": "Metadata", "DatasetName": "CompactionPolicy", "DataTypeName": "CompactionPolicyRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "CompactionPolicy" ], "PrimaryKey": [ "DataverseName", "CompactionPolicy" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 13, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataset", "DataTypeName": "DatasetRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName" ], "PrimaryKey": [ "DataverseName", "DatasetName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 2, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "DatasourceAdapter", "DataTypeName": "DatasourceAdapterRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "Name" ], "PrimaryKey": [ "DataverseName", "Name" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 8, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Datatype", "DataTypeName": "DatatypeRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatatypeName" ], "PrimaryKey": [ "DataverseName", "DatatypeName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 3, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "Dataverse", "DataTypeName": "DataverseRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName" ], "PrimaryKey": [ "DataverseName" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 1, "PendingOp": 0 }
+{ "DataverseName": "Metadata", "DatasetName": "ExternalFile", "DataTypeName": "ExternalFileRecordType", "DatasetType": "INTERNAL", "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ "DataverseName", "DatasetName", "FileNumber" ], "PrimaryKey": [ "DataverseName", "DatasetName", "FileNumber" ], "GroupName": "MetadataGroup", "Autogenerated": false, "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ] }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Sun Jun 08 13:29:06 PDT 2014", "DatasetId": 14, "PendingOp": 0 }
\ No newline at end of file

diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 8cf4cd9..6344bc1 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml

@@ -4896,6 +4896,38 @@
       </compilation-unit>
     </test-case>
   </test-group>
+  <test-group name="external-indexing">
+  	<test-case FilePath="external-indexing">
+      <compilation-unit name="text-format">
+        <output-dir compare="Text">text-format</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="sequence-format">
+        <output-dir compare="Text">sequence-format</output-dir>
+      </compilation-unit>
+    </test-case>
+  	<test-case FilePath="external-indexing">
+      <compilation-unit name="rc-format">
+        <output-dir compare="Text">rc-format</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="rtree-index">
+        <output-dir compare="Text">rtree-index</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="leftouterjoin">
+        <output-dir compare="Text">leftouterjoin</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-indexing">
+      <compilation-unit name="leftouterjoin-rtree">
+        <output-dir compare="Text">leftouterjoin-rtree</output-dir>
+      </compilation-unit>
+    </test-case>
+  </test-group>
   <test-group name="temporal">
     <test-case FilePath="temporal">
      <compilation-unit name="parse_02">
commit	d905773df2460db0d163301f3d7685c972932204	[log] [tgz]
author	Abdullah Alamoudi <alamouda@uci.edu>	Thu Jun 12 13:38:27 2014 -0700
committer	Ian Maxon <imaxon@uci.edu>	Thu Jun 12 18:19:01 2014 -0800
tree	f6d31d0a1976d21465bc286be83ca67f908e739a
parent	d5f0fc5bb57f1b3b7bb2f647355d11c2fb137bba [diff]