Merge asterix-experiments to master
Change-Id: I79b91e045bd056e62adb6e48795a6625baee0062
Reviewed-on: https://asterix-gerrit.ics.uci.edu/642
Reviewed-by: Ian Maxon <imaxon@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/1.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/1.aql
new file mode 100644
index 0000000..0446799
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/1.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_a.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_a.aql
new file mode 100644
index 0000000..20e7ad5
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_a.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index BtreeExtraFieldIdx on Tweets(btree-extra-field1) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_b.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_b.aql
new file mode 100644
index 0000000..aec3bff
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_b.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index RtreeLocationIdx on Tweets(sender-location) type rtree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_c.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_c.aql
new file mode 100644
index 0000000..c3ea516
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_c.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index InvMessageTextIdx on Tweets(message-text) type ngram(3);
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_d.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_d.aql
new file mode 100644
index 0000000..ea9eb8d
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/2_d.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index InvMessageTextIdx on Tweets(message-text) type keyword;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/3.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/3.aql
new file mode 100644
index 0000000..a143dd1
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/3.aql
@@ -0,0 +1,7 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index BtreeExtraFieldIdx on Tweets(btree-extra-field1) type btree;
+create index RtreeLocationIdx on Tweets(sender-location) type rtree;
+create index InvMessageKeywordIdx on Tweets(message-text) type keyword;
+create index InvMessagetNgramIdx on Tweets(message-text) type ngram(3);
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_1.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_1.aql
new file mode 100644
index 0000000..c3e7425
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_1.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index BtreeExtraFieldIdx1 on Tweets(btree-extra-field1) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_2.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_2.aql
new file mode 100644
index 0000000..3f9e0d1
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_2.aql
@@ -0,0 +1,5 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index BtreeExtraFieldIdx1 on Tweets(btree-extra-field1) type btree;
+create index BtreeExtraFieldIdx2 on Tweets(btree-extra-field2) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_4.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_4.aql
new file mode 100644
index 0000000..44f230d
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_4.aql
@@ -0,0 +1,7 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index BtreeExtraFieldIdx1 on Tweets(btree-extra-field1) type btree;
+create index BtreeExtraFieldIdx2 on Tweets(btree-extra-field2) type btree;
+create index BtreeExtraFieldIdx3 on Tweets(btree-extra-field3) type btree;
+create index BtreeExtraFieldIdx4 on Tweets(btree-extra-field4) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_8.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_8.aql
new file mode 100644
index 0000000..f126aca
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/4_8.aql
@@ -0,0 +1,11 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index BtreeExtraFieldIdx1 on Tweets(btree-extra-field1) type btree;
+create index BtreeExtraFieldIdx2 on Tweets(btree-extra-field2) type btree;
+create index BtreeExtraFieldIdx3 on Tweets(btree-extra-field3) type btree;
+create index BtreeExtraFieldIdx4 on Tweets(btree-extra-field4) type btree;
+create index BtreeExtraFieldIdx5 on Tweets(btree-extra-field5) type btree;
+create index BtreeExtraFieldIdx6 on Tweets(btree-extra-field6) type btree;
+create index BtreeExtraFieldIdx7 on Tweets(btree-extra-field7) type btree;
+create index BtreeExtraFieldIdx8 on Tweets(btree-extra-field8) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_1.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_1.aql
new file mode 100644
index 0000000..20c0d3e
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_1.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+create dataset Tweets1(TweetMessageType) primary key tweetid;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_1_count.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_1_count.aql
new file mode 100644
index 0000000..fab917d2
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_1_count.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+let $count1 := count(for $t in dataset Tweets1 return $t.tweetid)
+return {"count1": $count1}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_2.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_2.aql
new file mode 100644
index 0000000..5efcfc1
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_2.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets1(TweetMessageType) primary key tweetid;
+create dataset Tweets2(TweetMessageType) primary key tweetid;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_2_count.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_2_count.aql
new file mode 100644
index 0000000..ce3d880
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_2_count.aql
@@ -0,0 +1,7 @@
+use dataverse experiments;
+
+let $count1 := count(for $t in dataset Tweets1 return $t.tweetid)
+return {"count1": $count1}
+
+let $count2 := count(for $t in dataset Tweets2 return $t.tweetid)
+return {"count2": $count2}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_3.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_3.aql
new file mode 100644
index 0000000..e6d9c30
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_3.aql
@@ -0,0 +1,5 @@
+use dataverse experiments;
+
+create dataset Tweets1(TweetMessageType) primary key tweetid;
+create dataset Tweets2(TweetMessageType) primary key tweetid;
+create dataset Tweets3(TweetMessageType) primary key tweetid;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_3_count.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_3_count.aql
new file mode 100644
index 0000000..fa2c97f
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_3_count.aql
@@ -0,0 +1,10 @@
+use dataverse experiments;
+
+let $count1 := count(for $t in dataset Tweets1 return $t.tweetid)
+return {"count1": $count1}
+
+let $count2 := count(for $t in dataset Tweets2 return $t.tweetid)
+return {"count2": $count2}
+
+let $count3 := count(for $t in dataset Tweets3 return $t.tweetid)
+return {"count3": $count3}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_4.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_4.aql
new file mode 100644
index 0000000..61d87ae
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_4.aql
@@ -0,0 +1,6 @@
+use dataverse experiments;
+
+create dataset Tweets1(TweetMessageType) primary key tweetid;
+create dataset Tweets2(TweetMessageType) primary key tweetid;
+create dataset Tweets3(TweetMessageType) primary key tweetid;
+create dataset Tweets4(TweetMessageType) primary key tweetid;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_4_count.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_4_count.aql
new file mode 100644
index 0000000..12b6210
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/5_4_count.aql
@@ -0,0 +1,13 @@
+use dataverse experiments;
+
+let $count1 := count(for $t in dataset Tweets1 return $t.tweetid)
+return {"count1": $count1}
+
+let $count2 := count(for $t in dataset Tweets2 return $t.tweetid)
+return {"count2": $count2}
+
+let $count3 := count(for $t in dataset Tweets3 return $t.tweetid)
+return {"count3": $count3}
+
+let $count4 := count(for $t in dataset Tweets4 return $t.tweetid)
+return {"count4": $count4}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_a.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_a.aql
new file mode 100644
index 0000000..0b41bc5
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_a.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="3"));
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_b.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_b.aql
new file mode 100644
index 0000000..ccd54d6
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_b.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy prefix(("max-mergable-component-size"="1073741824"),
+ ("max-tolerance-component-count"="5"));
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_c.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_c.aql
new file mode 100644
index 0000000..4022943
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/6_c.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="1000000"));
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_a.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_a.aql
new file mode 100644
index 0000000..0b41bc5
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_a.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="3"));
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_b.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_b.aql
new file mode 100644
index 0000000..ccd54d6
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_b.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy prefix(("max-mergable-component-size"="1073741824"),
+ ("max-tolerance-component-count"="5"));
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_c.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_c.aql
new file mode 100644
index 0000000..4022943
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_c.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="1000000"));
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_compact.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_compact.aql
new file mode 100644
index 0000000..578f0a4
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_compact.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+compact dataset Tweets;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_d.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_d.aql
new file mode 100644
index 0000000..4022943
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_d.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="1000000"));
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_q1.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_q1.aql
new file mode 100644
index 0000000..18b54ab
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_q1.aql
@@ -0,0 +1,8 @@
+use dataverse experiments;
+
+let $count := count(
+for $t in dataset Tweets
+where $t.tweetid = int64("1")
+return $t
+)
+return $count
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_q2.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_q2.aql
new file mode 100644
index 0000000..f691509
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/7_q2.aql
@@ -0,0 +1,8 @@
+use dataverse experiments;
+
+let $count := count(
+for $t in dataset Tweets
+where $t.tweetid >= int64("1") and $t.tweetid <= int64("1000")
+return $t
+)
+return $count
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_a.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_a.aql
new file mode 100644
index 0000000..38ee0a1
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_a.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="3"));
+create index BtreeExtraFieldIdx on Tweets(btree-extra-field1) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_b.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_b.aql
new file mode 100644
index 0000000..6eead85
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_b.aql
@@ -0,0 +1,5 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy prefix(("max-mergable-component-size"="1073741824"),
+ ("max-tolerance-component-count"="5"));
+create index BtreeExtraFieldIdx on Tweets(btree-extra-field1) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_c.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_c.aql
new file mode 100644
index 0000000..134f824
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_c.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="1000000"));
+create index BtreeExtraFieldIdx on Tweets(btree-extra-field1) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_compact.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_compact.aql
new file mode 100644
index 0000000..578f0a4
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_compact.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+
+compact dataset Tweets;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_d.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_d.aql
new file mode 100644
index 0000000..134f824
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_d.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+create dataset Tweets(TweetMessageType) primary key tweetid using compaction policy constant(("num-components"="1000000"));
+create index BtreeExtraFieldIdx on Tweets(btree-extra-field1) type btree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_q1.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_q1.aql
new file mode 100644
index 0000000..03bebb6
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_q1.aql
@@ -0,0 +1,8 @@
+use dataverse experiments;
+
+let $count := count(
+for $t in dataset Tweets
+where $t.tweetid = int64("$KEY$")
+return $t
+)
+return {"pointLookupCount": $count}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_q2.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_q2.aql
new file mode 100644
index 0000000..bd34067
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/8_q2.aql
@@ -0,0 +1,8 @@
+use dataverse experiments;
+
+let $count := count(
+for $t in dataset Tweets
+where $t.btree-extra-field1 > $LKEY$ and $t.btree-extra-field1 <= $HKEY$
+return $t
+)
+return {"rangeQueryCount": $count}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/avg_gram_tokens.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/avg_gram_tokens.aql
new file mode 100644
index 0000000..2daa62f
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/avg_gram_tokens.aql
@@ -0,0 +1,6 @@
+use dataverse experiments;
+
+let $avg := avg(for $t in dataset Tweets
+let $c := count(gram-tokens($t.message-text, 3, 1))
+return $c)
+return {"avg-grams-per-record": $avg}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/avg_keyword_tokens.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/avg_keyword_tokens.aql
new file mode 100644
index 0000000..a8dad49
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/avg_keyword_tokens.aql
@@ -0,0 +1,6 @@
+use dataverse experiments;
+
+let $avg := avg(for $t in dataset Tweets
+let $c := count(word-tokens($t.message-text))
+return $c)
+return {"avg-keywords-per-record": $avg}
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_1_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_1_ingest.aql
new file mode 100644
index 0000000..395f670
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_1_ingest.aql
@@ -0,0 +1,15 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001,128.195.9.24:10001,128.195.9.25:10001,
+ 128.195.9.26:10001,128.195.9.27:10001,128.195.9.28:10001,128.195.9.29:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets1 using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_2_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_2_ingest.aql
new file mode 100644
index 0000000..65f42cc
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_2_ingest.aql
@@ -0,0 +1,26 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001,128.195.9.24:10001,128.195.9.25:10001,
+ 128.195.9.26:10001,128.195.9.27:10001,128.195.9.28:10001,128.195.9.29:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10002,128.195.9.23:10002,128.195.9.24:10002,128.195.9.25:10002,
+ 128.195.9.26:10002,128.195.9.27:10002,128.195.9.28:10002,128.195.9.29:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets1 using policy Br;
+connect feed TweetFeed2 to dataset Tweets2 using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_3_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_3_ingest.aql
new file mode 100644
index 0000000..945332d
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_3_ingest.aql
@@ -0,0 +1,37 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001,128.195.9.24:10001,128.195.9.25:10001,
+ 128.195.9.26:10001,128.195.9.27:10001,128.195.9.28:10001,128.195.9.29:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10002,128.195.9.23:10002,128.195.9.24:10002,128.195.9.25:10002,
+ 128.195.9.26:10002,128.195.9.27:10002,128.195.9.28:10002,128.195.9.29:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed3 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10003,128.195.9.23:10003,128.195.9.24:10003,128.195.9.25:10003,
+ 128.195.9.26:10003,128.195.9.27:10003,128.195.9.28:10003,128.195.9.29:10003"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets1 using policy Br;
+connect feed TweetFeed2 to dataset Tweets2 using policy Br;
+connect feed TweetFeed3 to dataset Tweets3 using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_4_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_4_ingest.aql
new file mode 100644
index 0000000..a1112b1
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/5_4_ingest.aql
@@ -0,0 +1,48 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001,128.195.9.24:10001,128.195.9.25:10001,
+ 128.195.9.26:10001,128.195.9.27:10001,128.195.9.28:10001,128.195.9.29:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10002,128.195.9.23:10002,128.195.9.24:10002,128.195.9.25:10002,
+ 128.195.9.26:10002,128.195.9.27:10002,128.195.9.28:10002,128.195.9.29:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed3 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10003,128.195.9.23:10003,128.195.9.24:10003,128.195.9.25:10003,
+ 128.195.9.26:10003,128.195.9.27:10003,128.195.9.28:10003,128.195.9.29:10003"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed4 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10004,128.195.9.23:10004,128.195.9.24:10004,128.195.9.25:10004,
+ 128.195.9.26:10004,128.195.9.27:10004,128.195.9.28:10004,128.195.9.29:10004"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets1 using policy Br;
+connect feed TweetFeed2 to dataset Tweets2 using policy Br;
+connect feed TweetFeed3 to dataset Tweets3 using policy Br;
+connect feed TweetFeed4 to dataset Tweets4 using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_1_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_1_ingest.aql
new file mode 100644
index 0000000..8f5654d
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_1_ingest.aql
@@ -0,0 +1,24 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="127.0.0.1:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="127.0.0.1:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets using policy Br;
+connect feed TweetFeed2 to dataset Tweets using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_1_ingest_query.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_1_ingest_query.aql
new file mode 100644
index 0000000..c7614dc
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_1_ingest_query.aql
@@ -0,0 +1,24 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="127.0.0.1:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="127.0.0.1:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets using policy Brittle;
+connect feed TweetFeed2 to dataset Tweets using policy Brittle;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_2_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_2_ingest.aql
new file mode 100644
index 0000000..ba4c18e
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_2_ingest.aql
@@ -0,0 +1,24 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10002,128.195.9.23:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets using policy Br;
+connect feed TweetFeed2 to dataset Tweets using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_4_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_4_ingest.aql
new file mode 100644
index 0000000..c5dd828
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_4_ingest.aql
@@ -0,0 +1,24 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001,128.195.9.24:10001,128.195.9.25:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10002,128.195.9.23:10002,128.195.9.24:10002,128.195.9.25:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets using policy Br;
+connect feed TweetFeed2 to dataset Tweets using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_8_ingest.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_8_ingest.aql
new file mode 100644
index 0000000..6d337bd
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_8_ingest.aql
@@ -0,0 +1,24 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001,128.195.9.24:10001,128.195.9.25:10001,128.195.9.26:10001,128.195.9.27:10001,128.195.9.28:10001,128.195.9.29:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+create feed TweetFeed2 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10002,128.195.9.23:10002,128.195.9.24:10002,128.195.9.25:10002,128.195.9.26:10002,128.195.9.27:10002,128.195.9.28:10002,128.195.9.29:10002"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets using policy Br;
+connect feed TweetFeed2 to dataset Tweets using policy Br;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_8_ingest_query.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_8_ingest_query.aql
new file mode 100644
index 0000000..0d057a2
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_8_ingest_query.aql
@@ -0,0 +1,14 @@
+use dataverse experiments;
+
+create feed TweetFeed1 using socket_adapter
+(
+ ("sockets"="128.195.9.22:10001,128.195.9.23:10001,128.195.9.24:10001,128.195.9.25:10001,128.195.9.26:10001,128.195.9.27:10001,128.195.9.28:10001,128.195.9.29:10001"),
+ ("address-type"="IP"),
+ ("type-name"="TweetMessageType"),
+ ("format"="adm"),
+ ("duration"="1200")
+);
+
+set wait-for-completion-feed "false";
+
+connect feed TweetFeed1 to dataset Tweets using policy Brittle;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_types.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_types.aql
new file mode 100644
index 0000000..6a1168c
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/base/base_types.aql
@@ -0,0 +1,31 @@
+drop dataverse experiments if exists;
+create dataverse experiments;
+use dataverse experiments;
+
+create type TwitterUserType as closed {
+ screen-name: string,
+ lang: string,
+ friends_count: int32,
+ statuses_count: int32,
+ name: string,
+ followers_count: int32
+}
+
+create type TweetMessageType as closed {
+ tweetid: int64,
+ user: TwitterUserType,
+ sender-location: point,
+ send-time: datetime,
+ referred-topics: {{ string }},
+ message-text: string,
+
+ btree-extra-field1: int32,
+ btree-extra-field2: int32,
+ btree-extra-field3: int32,
+ btree-extra-field4: int32,
+ btree-extra-field5: int32,
+ btree-extra-field6: int32,
+ btree-extra-field7: int32,
+ btree-extra-field8: int32,
+ dummy-size-adjuster: string // 357
+}
\ No newline at end of file
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/count.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/count.aql
new file mode 100644
index 0000000..a913831
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/count.aql
@@ -0,0 +1,4 @@
+use dataverse experiments;
+
+let $count := count(for $t in dataset Tweets return $t.tweetid)
+return $count;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_dhbtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_dhbtree.aql
new file mode 100644
index 0000000..755e5d7
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_dhbtree.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index dhbtreeLocation on Tweets(sender-location) type dhbtree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_dhvbtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_dhvbtree.aql
new file mode 100644
index 0000000..b089c37
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_dhvbtree.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index dhvbtreeLocation on Tweets(sender-location) type dhvbtree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_rtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_rtree.aql
new file mode 100644
index 0000000..f46a27a
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_rtree.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index rtreeLocation on Tweets(sender-location) type rtree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_shbtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_shbtree.aql
new file mode 100644
index 0000000..d2bbe28
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_shbtree.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index shbtreeLocation on Tweets(sender-location) type shbtree(-180.0, -90.0, 180.0, 90.0);
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_sif.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_sif.aql
new file mode 100644
index 0000000..fce9298
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_1_sif.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create index sifLocation on Tweets(sender-location) type sif(-180.0, -90.0, 180.0, 90.0);
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3.aql
new file mode 100644
index 0000000..78b63c6
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3.aql
@@ -0,0 +1,3 @@
+use dataverse experiments;
+create dataset Tweets(TweetMessageType) primary key tweetid;
+create dataset JoinSeedTweets(TweetMessageType) primary key tweetid;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_dhbtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_dhbtree.aql
new file mode 100644
index 0000000..36060d7
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_dhbtree.aql
@@ -0,0 +1,2 @@
+use dataverse experiments;
+create index dhbtreeLocation on Tweets(sender-location) type dhbtree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_dhvbtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_dhvbtree.aql
new file mode 100644
index 0000000..2369814
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_dhvbtree.aql
@@ -0,0 +1,2 @@
+use dataverse experiments;
+create index dhvbtreeLocation on Tweets(sender-location) type dhvbtree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_rtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_rtree.aql
new file mode 100644
index 0000000..d46cd20
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_rtree.aql
@@ -0,0 +1,2 @@
+use dataverse experiments;
+create index rtreeLocation on Tweets(sender-location) type rtree;
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_shbtree.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_shbtree.aql
new file mode 100644
index 0000000..4f83444
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_shbtree.aql
@@ -0,0 +1,2 @@
+use dataverse experiments;
+create index shbtreeLocation on Tweets(sender-location) type shbtree(-180.0, -90.0, 180.0, 90.0);
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_sif.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_sif.aql
new file mode 100644
index 0000000..7faf699
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_create_sif.aql
@@ -0,0 +1,2 @@
+use dataverse experiments;
+create index sifLocation on Tweets(sender-location) type sif(-180.0, -90.0, 180.0, 90.0);
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_pidx_load.aql b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_pidx_load.aql
new file mode 100644
index 0000000..dce43c5
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/aql/spatial_3_pidx_load.aql
@@ -0,0 +1,26 @@
+use dataverse experiments;
+
+load dataset JoinSeedTweets
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="128.195.9.22:///scratch/youngsk2/spatial-index-experiment/data/QuerySeedTweets10K-from-SyntheticTweets100M-psi27-pid0.adm"),("format"="adm"));
+
+/* for testing */
+/*
+load dataset Tweets
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="128.195.9.22:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets2M-psi27-pid0.adm"),("format"="adm"));
+*/
+
+/* for measurement */
+load dataset Tweets
+using "org.apache.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"=
+"128.195.9.22:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi27-pid0.adm,
+128.195.9.23:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi26-pid1.adm,
+128.195.9.24:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi25-pid2.adm,
+128.195.9.25:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi24-pid3.adm,
+128.195.9.26:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi23-pid4.adm,
+128.195.9.27:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi22-pid5.adm,
+128.195.9.28:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi21-pid6.adm,
+128.195.9.29:///scratch/youngsk2/spatial-index-experiment/data/SyntheticTweets100M-psi20-pid7.adm"),
+("format"="adm"));
\ No newline at end of file
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/balloon.c b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/balloon.c
new file mode 100644
index 0000000..d109be8
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/balloon.c
@@ -0,0 +1,22 @@
+#include <stdio.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <stdint.h>
+
+ int main() {
+
+ void *mem;
+ int err;
+ uint64_t size = 4.5 * 1024 * 1024;
+ size *= 1024;
+
+ mem = mmap(NULL, size, PROT_READ|PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+ printf ("%d\n", mem);
+ err = mlock(mem, size);
+ printf ("err is %d\n", err);
+ while(1) {
+ sleep(10000);
+ }
+ return 0;
+ }
+
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/1node.xml b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/1node.xml
new file mode 100644
index 0000000..39c386f
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/1node.xml
@@ -0,0 +1,39 @@
+<!--
+ ! Copyright 2009-2013 by The Regents of the University of California
+ ! Licensed under the Apache License, Version 2.0 (the "License");
+ ! you may not use this file except in compliance with the License.
+ ! you may obtain a copy of the License from
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
+<cluster xmlns="cluster">
+ <name>1node</name>
+ <username>jenkins</username>
+ <working_dir>
+ <dir>/home/jenkins/asterixdb/working</dir>
+ <NFS>true</NFS>
+ </working_dir>
+ <log_dir>/home/jenkins/asterix/logs</log_dir>
+ <txn_log_dir>/home/jenkins/asterix/txnlogs</txn_log_dir>
+ <iodevices>/home/jenkins/asterix</iodevices>
+ <store>storage</store>
+ <java_home>/usr/lib/jvm/jre-1.8.0</java_home>
+ <master_node>
+ <id>master</id>
+ <cluster_ip>127.0.0.1</cluster_ip>
+ <cluster_port>1099</cluster_port>
+ <client_ip>127.0.0.1</client_ip>
+ <client_port>1098</client_port>
+ <http_port>8888</http_port>
+ </master_node>
+ <node>
+ <id>node1</id>
+ <cluster_ip>127.0.0.1</cluster_ip>
+ </node>
+</cluster>
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/2node.xml b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/2node.xml
new file mode 100644
index 0000000..bc027c4
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/2node.xml
@@ -0,0 +1,44 @@
+<!--
+ ! Copyright 2009-2013 by The Regents of the University of California
+ ! Licensed under the Apache License, Version 2.0 (the "License");
+ ! you may not use this file except in compliance with the License.
+ ! you may obtain a copy of the License from
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
+<cluster xmlns="cluster">
+ <name>2node</name>
+ <username>youngsk2</username>
+ <working_dir>
+ <dir>/home/youngsk2/asterixdb/working</dir>
+ <NFS>true</NFS>
+ </working_dir>
+ <log_dir>/mnt/data/sde/youngsk2/asterix/logs</log_dir>
+ <txn_log_dir>/lv_scratch/scratch/youngsk2/asterix/txnlogs</txn_log_dir>
+ <iodevices>/mnt/data/sdb/youngsk2/asterix,/mnt/data/sdc/youngsk2/asterix,/mnt/data/sdd/youngsk2/asterix</iodevices>
+ <store>storage</store>
+ <java_home>/home/youngsk2/jdk1.7.0_65</java_home>
+ <master_node>
+ <id>master</id>
+ <cluster_ip>128.195.9.21</cluster_ip>
+ <cluster_port>1099</cluster_port>
+ <client_ip>128.195.9.21</client_ip>
+ <client_port>1098</client_port>
+ <http_port>8888</http_port>
+ </master_node>
+ <node>
+ <id>node1</id>
+ <cluster_ip>128.195.9.22</cluster_ip>
+ </node>
+ <node>
+ <id>node2</id>
+ <cluster_ip>128.195.9.23</cluster_ip>
+ </node>
+</cluster>
+
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/4node.xml b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/4node.xml
new file mode 100644
index 0000000..aabb3f5
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/4node.xml
@@ -0,0 +1,52 @@
+<!--
+ ! Copyright 2009-2013 by The Regents of the University of California
+ ! Licensed under the Apache License, Version 2.0 (the "License");
+ ! you may not use this file except in compliance with the License.
+ ! you may obtain a copy of the License from
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
+<cluster xmlns="cluster">
+ <name>4node</name>
+ <username>youngsk2</username>
+ <working_dir>
+ <dir>/home/youngsk2/asterixdb/working</dir>
+ <NFS>true</NFS>
+ </working_dir>
+ <log_dir>/mnt/data/sde/youngsk2/asterix/logs</log_dir>
+ <txn_log_dir>/lv_scratch/scratch/youngsk2/asterix/txnlogs</txn_log_dir>
+ <iodevices>/mnt/data/sdb/youngsk2/asterix,/mnt/data/sdc/youngsk2/asterix,/mnt/data/sdd/youngsk2/asterix</iodevices>
+ <store>storage</store>
+ <java_home>/home/youngsk2/jdk1.7.0_65</java_home>
+ <master_node>
+ <id>master</id>
+ <cluster_ip>128.195.9.21</cluster_ip>
+ <cluster_port>1099</cluster_port>
+ <client_ip>128.195.9.21</client_ip>
+ <client_port>1098</client_port>
+ <http_port>8888</http_port>
+ </master_node>
+ <node>
+ <id>node1</id>
+ <cluster_ip>128.195.9.22</cluster_ip>
+ </node>
+ <node>
+ <id>node2</id>
+ <cluster_ip>128.195.9.23</cluster_ip>
+ </node>
+ <node>
+ <id>node3</id>
+ <cluster_ip>128.195.9.24</cluster_ip>
+ </node>
+ <node>
+ <id>node4</id>
+ <cluster_ip>128.195.9.25</cluster_ip>
+ </node>
+</cluster>
+
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/8node.xml b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/8node.xml
new file mode 100644
index 0000000..1586f5a
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/8node.xml
@@ -0,0 +1,68 @@
+<!--
+ ! Copyright 2009-2013 by The Regents of the University of California
+ ! Licensed under the Apache License, Version 2.0 (the "License");
+ ! you may not use this file except in compliance with the License.
+ ! you may obtain a copy of the License from
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
+<cluster xmlns="cluster">
+ <name>8node</name>
+ <username>youngsk2</username>
+ <working_dir>
+ <dir>/home/youngsk2/asterixdb/working</dir>
+ <NFS>true</NFS>
+ </working_dir>
+ <log_dir>/mnt/data/sde/youngsk2/asterix/logs</log_dir>
+ <txn_log_dir>/lv_scratch/scratch/youngsk2/asterix/txnlogs</txn_log_dir>
+ <iodevices>/mnt/data/sdb/youngsk2/asterix,/mnt/data/sdc/youngsk2/asterix,/mnt/data/sdd/youngsk2/asterix</iodevices>
+ <store>storage</store>
+ <java_home>/home/youngsk2/jdk1.7.0_65</java_home>
+ <master_node>
+ <id>master</id>
+ <cluster_ip>128.195.9.21</cluster_ip>
+ <cluster_port>1099</cluster_port>
+ <client_ip>128.195.9.21</client_ip>
+ <client_port>1098</client_port>
+ <http_port>8888</http_port>
+ </master_node>
+ <node>
+ <id>node1</id>
+ <cluster_ip>128.195.9.22</cluster_ip>
+ </node>
+ <node>
+ <id>node2</id>
+ <cluster_ip>128.195.9.23</cluster_ip>
+ </node>
+ <node>
+ <id>node3</id>
+ <cluster_ip>128.195.9.24</cluster_ip>
+ </node>
+ <node>
+ <id>node4</id>
+ <cluster_ip>128.195.9.25</cluster_ip>
+ </node>
+ <node>
+ <id>node5</id>
+ <cluster_ip>128.195.9.26</cluster_ip>
+ </node>
+ <node>
+ <id>node6</id>
+ <cluster_ip>128.195.9.27</cluster_ip>
+ </node>
+ <node>
+ <id>node7</id>
+ <cluster_ip>128.195.9.28</cluster_ip>
+ </node>
+ <node>
+ <id>node8</id>
+ <cluster_ip>128.195.9.29</cluster_ip>
+ </node>
+</cluster>
+
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
new file mode 100644
index 0000000..0159194
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/asterix-configuration.xml
@@ -0,0 +1,259 @@
+<!--
+ ! Copyright 2009-2013 by The Regents of the University of California
+ ! Licensed under the Apache License, Version 2.0 (the "License");
+ ! you may not use this file except in compliance with the License.
+ ! you may obtain a copy of the License from
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
+<asterixConfiguration xmlns="asterixconf">
+
+ <property>
+ <name>nc.java.opts</name>
+ <value>-Xms6144m -Xmx7168m</value>
+ <description>JVM parameters for each Node Contoller (NC)</description>
+ </property>
+
+ <property>
+ <name>cc.java.opts</name>
+ <value>-Xms6144m -Xmx6144m</value>
+ <description>JVM parameters for each Cluster Contoller (CC)
+ </description>
+ </property>
+
+ <property>
+ <name>max.wait.active.cluster</name>
+ <value>60</value>
+ <description>Maximum wait (in seconds) for a cluster to be ACTIVE (all nodes are available)
+ before a submitted query/statement can be executed. (Default = 60 seconds)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.buffercache.pagesize</name>
+ <value>131072</value>
+ <description>The page size in bytes for pages in the buffer cache.
+ (Default = "131072" // 128KB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.buffercache.size</name>
+ <value>3221225472</value>
+ <description>[3GB] The size of memory allocated to the disk buffer cache.
+ The value should be a multiple of the buffer cache page size(Default
+ = "536870912" // 512MB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.buffercache.maxopenfiles</name>
+ <value>214748364</value>
+ <description>The maximum number of open files in the buffer cache.
+ (Default = "214748364")
+ </description>
+ </property>
+
+ <!-- Buffer size per dataset for in-memory components: 1GB -->
+ <property>
+ <name>storage.memorycomponent.pagesize</name>
+ <value>131072</value>
+ <description>The page size in bytes for pages allocated to memory
+ components. (Default = "131072" // 128KB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.numpages</name>
+ <value>8192</value>
+ <description>
+ </description>
+ </property>
+ <!--// Buffer size per dataset for in-memory components -->
+
+ <property>
+ <name>storage.metadata.memorycomponent.numpages</name>
+ <value>64</value>
+ <description>The number of pages to allocate for a memory component.
+ (Default = 64)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.numcomponents</name>
+ <value>2</value>
+ <description>The number of memory components to be used per lsm index.
+ (Default = 2)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>4399824896</value>
+ <description>[4GB + 8MB]The total size of memory in bytes that the sum of all
+ open memory
+ components cannot exceed. (Default = "536870192" // 512MB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.lsm.bloomfilter.falsepositiverate</name>
+ <value>0.01</value>
+ <description>The maximum acceptable false positive rate for bloom
+ filters associated with LSM indexes. (Default = "0.01" // 1%)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.buffer.numpages</name>
+ <value>2</value>
+ <description>The number of in-memory log buffer pages. (Default = "8")
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.buffer.pagesize</name>
+ <value>8388608</value>
+ <description>[8MB]The size of pages in the in-memory log buffer. (Default =
+ "524288" // 512KB)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.partitionsize</name>
+ <value>2147483648</value>
+ <description>The maximum size of a log file partition allowed before
+ rotating the log to the next partition. (Default = "2147483648" //
+ 2GB)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.checkpoint.lsnthreshold</name>
+ <value>67108864</value>
+ <description>The size of the window that the maximum LSN is allowed to
+ be ahead of the checkpoint LSN by. (Default = ""67108864" // 64M)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.checkpoint.pollfrequency</name>
+ <value>120</value>
+ <description>The time in seconds between that the checkpoint thread
+ waits between polls. (Default = "120" // 120s)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.checkpoint.history</name>
+ <value>0</value>
+ <description>The number of old log partition files to keep before
+ discarding. (Default = "0")
+ </description>
+ </property>
+
+ <property>
+ <name>txn.lock.escalationthreshold</name>
+ <value>1000</value>
+ <description>The number of entity level locks that need to be acquired
+ before the locks are coalesced and escalated into a dataset level
+ lock. (Default = "1000")
+ </description>
+ </property>
+
+ <property>
+ <name>txn.lock.shrinktimer</name>
+ <value>5000</value>
+ <description>The time in milliseconds to wait before deallocating
+ unused lock manager memory. (Default = "5000" // 5s)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.lock.timeout.waitthreshold</name>
+ <value>60000</value>
+ <description>The time in milliseconds to wait before labeling a
+ transaction which has been waiting for a lock timed-out. (Default =
+ "60000" // 60s)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.lock.timeout.sweepthreshold</name>
+ <value>10000</value>
+ <description>The time in milliseconds the timeout thread waits between
+ sweeps to detect timed-out transactions. (Default = "10000" // 10s)
+ </description>
+ </property>
+
+ <property>
+ <name>compiler.sortmemory</name>
+ <value>67108864</value>
+ <description>[64MB]The amount of memory in bytes given to sort operations.
+ (Default = "33554432" // 32mb)
+ </description>
+ </property>
+
+ <property>
+ <name>compiler.joinmemory</name>
+ <value>67108864</value>
+ <description>[64MB]The amount of memory in bytes given to join operations.
+ (Default = "33554432" // 32mb)
+ </description>
+ </property>
+
+ <property>
+ <name>compiler.framesize</name>
+ <value>131072</value>
+ <description>The Hyracks frame size that the compiler configures per
+ job. (Default = "131072" // 128KB)
+ </description>
+ </property>
+
+ <property>
+ <name>web.port</name>
+ <value>19001</value>
+ <description>The port for the ASTERIX web interface. (Default = 19001)
+ </description>
+ </property>
+
+ <property>
+ <name>api.port</name>
+ <value>19002</value>
+ <description>The port for the ASTERIX API server. (Default = 19002)
+ </description>
+ </property>
+
+ <property>
+ <name>log.level</name>
+ <value>ALL</value>
+ <description>The minimum log level to be displayed. (Default = INFO)
+ </description>
+ </property>
+ <property>
+ <name>feed.memory.global.budget</name>
+ <value>1073741824</value>
+ <description>Feed memory budget (1 GB = 1073741824 Bytes)
+ </description>
+ </property>
+
+ <property>
+ <name>feed.pending.work.threshold</name>
+ <value>1000</value>
+ <description>Feed pending work threshold
+ </description>
+ </property>
+
+ <property>
+ <name>feed.max.threshold.period</name>
+ <value>100</value>
+ <description>Feed max threshold period
+ </description>
+ </property>
+</asterixConfiguration>
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/managix-conf.xml b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/managix-conf.xml
new file mode 100644
index 0000000..a21935f
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/configs/managix-conf.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<configuration xmlns="installer">
+ <backup>
+ <hdfs>
+ <version>0.20.2</version>
+ <url></url>
+ </hdfs>
+ <backupDir>/home/youngsk2/managix/clusters/local/working_dir/backup</backupDir>
+ </backup>
+ <zookeeper>
+ <homeDir>/home/youngsk2/managix/.installer/zookeeper_home</homeDir>
+ <clientPort>2900</clientPort>
+ <servers>
+ <java_home>/home/youngsk2/jdk1.7.0_65</java_home>
+ <server>128.195.9.21</server>
+ </servers>
+ </zookeeper>
+</configuration>
\ No newline at end of file
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/1.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/1.dgen
new file mode 100644
index 0000000..428b3a3
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/1.dgen
@@ -0,0 +1 @@
+127.0.0.1 127.0.0.1:10001
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/2.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/2.dgen
new file mode 100644
index 0000000..7a51869
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/2.dgen
@@ -0,0 +1,2 @@
+128.195.11.61 128.195.9.22:10001
+128.195.11.62 128.195.9.23:10001
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/4.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/4.dgen
new file mode 100644
index 0000000..0472cf6
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/4.dgen
@@ -0,0 +1,4 @@
+128.195.11.61 128.195.9.22:10001
+128.195.11.62 128.195.9.23:10001
+128.195.11.63 128.195.9.24:10001
+128.195.11.64 128.195.9.25:10001
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_1.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_1.dgen
new file mode 100644
index 0000000..e671eca
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_1.dgen
@@ -0,0 +1,8 @@
+128.195.9.30 128.195.9.22:10001
+128.195.9.30 128.195.9.23:10001
+128.195.9.21 128.195.9.24:10001
+128.195.9.21 128.195.9.25:10001
+128.195.11.54 128.195.9.26:10001
+128.195.11.54 128.195.9.27:10001
+128.195.11.55 128.195.9.28:10001
+128.195.11.55 128.195.9.29:10001
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_2.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_2.dgen
new file mode 100644
index 0000000..e82412a
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_2.dgen
@@ -0,0 +1,16 @@
+128.195.9.30 128.195.9.22:10001
+128.195.9.30 128.195.9.23:10001
+128.195.9.21 128.195.9.24:10001
+128.195.9.21 128.195.9.25:10001
+128.195.11.54 128.195.9.26:10001
+128.195.11.54 128.195.9.27:10001
+128.195.11.55 128.195.9.28:10001
+128.195.11.55 128.195.9.29:10001
+128.195.11.56 128.195.9.22:10002
+128.195.11.56 128.195.9.23:10002
+128.195.11.57 128.195.9.24:10002
+128.195.11.57 128.195.9.25:10002
+128.195.11.59 128.195.9.26:10002
+128.195.11.59 128.195.9.27:10002
+128.195.11.60 128.195.9.28:10002
+128.195.11.60 128.195.9.29:10002
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_3.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_3.dgen
new file mode 100644
index 0000000..5846b08
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_3.dgen
@@ -0,0 +1,24 @@
+128.195.9.30 128.195.9.22:10001
+128.195.9.30 128.195.9.23:10001
+128.195.9.21 128.195.9.24:10001
+128.195.9.21 128.195.9.25:10001
+128.195.11.54 128.195.9.26:10001
+128.195.11.54 128.195.9.27:10001
+128.195.11.55 128.195.9.28:10001
+128.195.11.55 128.195.9.29:10001
+128.195.11.56 128.195.9.22:10002
+128.195.11.56 128.195.9.23:10002
+128.195.11.57 128.195.9.24:10002
+128.195.11.57 128.195.9.25:10002
+128.195.11.59 128.195.9.26:10002
+128.195.11.59 128.195.9.27:10002
+128.195.11.60 128.195.9.28:10002
+128.195.11.60 128.195.9.29:10002
+128.195.11.62 128.195.9.22:10003
+128.195.11.62 128.195.9.23:10003
+128.195.11.63 128.195.9.24:10003
+128.195.11.63 128.195.9.25:10003
+128.195.11.64 128.195.9.26:10003
+128.195.11.64 128.195.9.27:10003
+128.195.11.65 128.195.9.28:10003
+128.195.11.65 128.195.9.29:10003
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_4.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_4.dgen
new file mode 100644
index 0000000..ce6dbb2
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/5_4.dgen
@@ -0,0 +1,32 @@
+128.195.9.30 128.195.9.22:10001
+128.195.9.30 128.195.9.23:10001
+128.195.9.21 128.195.9.24:10001
+128.195.9.21 128.195.9.25:10001
+128.195.11.54 128.195.9.26:10001
+128.195.11.54 128.195.9.27:10001
+128.195.11.55 128.195.9.28:10001
+128.195.11.55 128.195.9.29:10001
+128.195.11.56 128.195.9.22:10002
+128.195.11.56 128.195.9.23:10002
+128.195.11.57 128.195.9.24:10002
+128.195.11.57 128.195.9.25:10002
+128.195.11.59 128.195.9.26:10002
+128.195.11.59 128.195.9.27:10002
+128.195.11.60 128.195.9.28:10002
+128.195.11.60 128.195.9.29:10002
+128.195.11.62 128.195.9.22:10003
+128.195.11.62 128.195.9.23:10003
+128.195.11.63 128.195.9.24:10003
+128.195.11.63 128.195.9.25:10003
+128.195.11.64 128.195.9.26:10003
+128.195.11.64 128.195.9.27:10003
+128.195.11.65 128.195.9.28:10003
+128.195.11.65 128.195.9.29:10003
+128.195.11.66 128.195.9.22:10004
+128.195.11.66 128.195.9.23:10004
+128.195.11.68 128.195.9.24:10004
+128.195.11.68 128.195.9.25:10004
+128.195.11.72 128.195.9.26:10004
+128.195.11.72 128.195.9.27:10004
+128.195.11.86 128.195.9.28:10004
+128.195.11.86 128.195.9.29:10004
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/8.dgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/8.dgen
new file mode 100644
index 0000000..8571cbe
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/8.dgen
@@ -0,0 +1,8 @@
+128.195.11.61 128.195.9.22:10001
+128.195.11.62 128.195.9.23:10001
+128.195.11.63 128.195.9.24:10001
+128.195.11.64 128.195.9.25:10001
+128.195.11.65 128.195.9.26:10001
+128.195.11.66 128.195.9.27:10001
+128.195.11.67 128.195.9.28:10001
+128.195.11.68 128.195.9.29:10001
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/8.dqgen b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/8.dqgen
new file mode 100644
index 0000000..8571cbe
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/dgen/8.dqgen
@@ -0,0 +1,8 @@
+128.195.11.61 128.195.9.22:10001
+128.195.11.62 128.195.9.23:10001
+128.195.11.63 128.195.9.24:10001
+128.195.11.64 128.195.9.25:10001
+128.195.11.65 128.195.9.26:10001
+128.195.11.66 128.195.9.27:10001
+128.195.11.67 128.195.9.28:10001
+128.195.11.68 128.195.9.29:10001
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/killdrivers.sh b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/killdrivers.sh
new file mode 100644
index 0000000..9e1e7b0
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/killdrivers.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+jps | grep Driver | awk '{print $1}' | xargs kill -9
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/killscreens.sh b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/killscreens.sh
new file mode 100644
index 0000000..9b6e5f9
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/killscreens.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+screen -ls | grep asterix | cut -d. -f1 | awk '{print $1}' | xargs kill -9
+screen -wipe
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/logging.properties b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/logging.properties
new file mode 100644
index 0000000..e3c448c
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/logging.properties
@@ -0,0 +1,84 @@
+#/*
+# Copyright 2009-2013 by The Regents of the University of California
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# you may obtain a copy of the License from
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#*/
+############################################################
+# Default Logging Configuration File
+#
+# You can use a different file by specifying a filename
+# with the java.util.logging.config.file system property.
+# For example java -Djava.util.logging.config.file=myfile
+############################################################
+
+############################################################
+# Global properties
+############################################################
+
+# "handlers" specifies a comma separated list of log Handler
+# classes. These handlers will be installed during VM startup.
+# Note that these classes must be on the system classpath.
+# By default we only configure a ConsoleHandler, which will only
+# show messages at the INFO and above levels.
+
+handlers= java.util.logging.ConsoleHandler
+
+# To also add the FileHandler, use the following line instead.
+
+# handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler
+
+# Default global logging level.
+# This specifies which kinds of events are logged across
+# all loggers. For any given facility this global level
+# can be overriden by a facility specific level
+# Note that the ConsoleHandler also has a separate level
+# setting to limit messages printed to the console.
+
+.level= INFO
+# .level= INFO
+# .level= FINE
+# .level = FINEST
+
+############################################################
+# Handler specific properties.
+# Describes specific configuration info for Handlers.
+############################################################
+
+# default file output is in user's home directory.
+
+# java.util.logging.FileHandler.pattern = %h/java%u.log
+# java.util.logging.FileHandler.limit = 50000
+# java.util.logging.FileHandler.count = 1
+# java.util.logging.FileHandler.formatter = java.util.logging.XMLFormatter
+
+# Limit the message that are printed on the console to FINE and above.
+
+java.util.logging.ConsoleHandler.level = INFO
+java.util.logging.ConsoleHandler.formatter = org.apache.asterix.experiment.logging.ExperimentLogFormatter
+#java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+
+
+############################################################
+# Facility specific properties.
+# Provides extra control for each logger.
+############################################################
+
+# For example, set the com.xyz.foo logger to only log SEVERE
+# messages:
+
+
+#org.apache.asterix.test.level = INFO
+#org.apache.asterix.level = INFO
+#org.apache.hyracks.algebricks.level = FINE
+#org.apache.hyracks.level = INFO
+#org.apache.asterix.test = INFO
+#org.apache.asterix.installer.test = INFO
diff --git a/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/updateMain.sh b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/updateMain.sh
new file mode 100644
index 0000000..9fc1676
--- /dev/null
+++ b/asterix-experiments/src/main/resources/ingestion-experiment-binary-and-configs/updateMain.sh
@@ -0,0 +1,6 @@
+rm -rf /scratch/youngsk2/spatial-index-experiment/ingestion-experiment-root/ingestion-experiment-binary-and-configs/
+unzip /home/youngsk2/spatial-index-experiment/ingestion-experiment-binary-and-configs/ingestion-experiment-binary-and-configs.zip -d /scratch/youngsk2/spatial-index-experiment/ingestion-experiment-root/
+unzip /home/youngsk2/spatial-index-experiment/ingestion-experiment-binary-and-configs/asterix-experiments-0.8.7-SNAPSHOT-binary-assembly.zip -d /scratch/youngsk2/spatial-index-experiment/ingestion-experiment-root/ingestion-experiment-binary-and-configs/;
+rm -rf /scratch/youngsk2/spatial-index-experiment/asterix-instance/* /scratch/youngsk2/spatial-index-experiment/asterix-instance/.installer
+unzip /home/youngsk2/spatial-index-experiment/ingestion-experiment-binary-and-configs/asterix-installer-0.8.7-SNAPSHOT-binary-assembly.zip -d /scratch/youngsk2/spatial-index-experiment/asterix-instance/;
+cp -rf /home/youngsk2/spatial-index-experiment/ingestion-experiment-binary-and-configs/managix-conf.xml /scratch/youngsk2/spatial-index-experiment/asterix-instance/conf/