Added asterix project
git-svn-id: https://asterixdb.googlecode.com/svn/trunk/asterix@12 eaa15691-b419-025a-1212-ee371bd00084
diff --git a/asterix-tools/src/main/assembly/binary-assembly.xml b/asterix-tools/src/main/assembly/binary-assembly.xml
new file mode 100644
index 0000000..68d424a
--- /dev/null
+++ b/asterix-tools/src/main/assembly/binary-assembly.xml
@@ -0,0 +1,19 @@
+<assembly>
+ <id>binary-assembly</id>
+ <formats>
+ <format>zip</format>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>target/appassembler/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>target/appassembler/lib</directory>
+ <outputDirectory>lib</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdgClientDriver.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdgClientDriver.java
new file mode 100644
index 0000000..3fd6fbc
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdgClientDriver.java
@@ -0,0 +1,34 @@
+package edu.uci.ics.asterix.tools.datagen;
+
+import java.io.File;
+
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.CmdLineParser;
+
+public class AdgClientDriver {
+
+ public static final int NUMBER_OF_ARGUMENTS = 2;
+
+ public static class AdgClientConfig {
+
+ @Argument(index = 0, required = true, metaVar = "ARG1", usage = "The file containing the annotated schema.")
+ private File schemaFile;
+
+ @Argument(index = 1, required = true, metaVar = "ARG2", usage = "The output directory path.")
+ private File outputDir;
+ }
+
+ public static void main(String[] args) throws Exception {
+ AdgClientConfig acc = new AdgClientConfig();
+ CmdLineParser cmdParser = new CmdLineParser(acc);
+ try {
+ cmdParser.parseArgument(args);
+ } catch (Exception e) {
+ cmdParser.printUsage(System.err);
+ throw e;
+ }
+ AdmDataGen adg = new AdmDataGen(acc.schemaFile, acc.outputDir);
+ adg.init();
+ adg.dataGen();
+ }
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java
new file mode 100644
index 0000000..40ea3fd
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java
@@ -0,0 +1,1001 @@
+package edu.uci.ics.asterix.tools.datagen;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import edu.uci.ics.asterix.aql.expression.Query;
+import edu.uci.ics.asterix.aql.parser.AQLParser;
+import edu.uci.ics.asterix.aql.parser.ParseException;
+import edu.uci.ics.asterix.common.annotations.AutoDataGen;
+import edu.uci.ics.asterix.common.annotations.DateBetweenYearsDataGen;
+import edu.uci.ics.asterix.common.annotations.DatetimeAddRandHoursDataGen;
+import edu.uci.ics.asterix.common.annotations.DatetimeBetweenYearsDataGen;
+import edu.uci.ics.asterix.common.annotations.FieldIntervalDataGen;
+import edu.uci.ics.asterix.common.annotations.FieldValFileDataGen;
+import edu.uci.ics.asterix.common.annotations.FieldValFileSameIndexDataGen;
+import edu.uci.ics.asterix.common.annotations.IRecordFieldDataGen;
+import edu.uci.ics.asterix.common.annotations.IRecordTypeAnnotation;
+import edu.uci.ics.asterix.common.annotations.IRecordTypeAnnotation.Kind;
+import edu.uci.ics.asterix.common.annotations.InsertRandIntDataGen;
+import edu.uci.ics.asterix.common.annotations.ListDataGen;
+import edu.uci.ics.asterix.common.annotations.ListValFileDataGen;
+import edu.uci.ics.asterix.common.annotations.RecordDataGenAnnotation;
+import edu.uci.ics.asterix.common.annotations.TypeDataGen;
+import edu.uci.ics.asterix.common.annotations.UndeclaredFieldsDataGen;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.declared.AqlCompiledMetadataDeclarations;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.AUnionType;
+import edu.uci.ics.asterix.om.types.AbstractCollectionType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.tools.translator.ADGenDmlTranslator;
+import edu.uci.ics.asterix.transaction.management.exception.ACIDException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;
+
+public class AdmDataGen {
+
+ class DataGeneratorContext {
+ private Map<File, String[]> fileToLoadedDataMap = new HashMap<File, String[]>();
+
+ public Map<File, String[]> getFileToLoadedDataMap() {
+ return fileToLoadedDataMap;
+ }
+ }
+
+ public static void printDate(int year, int month, int day, PrintStream out) throws IOException {
+ WriteValueTools.writeInt(year, out);
+ out.print("-");
+ if (month < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(month, out);
+ out.print("-");
+ if (day < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(day, out);
+ }
+
+ abstract class AbstractValueGenerator {
+ protected PrintStream out;
+ protected DataGeneratorContext ctx;
+
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ this.out = out;
+ this.ctx = ctx;
+ }
+
+ public abstract void generate() throws IOException;
+ }
+
+ abstract class RandomValueGenerator extends AbstractValueGenerator {
+ protected Random rnd;
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ this.rnd = new Random(51);
+ }
+
+ }
+
+ class IntAutoGenerator extends AbstractValueGenerator {
+
+ private final int initValue;
+ private int val;
+
+ public IntAutoGenerator(int initValue) {
+ this.initValue = initValue;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ val = initValue;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ WriteValueTools.writeInt(val, out);
+ val++;
+ }
+
+ }
+
+ class LongAutoGenerator extends AbstractValueGenerator {
+
+ private final long initValue;
+ private long val;
+
+ public LongAutoGenerator(long initValue) {
+ this.initValue = initValue;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ val = initValue;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ WriteValueTools.writeLong(val, out);
+ out.print("i64");
+ val++;
+ }
+
+ }
+
+ class StringFromArrayGenerator extends RandomValueGenerator {
+ private final String[][] array;
+ private int lastIndex;
+ private final String constructor;
+ private String[] chosen;
+
+ public StringFromArrayGenerator(String[][] array, String constructor) {
+ this.array = array;
+ this.constructor = constructor;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ this.lastIndex = -1;
+ this.chosen = new String[array.length];
+ }
+
+ @Override
+ public void generate() throws IOException {
+ for (int i = 0; i < array.length; i++) {
+ lastIndex = Math.abs(rnd.nextInt()) % array[i].length;
+ chosen[i] = array[i][lastIndex];
+ }
+ if (constructor != null) {
+ out.print(constructor);
+ out.print("(");
+ }
+ out.print("\"");
+ for (int i = 0; i < chosen.length; i++) {
+ out.print(chosen[i]);
+ }
+ out.print("\"");
+ if (constructor != null) {
+ out.print(")");
+ }
+ }
+
+ public int getLastIndex() {
+ return lastIndex;
+ }
+ }
+
+ abstract class AbstractCollectionDataGenerator extends RandomValueGenerator {
+ protected final AbstractCollectionType act;
+ protected final int min;
+ protected final int max;
+ protected final String startList;
+ protected final String endList;
+
+ public AbstractCollectionDataGenerator(AbstractCollectionType act, int min, int max) {
+ this.act = act;
+ this.min = min;
+ this.max = max;
+ if (act.getTypeTag() == ATypeTag.ORDEREDLIST) {
+ startList = "[";
+ endList = "]";
+ } else {
+ startList = "{{";
+ endList = "}}";
+ }
+ }
+
+ }
+
+ class ListDataGenerator extends AbstractCollectionDataGenerator {
+
+ private AbstractValueGenerator itemGen;
+
+ public ListDataGenerator(AbstractCollectionType act, int min, int max) {
+ super(act, min, max);
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ IAType t = act.getItemType();
+ if (t.getTypeTag() != ATypeTag.RECORD) {
+ throw new NotImplementedException("list annotation only works with record item types for now.");
+ }
+ ARecordType rt = (ARecordType) t;
+ RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
+ if (dga == null) {
+ throw new Exception("No annotation on item type " + t);
+ }
+ itemGen = new RecordGenerator(rt, dga, null);
+ itemGen.init(out, ctx);
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print(startList);
+ int numItems = rnd.nextInt(max - min + 1) + min;
+ for (int i = 0; i < numItems; i++) {
+ if (i > 0) {
+ out.print(", ");
+ }
+ itemGen.generate();
+ }
+ out.print(endList);
+ }
+
+ }
+
+ class ListFromArrayGenerator extends AbstractCollectionDataGenerator {
+
+ private final String[] array;
+ private String constructor;
+
+ public ListFromArrayGenerator(AbstractCollectionType act, String[] array, int min, int max) {
+ super(act, min, max);
+ this.array = array;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ if (act.getItemType().getTypeTag() == ATypeTag.STRING) {
+ constructor = null;
+ } else {
+ constructor = getConstructor(act.getItemType());
+ }
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print(startList);
+ int numItems = rnd.nextInt(max - min + 1) + min;
+ for (int i = 0; i < numItems; i++) {
+ if (i > 0) {
+ out.print(", ");
+ }
+ int c = Math.abs(rnd.nextInt()) % array.length;
+ if (constructor != null) {
+ out.print(constructor);
+ out.print("(");
+ }
+ out.print("\"");
+ out.print(array[c]);
+ out.print("\"");
+ if (constructor != null) {
+ out.print(")");
+ }
+ }
+ out.print(endList);
+ }
+
+ }
+
+ class StringFromArraySameIndexGenerator extends AbstractValueGenerator {
+ private final String[] array;
+ private final StringFromArrayGenerator sfag;
+ private final String constructor;
+
+ public StringFromArraySameIndexGenerator(String[] array, StringFromArrayGenerator sfag, String constructor) {
+ this.array = array;
+ this.sfag = sfag;
+ this.constructor = constructor;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ String val = array[sfag.getLastIndex()];
+ if (constructor != null) {
+ out.print(constructor);
+ out.print("(");
+ }
+ out.print("\"");
+ out.print(val);
+ out.print("\"");
+ if (constructor != null) {
+ out.print(")");
+ }
+ }
+ }
+
+ class IntIntervalGenerator extends RandomValueGenerator {
+
+ private final int min;
+ private final int max;
+ private final String prefix;
+ private final String suffix;
+
+ public IntIntervalGenerator(int min, int max, String prefix, String suffix) {
+ this.min = min;
+ this.max = max;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int v = Math.abs(rnd.nextInt() % (max - min + 1)) + min;
+ if (prefix != null) {
+ out.print(prefix);
+ }
+ WriteValueTools.writeInt(v, out);
+ if (suffix != null) {
+ out.print(suffix);
+ }
+ }
+
+ }
+
+ class LongIntervalGenerator extends RandomValueGenerator {
+
+ private final long min;
+ private final long max;
+ private final String prefix;
+ private final String suffix;
+
+ public LongIntervalGenerator(long min, long max, String prefix, String suffix) {
+ this.min = min;
+ this.max = max;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ long v = Math.abs(rnd.nextLong() % (max - min + 1)) + min;
+ if (prefix != null) {
+ out.print(prefix);
+ }
+ WriteValueTools.writeLong(v, out);
+ if (suffix != null) {
+ out.print(suffix);
+ }
+ }
+
+ }
+
+ class DoubleIntervalGenerator extends RandomValueGenerator {
+
+ private final double min;
+ private final double max;
+ private final String prefix;
+ private final String suffix;
+
+ public DoubleIntervalGenerator(double min, double max, String prefix, String suffix) {
+ this.min = min;
+ this.max = max;
+ this.prefix = prefix;
+ this.suffix = suffix;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ double v = Math.abs(rnd.nextDouble() % (max - min)) + min;
+ if (prefix != null) {
+ out.print(prefix);
+ }
+ out.print(v);
+ if (suffix != null) {
+ out.print(suffix);
+ }
+ }
+
+ }
+
+ class InsertRandIntGenerator extends RandomValueGenerator {
+
+ private final String str1;
+ private final String str2;
+
+ public InsertRandIntGenerator(String str1, String str2) {
+ this.str1 = "\"" + str1;
+ this.str2 = str2 + "\"";
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int v = Math.abs(rnd.nextInt());
+ out.print(str1);
+ WriteValueTools.writeInt(v, out);
+ out.print(str2);
+ }
+
+ }
+
+ interface AccessibleDateGenerator {
+ public int getYear();
+
+ public int getMonth();
+
+ public int getDay();
+ }
+
+ abstract class DateGenerator extends RandomValueGenerator implements AccessibleDateGenerator {
+
+ private int year;
+ private int month;
+ private int day;
+
+ protected void generateDate(int minYear, int maxYear) throws IOException {
+ year = rnd.nextInt(maxYear - minYear + 1) + minYear;
+ month = Math.abs(rnd.nextInt()) % 12 + 1;
+ day = Math.abs(rnd.nextInt()) % 28 + 1;
+ printDate(year, month, day, out);
+ }
+
+ @Override
+ public int getYear() {
+ return year;
+ }
+
+ @Override
+ public int getMonth() {
+ return month;
+ }
+
+ @Override
+ public int getDay() {
+ return day;
+ }
+ }
+
+ class DateBetweenYearsGenerator extends DateGenerator {
+
+ private final int minYear;
+ private final int maxYear;
+
+ public DateBetweenYearsGenerator(int minYear, int maxYear) {
+ this.minYear = minYear;
+ this.maxYear = maxYear;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print("date(\"");
+ generateDate(minYear, maxYear);
+ out.print("\")");
+ }
+
+ }
+
+ interface AccessibleDatetimeGenerator extends AccessibleDateGenerator {
+ public int getHour();
+
+ public int getMinute();
+
+ public int getSecond();
+ }
+
+ class DatetimeBetweenYearsGenerator extends DateGenerator implements AccessibleDatetimeGenerator {
+ private final int minYear;
+ private final int maxYear;
+ private int hour;
+
+ public DatetimeBetweenYearsGenerator(int minYear, int maxYear) {
+ this.minYear = minYear;
+ this.maxYear = maxYear;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print("datetime(\"");
+ generateDate(minYear, maxYear);
+ out.print("T");
+ hour = rnd.nextInt(24);
+ if (hour < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(hour, out);
+ out.print(":00:00\")");
+ }
+
+ @Override
+ public int getHour() {
+ return hour;
+ }
+
+ @Override
+ public int getMinute() {
+ return 0;
+ }
+
+ @Override
+ public int getSecond() {
+ return 0;
+ }
+
+ }
+
+ class DatetimeAddRandHoursGenerator extends RandomValueGenerator {
+
+ private final int minHours;
+ private final int maxHours;
+ private final AccessibleDatetimeGenerator adtg;
+
+ public DatetimeAddRandHoursGenerator(int minHours, int maxHours, AccessibleDatetimeGenerator adtg) {
+ this.minHours = minHours;
+ this.maxHours = maxHours;
+ this.adtg = adtg;
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int addHours = rnd.nextInt(maxHours - minHours + 1) + minHours;
+
+ out.print("datetime(\"");
+ printDate(adtg.getYear(), adtg.getMonth(), adtg.getDay(), out);
+ out.print("T");
+ int h = adtg.getHour() + addHours;
+ if (h > 23) {
+ h = 23;
+ }
+ if (h < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(h, out);
+ out.print(":");
+ int m = adtg.getMinute();
+ if (m < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(m, out);
+ out.print(":");
+ int s = adtg.getSecond();
+ if (s < 10) {
+ out.print("0");
+ }
+ WriteValueTools.writeInt(s, out);
+ out.print("\")");
+ }
+
+ }
+
+ class GenFieldsIntGenerator extends RandomValueGenerator {
+ private final int minFields;
+ private final int maxFields;
+ private final String prefix;
+ private final int startIndex;
+ private String[] fieldNames;
+ private int[] id;
+ private int[] x;
+
+ public GenFieldsIntGenerator(int startIndex, int minFields, int maxFields, String prefix) {
+ this.startIndex = startIndex;
+ this.minFields = minFields;
+ this.maxFields = maxFields;
+ this.prefix = prefix;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ fieldNames = new String[maxFields];
+ for (int i = 0; i < maxFields; i++) {
+ fieldNames[i] = prefix + "_" + i;
+ }
+ id = new int[maxFields];
+ for (int i = 0; i < maxFields; i++) {
+ id[i] = i;
+ }
+ x = new int[maxFields];
+ }
+
+ @Override
+ public void generate() throws IOException {
+ int numFields = rnd.nextInt(maxFields - minFields + 1) + minFields;
+ System.arraycopy(id, 0, x, 0, maxFields);
+ int n = numFields;
+ boolean first = startIndex < 1;
+ while (n > 0) {
+ int c = rnd.nextInt(n);
+ if (first) {
+ first = false;
+ } else {
+ out.print(",");
+ }
+ out.print(" \"");
+ out.print(fieldNames[x[c]]);
+ out.print("\": ");
+ WriteValueTools.writeInt(Math.abs(rnd.nextInt()), out);
+ x[c] = x[n - 1];
+ n--;
+ }
+ }
+ }
+
+ class RecordGenerator extends RandomValueGenerator {
+
+ private AbstractValueGenerator[] declaredFieldsGenerators;
+ private boolean[] nullable;
+ private AbstractValueGenerator undeclaredFieldsGenerator;
+ private final ARecordType recType;
+ private final RecordDataGenAnnotation annot;
+ private final String printAfter;
+
+ public RecordGenerator(ARecordType type, RecordDataGenAnnotation annot, String printAfter) {
+ this.recType = type;
+ this.annot = annot;
+ this.printAfter = printAfter;
+ }
+
+ @Override
+ public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+ super.init(out, ctx);
+ if (declaredFieldsGenerators == null) {
+ int m = annot.getDeclaredFieldsDatagen().length;
+ declaredFieldsGenerators = new AbstractValueGenerator[m];
+ nullable = new boolean[m];
+ for (int i = 0; i < m; i++) {
+ IAType ti = recType.getFieldTypes()[i];
+ if (ti.getTypeTag() == ATypeTag.UNION) {
+ AUnionType ut = (AUnionType) ti;
+ if (ut.isNullableType()) {
+ ti = ut.getUnionList().get(1);
+ nullable[i] = true;
+ }
+ }
+ IRecordFieldDataGen rfdg = annot.getDeclaredFieldsDatagen()[i];
+ if (rfdg == null) {
+ if (ti.getTypeTag() == ATypeTag.RECORD) {
+ ARecordType rt = (ARecordType) ti;
+ RecordDataGenAnnotation dga = null;
+ for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
+ if (annot.getKind() == Kind.RECORD_DATA_GEN) {
+ dga = (RecordDataGenAnnotation) annot;
+ break;
+ }
+ }
+ if (dga != null) {
+ declaredFieldsGenerators[i] = new RecordGenerator(rt, dga, null);
+ continue;
+ }
+ }
+ if (declaredFieldsGenerators[i] == null) {
+ throw new Exception("No data generator annotation for field " + recType.getFieldNames()[i]
+ + " in type " + recType);
+ }
+ }
+ switch (rfdg.getKind()) {
+ case VALFILE: {
+ FieldValFileDataGen vf = (FieldValFileDataGen) rfdg;
+ int numFiles = vf.getFiles().length;
+ String[][] a = new String[numFiles][];
+ for (int k = 0; k < numFiles; k++) {
+ File f = vf.getFiles()[k];
+ a[k] = ctx.getFileToLoadedDataMap().get(f);
+ if (a[k] == null) {
+ a[k] = readFileAsStringArray(f);
+ ctx.getFileToLoadedDataMap().put(f, a[k]);
+ }
+ }
+ String constructor;
+ if (ti.getTypeTag() == ATypeTag.STRING) {
+ constructor = null;
+ } else {
+ constructor = getConstructor(ti);
+ }
+ declaredFieldsGenerators[i] = new StringFromArrayGenerator(a, constructor);
+ break;
+ }
+ case LISTVALFILE: {
+ ListValFileDataGen lvf = (ListValFileDataGen) rfdg;
+ String[] a = ctx.getFileToLoadedDataMap().get(lvf.getFile());
+ if (a == null) {
+ a = readFileAsStringArray(lvf.getFile());
+ ctx.getFileToLoadedDataMap().put(lvf.getFile(), a);
+ }
+ if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
+ throw new Exception("list-val-file annotation cannot be used for field of type "
+ + ti.getTypeTag());
+ }
+ AbstractCollectionType act = (AbstractCollectionType) ti;
+ declaredFieldsGenerators[i] = new ListFromArrayGenerator(act, a, lvf.getMin(), lvf.getMax());
+ break;
+ }
+ case VALFILESAMEINDEX: {
+ FieldValFileSameIndexDataGen vfsi = (FieldValFileSameIndexDataGen) rfdg;
+ String[] a = ctx.getFileToLoadedDataMap().get(vfsi.getFile());
+ if (a == null) {
+ a = readFileAsStringArray(vfsi.getFile());
+ ctx.getFileToLoadedDataMap().put(vfsi.getFile(), a);
+ }
+ StringFromArrayGenerator sfag = null;
+ for (int j = 0; j < i; j++) {
+ if (recType.getFieldNames()[j].equals(vfsi.getSameAsField())) {
+ if (declaredFieldsGenerators[j] instanceof StringFromArrayGenerator) {
+ sfag = (StringFromArrayGenerator) declaredFieldsGenerators[j];
+ break;
+ } else {
+ throw new Exception(
+ "Data generator for field "
+ + recType.getFieldNames()[j]
+ + " is not based on values from a text file, as required by generator for field "
+ + recType.getFieldNames()[i]);
+ }
+ }
+ }
+ if (sfag == null) {
+ throw new Exception("Couldn't find field " + vfsi.getSameAsField() + " before field "
+ + recType.getFieldNames()[i]);
+ }
+ String constructor;
+ if (ti.getTypeTag() == ATypeTag.STRING) {
+ constructor = null;
+ } else {
+ constructor = getConstructor(ti);
+ }
+ declaredFieldsGenerators[i] = new StringFromArraySameIndexGenerator(a, sfag, constructor);
+ break;
+ }
+ case INTERVAL: {
+ FieldIntervalDataGen fi = (FieldIntervalDataGen) rfdg;
+ String prefix = null;
+ String suffix = null;
+ if (ti.getTypeTag() == ATypeTag.STRING) {
+ prefix = "\"";
+ suffix = "\"";
+ }
+ switch (fi.getValueType()) {
+ case INT: {
+ declaredFieldsGenerators[i] = new IntIntervalGenerator(
+ Integer.parseInt(fi.getMin()), Integer.parseInt(fi.getMax()), prefix,
+ suffix);
+ break;
+ }
+ case LONG: {
+ declaredFieldsGenerators[i] = new LongIntervalGenerator(
+ Long.parseLong(fi.getMin()), Long.parseLong(fi.getMax()), prefix, suffix);
+ break;
+ }
+ case DOUBLE: {
+ declaredFieldsGenerators[i] = new DoubleIntervalGenerator(Double.parseDouble(fi
+ .getMin()), Double.parseDouble(fi.getMax()), prefix, suffix);
+ break;
+ }
+ default: {
+ throw new NotImplementedException();
+ }
+ }
+ break;
+ }
+ case INSERTRANDINT: {
+ InsertRandIntDataGen iri = (InsertRandIntDataGen) rfdg;
+ declaredFieldsGenerators[i] = new InsertRandIntGenerator(iri.getStr1(), iri.getStr2());
+ break;
+ }
+ case LIST: {
+ ListDataGen l = (ListDataGen) rfdg;
+ if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
+ throw new Exception("list-val-file annotation cannot be used for field of type "
+ + ti.getTypeTag());
+ }
+ AbstractCollectionType act = (AbstractCollectionType) ti;
+ declaredFieldsGenerators[i] = new ListDataGenerator(act, l.getMin(), l.getMax());
+ break;
+ }
+ case DATEBETWEENYEARS: {
+ DateBetweenYearsDataGen dby = (DateBetweenYearsDataGen) rfdg;
+ declaredFieldsGenerators[i] = new DateBetweenYearsGenerator(dby.getMinYear(),
+ dby.getMaxYear());
+ break;
+ }
+ case DATETIMEBETWEENYEARS: {
+ DatetimeBetweenYearsDataGen dtby = (DatetimeBetweenYearsDataGen) rfdg;
+ declaredFieldsGenerators[i] = new DatetimeBetweenYearsGenerator(dtby.getMinYear(),
+ dtby.getMaxYear());
+ break;
+ }
+ case DATETIMEADDRANDHOURS: {
+ DatetimeAddRandHoursDataGen dtarh = (DatetimeAddRandHoursDataGen) rfdg;
+ AccessibleDatetimeGenerator adtg = null;
+ for (int j = 0; j < i; j++) {
+ if (recType.getFieldNames()[j].equals(dtarh.getAddToField())) {
+ if (declaredFieldsGenerators[j] instanceof AccessibleDatetimeGenerator) {
+ adtg = (AccessibleDatetimeGenerator) declaredFieldsGenerators[j];
+ break;
+ } else {
+ throw new Exception(
+ "Data generator for field "
+ + recType.getFieldNames()[j]
+ + " is not based on accessible datetime values, as required by generator for field "
+ + recType.getFieldNames()[i]);
+ }
+ }
+ }
+ if (adtg == null) {
+ throw new Exception("Couldn't find field " + dtarh.getAddToField() + " before field "
+ + recType.getFieldNames()[i]);
+ }
+ declaredFieldsGenerators[i] = new DatetimeAddRandHoursGenerator(dtarh.getMinHour(),
+ dtarh.getMaxHour(), adtg);
+ break;
+ }
+ case AUTO: {
+ AutoDataGen auto = (AutoDataGen) rfdg;
+ switch (ti.getTypeTag()) {
+ case INT32: {
+ declaredFieldsGenerators[i] = new IntAutoGenerator(Integer.parseInt(auto
+ .getInitValueStr()));
+ break;
+ }
+ case INT64: {
+ declaredFieldsGenerators[i] = new LongAutoGenerator(Long.parseLong(auto
+ .getInitValueStr()));
+ break;
+ }
+ default: {
+ throw new IllegalStateException(rfdg.getKind()
+ + " annotation is not implemented for type " + ti.getTypeTag());
+ }
+ }
+ break;
+ }
+ default: {
+ throw new NotImplementedException(rfdg.getKind() + " is not implemented");
+ }
+ }
+ }
+ }
+ for (int i = 0; i < declaredFieldsGenerators.length; i++) {
+ declaredFieldsGenerators[i].init(out, ctx);
+ }
+ if (undeclaredFieldsGenerator == null) {
+ UndeclaredFieldsDataGen ufdg = annot.getUndeclaredFieldsDataGen();
+ if (ufdg != null) {
+ if (!recType.isOpen()) {
+ throw new Exception("Cannot generate undeclared fields for closed type " + recType);
+ }
+ undeclaredFieldsGenerator = new GenFieldsIntGenerator(declaredFieldsGenerators.length,
+ ufdg.getMinUndeclaredFields(), ufdg.getMaxUndeclaredFields(),
+ ufdg.getUndeclaredFieldsPrefix());
+ }
+ }
+ if (undeclaredFieldsGenerator != null) {
+ undeclaredFieldsGenerator.init(out, ctx);
+ }
+ }
+
+ @Override
+ public void generate() throws IOException {
+ out.print("{");
+ boolean first = true;
+ for (int i = 0; i < declaredFieldsGenerators.length; i++) {
+ boolean pick;
+ if (nullable[i]) {
+ pick = rnd.nextBoolean();
+ } else {
+ pick = true;
+ }
+ if (pick) {
+ if (first) {
+ first = false;
+ } else {
+ out.print(",");
+ }
+ out.print(" \"");
+ out.print(this.recType.getFieldNames()[i]);
+ out.print("\": ");
+ declaredFieldsGenerators[i].generate();
+ }
+ }
+ if (undeclaredFieldsGenerator != null) {
+ undeclaredFieldsGenerator.generate();
+ }
+ out.print(" }");
+ if (printAfter != null) {
+ out.print(printAfter);
+ }
+ }
+
+ }
+
+ private final File schemaFile;
+ private final File outputDir;
+ private Map<String, IAType> typeMap;
+ private Map<String, TypeDataGen> typeAnnotMap;
+ private DataGeneratorContext dgCtx;
+
+ public AdmDataGen(File schemaFile, File outputDir) {
+ this.schemaFile = schemaFile;
+ this.outputDir = outputDir;
+ }
+
+ public void init() throws IOException, ParseException, AlgebricksException, ACIDException, MetadataException {
+ FileReader aql = new FileReader(schemaFile);
+ AQLParser parser = new AQLParser(aql);
+ Query q = (Query) parser.Statement();
+ aql.close();
+ // TODO: Need to fix how to use transactions here.
+ MetadataTransactionContext mdTxnCtx = new MetadataTransactionContext(-1);
+ ADGenDmlTranslator dmlt = new ADGenDmlTranslator(mdTxnCtx, q.getPrologDeclList());
+ dmlt.translate();
+ AqlCompiledMetadataDeclarations acmd = dmlt.getCompiledDeclarations();
+ typeMap = acmd.getTypeDeclarations();
+ typeAnnotMap = acmd.getTypeDataGenMap();
+ dgCtx = new DataGeneratorContext();
+ }
+
+ public void dataGen() throws Exception {
+ for (Map.Entry<String, IAType> me : typeMap.entrySet()) {
+ String tn = me.getKey();
+ TypeDataGen tdg = typeAnnotMap.get(tn);
+ if (tdg.isDataGen()) {
+ IAType t = me.getValue();
+
+ if (t.getTypeTag() != ATypeTag.RECORD) {
+ throw new NotImplementedException();
+ }
+ ARecordType rt = (ARecordType) t;
+ RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
+ if (dga == null) {
+ throw new Exception("No data generator annotations for type " + tn);
+ }
+ File outFile = new File(outputDir + File.separator + tdg.getOutputFileName());
+ PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFile)));
+ RecordGenerator rg = new RecordGenerator(rt, dga, "\n");
+ rg.init(outStream, dgCtx);
+ for (long i = 0; i < tdg.getNumValues(); i++) {
+ rg.generate();
+ }
+ outStream.close();
+ }
+ }
+ }
+
+ private static RecordDataGenAnnotation firstDataGenAnnotation(ARecordType rt) {
+ for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
+ if (annot.getKind() == Kind.RECORD_DATA_GEN) {
+ return (RecordDataGenAnnotation) annot;
+ }
+ }
+ return null;
+ }
+
+ private static String[] readFileAsStringArray(File file) throws IOException {
+ List<String> tmp = new ArrayList<String>();
+ FileInputStream fstream = new FileInputStream(file);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ tmp.add(strLine);
+ }
+ in.close();
+ return tmp.toArray(new String[0]);
+ }
+
+ private static String getConstructor(IAType t) throws Exception {
+ if (t instanceof BuiltinType) {
+ String s = ((BuiltinType) t).getConstructor();
+ if (s == null) {
+ throw new Exception("Type " + t + " has no constructors.");
+ }
+ return s;
+ } else {
+ throw new Exception("No string constructor for type " + t);
+ }
+ }
+
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/CustOrdDataGen.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/CustOrdDataGen.java
new file mode 100644
index 0000000..29c5330
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/CustOrdDataGen.java
@@ -0,0 +1,457 @@
+package edu.uci.ics.asterix.tools.datagen;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+public class CustOrdDataGen {
+
+ // for customers
+ private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
+ private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
+ private static final int MIN_AGE = 10;
+ private static final int MAX_AGE = 90;
+
+ private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
+ "Lake St.", "Hill St.", "Park St.", "View St." };
+ private static final int MIN_STREET_NUM = 1;
+ private static final int MAX_STREET_NUM = 10000;
+ private static final String[] CITIES = { "Seattle", "San Jose", "Mountain View", "Los Angeles", "Sunnyvale",
+ "Portland" };
+
+ private static final int MIN_INTERESTS = 0;
+ private static final int MAX_INTERESTS = 5;
+ private String[] INTERESTS = { "Bass", "Music", "Databases", "Fishing", "Tennis", "Squash", "Computers", "Books",
+ "Movies", "Cigars", "Wine", "Running", "Walking", "Skiing", "Basketball", "Video Games", "Cooking",
+ "Coffee", "Base Jumping", "Puzzles" };
+
+ private static final int MIN_CHILD_AGE = 0;
+
+ private static final int MIN_CHILDREN = 0;
+ private static final int MAX_CHILDREN = 5;
+
+ // for orders
+ private static final int MIN_ORDERS_PER_CUST = 1;
+ private static final int MAX_ORDERS_PER_CUST = 10;
+
+ private String[] CLERKS = { "Kathrin", "Katherine", "Cathryn", "Catherine", "Cat", "Kathryne", "Cathrin" };
+ private String[] ORDER_PRIORITIES = { "LOW", "MEDIUM", "HIGH", "PREMIUM" };
+ private String[] ORDER_STATUSES = { "ORDER_PLACED", "PAYMENT_RECEIVED", "ORDER_SHIPPED", "ORDER_DELIVERED" };
+
+ private String[] firstNames = { "Joe", "John", "Jill", "Gill", "Bill", "William", "Kathy", "Cathey", "Jane",
+ "Albert" };
+ private String[] lastNames = { "Doe", "Smith", "Li", "Singh", "Williams", "Davis", "Brown", "Wilson", "Moore",
+ "Thomas" };
+
+ private static final String[] UNDECLARED_FIELD_NAMES = { "param1", "param2", "param3", "param4", "param5",
+ "param6", "param7", "param8", "param9", "param10" };
+
+ private int currentCID = 0;
+ private int currentOID = 0;
+
+ private Random rndValue = new Random(50);
+
+ private Random rndChildAgeField = new Random(50);
+ private Random rndCustAgeField = new Random(50);
+ private Random rndCustAddressField = new Random(50);
+
+ private Random[] rndUndeclaredOrderFields;
+
+ private class Child {
+ public String name;
+ public int age;
+
+ public void generateFieldValues(String lastName, int maxChildAge) {
+ // name
+ int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
+ name = firstNames[firstNameIx] + " " + lastName;
+
+ // age
+ age = -1;
+ if (rndChildAgeField.nextBoolean()) {
+ if (maxChildAge >= 0) {
+ if (maxChildAge == MIN_CHILD_AGE) {
+ age = maxChildAge;
+ } else {
+ age = Math.abs((rndValue.nextInt()) % (maxChildAge - MIN_CHILD_AGE)) + MIN_AGE;
+ }
+ }
+ }
+ }
+
+ public String getJSON() {
+ StringBuilder jsonString = new StringBuilder();
+
+ jsonString.append("{ "); // start child
+
+ // name
+ jsonString.append(" \"name\": ");
+ jsonString.append("\"" + name + "\"");
+
+ // age
+ if (age >= 0) {
+ jsonString.append(", ");
+ jsonString.append(" \"age\": ");
+ jsonString.append(age);
+ }
+
+ jsonString.append(" }"); // end child
+
+ return jsonString.toString();
+ }
+
+ }
+
+ private class Customer {
+
+ private int cid;
+ private String name;
+ private int age;
+
+ private String streetName;
+ private int streetNumber;
+ private String city;
+
+ private int[] custInterests;
+ private Child[] custChildren;
+
+ public void generateFieldValues() {
+ cid = currentCID++;
+
+ int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
+ int lastNameIx = Math.abs(rndValue.nextInt()) % lastNames.length;
+ name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
+
+ if (rndCustAgeField.nextBoolean()) {
+ age = Math.abs((rndValue.nextInt()) % (MAX_AGE - MIN_AGE)) + MIN_AGE;
+ } else {
+ age = -1;
+ }
+
+ if (rndCustAddressField.nextBoolean()) {
+ streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
+
+ int streetIx = Math.abs(rndValue.nextInt()) % STREETS.length;
+ streetName = STREETS[streetIx];
+
+ int cityIx = Math.abs(rndValue.nextInt()) % CITIES.length;
+ city = CITIES[cityIx];
+ } else {
+ streetNumber = -1;
+ streetName = null;
+ city = null;
+ }
+
+ int numInterests = Math.abs((rndValue.nextInt()) % (MAX_INTERESTS - MIN_INTERESTS)) + MIN_INTERESTS;
+ custInterests = new int[numInterests];
+ for (int i = 0; i < numInterests; i++) {
+ custInterests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
+ }
+
+ int numChildren = Math.abs((rndValue.nextInt()) % (MAX_CHILDREN - MIN_CHILDREN)) + MIN_CHILDREN;
+ custChildren = new Child[numChildren];
+ for (int i = 0; i < numChildren; i++) {
+ Child c = new Child();
+ int maxChildAge = age <= 0 ? 50 : (age - 20);
+ c.generateFieldValues(lastNames[lastNameIx], maxChildAge);
+ custChildren[i] = c;
+ }
+ }
+
+ public String getJSON() {
+ StringBuilder jsonString = new StringBuilder();
+
+ jsonString.append("{ "); // start customer
+
+ // customer id
+ jsonString.append(" \"cid\": ");
+ jsonString.append(cid);
+ jsonString.append(", ");
+
+ // name
+ jsonString.append(" \"name\": ");
+ jsonString.append("\"" + name + "\"");
+
+ // age
+ if (age >= 0) {
+ jsonString.append(", ");
+ jsonString.append(" \"age\": ");
+ jsonString.append(age);
+ }
+
+ // nested address
+ if (streetNumber >= 0) {
+ jsonString.append(", ");
+ jsonString.append(" \"address\": ");
+ jsonString.append("{ "); // start address
+
+ // number
+ jsonString.append(" \"number\": ");
+ jsonString.append(streetNumber);
+ jsonString.append(", ");
+
+ // street
+ jsonString.append(" \"street\": ");
+ jsonString.append("\"" + streetName + "\"");
+ jsonString.append(", ");
+
+ // city
+ jsonString.append(" \"city\": ");
+ jsonString.append("\"" + city + "\"");
+
+ jsonString.append(" }"); // end address
+ }
+
+ jsonString.append(", ");
+
+ // interests
+ jsonString.append(" \"interests\": ");
+ jsonString.append("{{ "); // start interests
+ for (int i = 0; i < custInterests.length; i++) {
+ jsonString.append("\"" + INTERESTS[custInterests[i]] + "\"");
+ if (i != custInterests.length - 1) {
+ jsonString.append(", ");
+ }
+ }
+ jsonString.append(" }}"); // end interests
+ jsonString.append(", ");
+
+ // children
+ jsonString.append(" \"children\": ");
+ jsonString.append("[ "); // start children
+ for (int i = 0; i < custChildren.length; i++) {
+ String jsonChild = custChildren[i].getJSON();
+ jsonString.append(jsonChild);
+ if (i != custChildren.length - 1) {
+ jsonString.append(", ");
+ }
+ }
+ jsonString.append(" ]"); // end children
+
+ jsonString.append(" }"); // end customer
+
+ return jsonString.toString();
+ }
+ }
+
+ private class Order {
+ public int oid;
+ public int cid;
+ public String orderStatus;
+ public String orderPriority;
+ public String clerk;
+ public String total;
+
+ private int[] undeclaredFields;
+ private BitSet nullMap;
+
+ public void generateFieldValues(Customer cust) {
+ cid = cust.cid;
+ oid = currentOID++;
+ float t = Math.abs(rndValue.nextFloat()) * 100;
+ total = t + "f";
+
+ int orderStatusIx = Math.abs(rndValue.nextInt()) % ORDER_STATUSES.length;
+ orderStatus = ORDER_STATUSES[orderStatusIx];
+
+ int orderPriorityIx = Math.abs(rndValue.nextInt()) % ORDER_PRIORITIES.length;
+ orderPriority = ORDER_PRIORITIES[orderPriorityIx];
+
+ int clerkIx = Math.abs(rndValue.nextInt()) % CLERKS.length;
+ clerk = CLERKS[clerkIx];
+
+ int m = rndUndeclaredOrderFields.length;
+ undeclaredFields = new int[m];
+ nullMap = new BitSet(m);
+ for (int i = 0; i < m; i++) {
+ if (rndUndeclaredOrderFields[i].nextBoolean()) {
+ undeclaredFields[i] = rndValue.nextInt();
+ } else {
+ nullMap.set(i);
+ }
+ }
+ }
+
+ public String getJSON() {
+ StringBuilder jsonString = new StringBuilder();
+
+ jsonString.append("{ "); // start order
+
+ // oid
+ jsonString.append(" \"oid\": ");
+ jsonString.append(oid);
+ jsonString.append(", ");
+
+ // cid
+ jsonString.append(" \"cid\": ");
+ jsonString.append(cid);
+ jsonString.append(", ");
+
+ // orderStatus
+ jsonString.append(" \"orderstatus\": ");
+ jsonString.append("\"" + orderStatus + "\"");
+ jsonString.append(", ");
+
+ // orderPriority
+ jsonString.append(" \"orderpriority\": ");
+ jsonString.append("\"" + orderPriority + "\"");
+ jsonString.append(", ");
+
+ // clerk
+ jsonString.append(" \"clerk\": ");
+ jsonString.append("\"" + clerk + "\"");
+ jsonString.append(", ");
+
+ // / cid
+ jsonString.append(" \"total\": ");
+ jsonString.append(total);
+
+ for (int i = 0; i < undeclaredFields.length; i++) {
+ if (!nullMap.get(i)) {
+ jsonString.append(", ");
+ jsonString.append(" \"" + UNDECLARED_FIELD_NAMES[i] + "\": ");
+ jsonString.append(undeclaredFields[i]);
+ }
+ }
+
+ jsonString.append(" }"); // end order
+ return jsonString.toString();
+ }
+ }
+
+ public void init() {
+ try {
+ List<String> tmpFirstNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpFirstNames.add(capitalized);
+ }
+ in.close();
+ firstNames = tmpFirstNames.toArray(firstNames);
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ List<String> tmpLastNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpLastNames.add(capitalized);
+ }
+ in.close();
+ lastNames = tmpLastNames.toArray(firstNames);
+
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ rndUndeclaredOrderFields = new Random[UNDECLARED_FIELD_NAMES.length];
+ for (int i = 0; i < rndUndeclaredOrderFields.length; i++) {
+ rndUndeclaredOrderFields[i] = new Random(50);
+ }
+ }
+
+ public void writeOrdersList(List<Order> ordersList, FileWriter ordersFile) throws IOException {
+ while (!ordersList.isEmpty()) {
+ int ix = Math.abs(rndValue.nextInt()) % ordersList.size();
+ ordersFile.write(ordersList.get(ix).getJSON() + "\n");
+ ordersList.remove(ix);
+ }
+ }
+
+ public void writeCustomerList(List<Customer> customerList, Order[] ordersBatch, List<Order> ordersList,
+ FileWriter customersFile, FileWriter ordersFile) throws IOException {
+ while (!customerList.isEmpty()) {
+ int ix = Math.abs(rndValue.nextInt()) % customerList.size();
+ customersFile.write(customerList.get(ix).getJSON() + "\n");
+
+ // generate orders
+ int numOrders = Math.abs(rndValue.nextInt()) % (MAX_ORDERS_PER_CUST - MIN_ORDERS_PER_CUST)
+ + MIN_ORDERS_PER_CUST;
+ for (int i = 0; i < numOrders; i++) {
+ ordersBatch[i].generateFieldValues(customerList.get(ix));
+ ordersList.add(ordersBatch[i]);
+ }
+ writeOrdersList(ordersList, ordersFile);
+
+ customerList.remove(ix);
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 2) {
+ System.err.println("MUST PROVIDE 2 PARAMS, 1. output dir name and 2. number of records to generate.");
+ System.exit(1);
+ }
+
+ String outputFile = args[0];
+ int numRecords = Integer.parseInt(args[1]);
+
+ FileWriter customersFile = new FileWriter(outputFile + File.separator + "customer.adm");
+ FileWriter ordersFile = new FileWriter(outputFile + File.separator + "orders.adm");
+
+ CustOrdDataGen dataGen = new CustOrdDataGen();
+ dataGen.init();
+
+ int batchSize = 1000;
+ Customer[] customerBatch = new Customer[batchSize];
+ for (int i = 0; i < batchSize; i++) {
+ customerBatch[i] = dataGen.new Customer();
+ }
+
+ Order[] ordersBatch = new Order[MAX_ORDERS_PER_CUST];
+ for (int i = 0; i < MAX_ORDERS_PER_CUST; i++) {
+ ordersBatch[i] = dataGen.new Order();
+ }
+
+ List<Customer> customerList = new LinkedList<Customer>();
+ List<Order> ordersList = new LinkedList<Order>();
+ int custIx = 0;
+ for (int i = 0; i < numRecords; i++) {
+
+ customerBatch[custIx].generateFieldValues();
+ customerList.add(customerBatch[custIx]);
+ custIx++;
+
+ if (customerList.size() >= batchSize) {
+ dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
+ custIx = 0;
+ }
+ }
+ dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
+ customersFile.flush();
+ customersFile.close();
+
+ ordersFile.flush();
+ ordersFile.close();
+ }
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/EventDataGen.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/EventDataGen.java
new file mode 100644
index 0000000..ad5a863
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/EventDataGen.java
@@ -0,0 +1,219 @@
+package edu.uci.ics.asterix.tools.datagen;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class EventDataGen {
+ private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
+ private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
+
+ private String[] firstNames = new String[10];
+ private String[] lastNames = new String[10];
+
+ private static final int MIN_USER_INTERESTS = 0;
+ private static final int MAX_USER_INTERESTS = 7;
+ private String[] INTERESTS = { "bass", "music", "databases", "fishing", "tennis", "squash", "computers", "books",
+ "movies", "cigars", "wine", "running", "walking", "skiing", "basketball", "video games", "cooking",
+ "coffee", "base jumping", "puzzles" };
+
+ private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
+ "Lake St.", "Hill St.", "Park St.", "View St." };
+ private static final int MIN_STREET_NUM = 1;
+ private static final int MAX_STREET_NUM = 10000;
+ private static final String[] CITIES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
+ "Huntington Beach", "Portland" };
+ private static final int MIN_ZIP = 100000;
+ private static final int MAX_ZIP = 999999;
+ private static final String[] LAT_LONGS = { "47,-122", "33,-117", "33,-117", "34,-118", "33,-117", "33,-117",
+ "45,-122" };
+
+ private static final int MIN_MEMBERSHIPS = 1;
+ private static final int MAX_MEMBERSHIPS = 10;
+ private static final int MIN_SIG_ID = 1;
+ private static final int MAX_SIG_ID = 100;
+ private static final String[] CHAPTER_NAMES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
+ "Huntington Beach", "Portland", "Newport Beach", "Kirkland" };
+ private static final int MEMBER_SINCE_MIN_YEAR = 1970;
+ private static final int MEMBER_SINCE_MAX_YEAR = 1998;
+
+ private Random rndValue = new Random(50);
+ private User user;
+
+ private final class User {
+ private int firstNameIdx;
+ private int lastNameIdx;
+ private int[] interests = new int[MAX_USER_INTERESTS - MIN_USER_INTERESTS];
+ int numInterests;
+ private int streetNumber;
+ private String street;
+ private String city;
+ private int zip;
+ private String latlong;
+ int numMemberships;
+ private int[] member_sigid = new int[MAX_MEMBERSHIPS];
+ private String[] member_chap_name = new String[MAX_MEMBERSHIPS];
+ private String[] member_since_date = new String[MAX_MEMBERSHIPS];
+
+ public void generateFieldValues() {
+ firstNameIdx = Math.abs(rndValue.nextInt()) % firstNames.length;
+ lastNameIdx = Math.abs(rndValue.nextInt()) % lastNames.length;
+ // name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
+ numInterests = Math.abs((rndValue.nextInt()) % (MAX_USER_INTERESTS - MIN_USER_INTERESTS))
+ + MIN_USER_INTERESTS;
+ for (int i = 0; i < numInterests; i++) {
+ interests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
+ }
+ streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
+ street = STREETS[Math.abs(rndValue.nextInt()) % STREETS.length];
+ int cityIdx = Math.abs(rndValue.nextInt()) % CITIES.length;
+ city = CITIES[cityIdx];
+ zip = Math.abs(rndValue.nextInt() % (MAX_ZIP - MIN_ZIP)) + MIN_ZIP;
+ latlong = LAT_LONGS[cityIdx];
+ numMemberships = Math.abs(rndValue.nextInt()) % (MAX_MEMBERSHIPS - MIN_MEMBERSHIPS) + MIN_MEMBERSHIPS;
+ for (int i = 0; i < numMemberships; i++) {
+ member_sigid[i] = Math.abs(rndValue.nextInt()) % (MAX_SIG_ID - MIN_SIG_ID) + MIN_SIG_ID;
+ int cnIdx = Math.abs(rndValue.nextInt()) % CHAPTER_NAMES.length;
+ member_chap_name[i] = CHAPTER_NAMES[cnIdx];
+ int msYear = Math.abs(rndValue.nextInt()) % (MEMBER_SINCE_MAX_YEAR - MEMBER_SINCE_MIN_YEAR)
+ + MEMBER_SINCE_MIN_YEAR;
+ int msMo = Math.abs(rndValue.nextInt()) % 12 + 1;
+ int msDay = Math.abs(rndValue.nextInt()) % 28 + 1;
+ member_since_date[i] = msYear + "-" + (msMo < 10 ? "0" : "") + msMo + "-" + (msDay < 10 ? "0" : "")
+ + msDay;
+ }
+ }
+
+ public void write(Writer writer) throws IOException {
+ writer.append("{");
+ writer.append(" \"name\": \"");
+ writer.append(firstNames[firstNameIdx]);
+ writer.append(" ");
+ writer.append(lastNames[lastNameIdx]);
+ writer.append("\", ");
+
+ writer.append(" \"email\": \"");
+ writer.append(firstNames[firstNameIdx]);
+ writer.append(".");
+ writer.append(lastNames[lastNameIdx]);
+ writer.append("@example.com\", ");
+
+ writer.append(" \"interests\": <");
+ for (int i = 0; i < numInterests; i++) {
+ if (i > 0) {
+ writer.append(", ");
+ }
+ writer.append("\"");
+ writer.append(INTERESTS[interests[i]]);
+ writer.append("\"");
+ }
+ writer.append(">, ");
+
+ writer.append(" \"address\": {");
+ writer.append(" \"street\": \"");
+ writer.append(streetNumber + " " + street);
+ writer.append("\",");
+ writer.append(" \"city\": \"");
+ writer.append(city);
+ writer.append("\",");
+ writer.append(" \"zip\": \"");
+ writer.append(zip + "\",");
+ writer.append(" \"latlong\": point(\"");
+ writer.append(latlong);
+ writer.append("\")");
+ writer.append("}, ");
+
+ writer.append(" \"member_of\": <");
+ for (int i = 0; i < numMemberships; i++) {
+ if (i > 0) {
+ writer.append(", ");
+ }
+ writer.append("{");
+ writer.append(" \"sig_id\": ");
+ writer.append(member_sigid[i] + ",");
+ writer.append(" \"chapter_name\": \"");
+ writer.append(member_chap_name[i]);
+ writer.append("\",");
+ writer.append(" \"member_since\": date(\"");
+ writer.append(member_since_date[i]);
+ writer.append("\") }");
+ }
+ writer.append(">");
+
+ writer.append(" }\n");
+ }
+ }
+
+ public void init() throws IOException {
+ {
+ List<String> tmpFirstNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpFirstNames.add(capitalized);
+ }
+ in.close();
+ firstNames = tmpFirstNames.toArray(firstNames);
+ }
+ {
+ List<String> tmpLastNames = new ArrayList<String>();
+
+ FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
+ DataInputStream in = new DataInputStream(fstream);
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ String strLine;
+ while ((strLine = br.readLine()) != null) {
+ String firstLetter = strLine.substring(0, 1);
+ String remainder = strLine.substring(1);
+ String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+ tmpLastNames.add(capitalized);
+ }
+ in.close();
+ lastNames = tmpLastNames.toArray(firstNames);
+ }
+ user = new User();
+ }
+
+ public void generate() {
+ user.generateFieldValues();
+ }
+
+ public void write(Writer w) throws IOException {
+ user.write(w);
+ }
+
+ public static void main(String[] args) throws IOException {
+ if (args.length != 2) {
+ System.err
+ .println("MUST PROVIDE 2 PARAMETERS, 1. output directory path and 2. number of records to generate.");
+ System.exit(1);
+ }
+ String outputFile = args[0];
+ int numRecords = Integer.parseInt(args[1]);
+ Writer userFile = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile + File.separator
+ + "user.adm")));
+ EventDataGen dgen = new EventDataGen();
+ dgen.init();
+ for (int i = 0; i < numRecords; i++) {
+ dgen.generate();
+ dgen.write(userFile);
+ }
+ userFile.close();
+ }
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/tbltoadm/TblToAdm.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/tbltoadm/TblToAdm.java
new file mode 100644
index 0000000..0a09f68
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/tbltoadm/TblToAdm.java
@@ -0,0 +1,79 @@
+package edu.uci.ics.asterix.tools.tbltoadm;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+public class TblToAdm {
+
+ private static final String SEPARATOR_STRING = "\\|\\s*";
+ private static final char SEPARATOR_CHAR = '|';
+
+ private static void convertFile(String inputFileName, String outputFileName, String scmFileName) throws IOException {
+ File scmFile = new File(scmFileName);
+ File inFile = new File(inputFileName);
+ File outFile = new File(outputFileName);
+
+ BufferedReader scmReader = new BufferedReader(new FileReader(scmFile));
+ String scmStr = scmReader.readLine();
+ String[] fieldInfo = scmStr.split(SEPARATOR_STRING);
+ String[] fieldNames = new String[fieldInfo.length];
+ boolean[] fieldIsString = new boolean[fieldInfo.length];
+ for (int i = 0; i < fieldInfo.length; i++) {
+ String[] f = fieldInfo[i].split(":\\s*");
+ fieldNames[i] = f[0];
+ fieldIsString[i] = f[1].equals("string");
+ }
+ scmReader.close();
+
+ BufferedReader reader = new BufferedReader(new FileReader(inFile));
+ PrintWriter writer = new PrintWriter(outFile);
+ String row;
+ while ((row = reader.readLine()) != null) {
+ // String[] fields = row.split(SEPARATOR);
+ if (row.length() == 0 || row.charAt(0) == ' ' || row.charAt(0) == '\n') {
+ continue;
+ }
+ writer.write("{ ");
+ int pos = 0;
+ int strlen = row.length();
+ for (int i = 0; i < fieldNames.length; i++) {
+ if (i > 0) {
+ writer.write(", ");
+ }
+ writer.print('"');
+ writer.write(fieldNames[i]);
+ writer.write("\": ");
+ if (fieldIsString[i]) {
+ writer.print('"');
+ }
+ for (; pos < strlen; pos++) {
+ char c = row.charAt(pos);
+ if (c != SEPARATOR_CHAR) {
+ writer.print(c);
+
+ } else {
+ ++pos;
+ break;
+ }
+ }
+ if (fieldIsString[i]) {
+ writer.print('"');
+ }
+ }
+ writer.write(" }\n");
+ }
+ writer.close();
+ reader.close();
+ }
+
+ public static void main(String args[]) throws Exception {
+ if (args.length != 3) {
+ System.err.println("Usage: bash tbl2adm <srcfile.tbl> <destfile.adm> <schema-file>");
+ return;
+ }
+ convertFile(args[0], args[1], args[2]);
+ }
+}
diff --git a/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/translator/ADGenDmlTranslator.java b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/translator/ADGenDmlTranslator.java
new file mode 100644
index 0000000..5023ec6
--- /dev/null
+++ b/asterix-tools/src/main/java/edu/uci/ics/asterix/tools/translator/ADGenDmlTranslator.java
@@ -0,0 +1,30 @@
+package edu.uci.ics.asterix.tools.translator;
+
+import java.util.List;
+
+import edu.uci.ics.asterix.aql.base.Statement;
+import edu.uci.ics.asterix.metadata.MetadataException;
+import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
+import edu.uci.ics.asterix.metadata.declared.AqlCompiledMetadataDeclarations;
+import edu.uci.ics.asterix.translator.AbstractAqlTranslator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class ADGenDmlTranslator extends AbstractAqlTranslator {
+
+ private final MetadataTransactionContext mdTxnCtx;
+ private List<Statement> aqlStatements;
+ private AqlCompiledMetadataDeclarations compiledDeclarations;
+
+ public ADGenDmlTranslator(MetadataTransactionContext mdTxnCtx, List<Statement> aqlStatements) {
+ this.mdTxnCtx = mdTxnCtx;
+ this.aqlStatements = aqlStatements;
+ }
+
+ public void translate() throws AlgebricksException, MetadataException {
+ compiledDeclarations = compileMetadata(mdTxnCtx, aqlStatements, false);
+ }
+
+ public AqlCompiledMetadataDeclarations getCompiledDeclarations() {
+ return compiledDeclarations;
+ }
+}
diff --git a/asterix-tools/src/main/resources/test.properties b/asterix-tools/src/main/resources/test.properties
new file mode 100755
index 0000000..e9ccc63
--- /dev/null
+++ b/asterix-tools/src/main/resources/test.properties
@@ -0,0 +1,4 @@
+MetadataNode=nc1
+NewUniverse=true
+nc1.stores=/tmp/nc1data/
+nc2.stores=/tmp/nc2data/, /tmp/nc2data1/
\ No newline at end of file
diff --git a/asterix-tools/src/test/java/edu/uci/ics/asterix/tools/test/AdmDataGenTest.java b/asterix-tools/src/test/java/edu/uci/ics/asterix/tools/test/AdmDataGenTest.java
new file mode 100644
index 0000000..65d0760
--- /dev/null
+++ b/asterix-tools/src/test/java/edu/uci/ics/asterix/tools/test/AdmDataGenTest.java
@@ -0,0 +1,197 @@
+package edu.uci.ics.asterix.tools.test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.logging.Logger;
+
+import org.junit.AfterClass;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import edu.uci.ics.asterix.test.base.AsterixTestHelper;
+import edu.uci.ics.asterix.tools.datagen.AdmDataGen;
+
+@RunWith(Parameterized.class)
+public class AdmDataGenTest {
+
+ private static final Logger LOGGER = Logger.getLogger(AdmDataGenTest.class.getName());
+
+ private static final String SEPARATOR = File.separator;
+ private static final String EXTENSION_QUERY = "adg";
+ private static final String FILENAME_IGNORE = "ignore.txt";
+ private static final String FILENAME_ONLY = "only.txt";
+ private static final String PATH_BASE = "src" + SEPARATOR + "test" + SEPARATOR + "resources" + SEPARATOR + "adgts"
+ + SEPARATOR;
+ private static final String PATH_QUERIES = PATH_BASE + "dgscripts" + SEPARATOR;
+ private static final String PATH_EXPECTED = PATH_BASE + "results" + SEPARATOR;
+ private static final String PATH_ACTUAL = "adgtest" + SEPARATOR;
+
+ private static final ArrayList<String> ignore = AsterixTestHelper.readFile(FILENAME_IGNORE, PATH_BASE);
+ private static final ArrayList<String> only = AsterixTestHelper.readFile(FILENAME_ONLY, PATH_BASE);
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ File outdir = new File(PATH_ACTUAL);
+ if (outdir.exists()) {
+ AsterixTestHelper.deleteRec(outdir);
+ }
+ outdir.mkdirs();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ // _bootstrap.stop();
+ File outdir = new File(PATH_ACTUAL);
+ File[] files = outdir.listFiles();
+ if (files == null || files.length == 0) {
+ outdir.delete();
+ }
+ }
+
+ private static void suiteBuild(File dir, Collection<Object[]> testArgs, String path) {
+ for (File file : dir.listFiles()) {
+ if (file.isDirectory() && !file.getName().startsWith(".")) {
+ suiteBuild(file, testArgs, path + file.getName() + SEPARATOR);
+ }
+ if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY)
+ // && !ignore.contains(path + file.getName())
+ ) {
+ File expectedDir = new File(PATH_EXPECTED + path);
+ File actualDir = new File(PATH_ACTUAL + SEPARATOR + path);
+ testArgs.add(new Object[] { file, expectedDir, actualDir });
+ }
+ }
+ }
+
+ @Parameters
+ public static Collection<Object[]> tests() {
+ Collection<Object[]> testArgs = new ArrayList<Object[]>();
+ suiteBuild(new File(PATH_QUERIES), testArgs, "");
+ return testArgs;
+ }
+
+ private File actualDir;
+ private File expectedDir;
+ private File scriptFile;
+
+ public AdmDataGenTest(File scriptFile, File expectedDir, File actualDir) {
+ this.scriptFile = scriptFile;
+ this.expectedDir = expectedDir;
+ this.actualDir = actualDir;
+ }
+
+ @Test
+ public void test() throws Exception {
+ String scriptFileShort = scriptFile.getPath().substring(PATH_QUERIES.length())
+ .replace(SEPARATOR.charAt(0), '/');
+ if (!only.isEmpty()) {
+ if (!only.contains(scriptFileShort)) {
+ LOGGER.info("SKIP TEST: \"" + scriptFile.getPath()
+ + "\" \"only.txt\" not empty and not in \"only.txt\".");
+ }
+ Assume.assumeTrue(only.contains(scriptFileShort));
+ }
+ if (ignore.contains(scriptFileShort)) {
+ LOGGER.info("SKIP TEST: \"" + scriptFile.getPath() + "\" in \"ignore.txt\".");
+ }
+ Assume.assumeTrue(!ignore.contains(scriptFileShort));
+
+ LOGGER.info("RUN TEST: \"" + scriptFile.getPath() + "\"");
+
+ actualDir.mkdirs();
+ AdmDataGen adg = new AdmDataGen(scriptFile, actualDir);
+ try {
+ adg.init();
+ adg.dataGen();
+ } catch (Exception e) {
+ throw new Exception("Data gen. ERROR for " + scriptFile + ": " + e.getMessage(), e);
+ }
+
+ if (!expectedDir.isDirectory()) {
+ throw new Exception(expectedDir + " is not a directory.");
+ }
+ if (!actualDir.isDirectory()) {
+ throw new Exception(expectedDir + " is not a directory.");
+ }
+
+ File[] expectedFileSet = expectedDir.listFiles(AdmFileFilter.INSTANCE);
+ File[] actualFileSet = actualDir.listFiles(AdmFileFilter.INSTANCE);
+
+ if (expectedFileSet.length != actualFileSet.length) {
+ throw new Exception("Expecting " + expectedFileSet.length + " files and found " + actualFileSet.length
+ + " files instead.");
+ }
+
+ for (File expectedFile : expectedFileSet) {
+ if (expectedFile.isHidden()) {
+ continue;
+ }
+ File actualFile = null;
+ for (File f : actualFileSet) {
+ if (f.getName().equals(expectedFile.getName())) {
+ actualFile = f;
+ break;
+ }
+ }
+ if (actualFile == null) {
+ throw new Exception("Could not find file " + expectedFile.getName());
+ }
+
+ BufferedReader readerExpected = new BufferedReader(new FileReader(expectedFile));
+ BufferedReader readerActual = new BufferedReader(new FileReader(actualFile));
+
+ String lineExpected, lineActual;
+ int num = 1;
+ try {
+ while ((lineExpected = readerExpected.readLine()) != null) {
+ lineActual = readerActual.readLine();
+ // Assert.assertEquals(lineExpected, lineActual);
+ if (lineActual == null) {
+ throw new Exception("Result for " + scriptFile + " changed at line " + num + ":\n< "
+ + lineExpected + "\n> ");
+ }
+ if (!lineExpected.equals(lineActual)) {
+ throw new Exception("Result for " + scriptFile + " changed at line " + num + ":\n< "
+ + lineExpected + "\n> " + lineActual);
+ }
+ ++num;
+ }
+ lineActual = readerActual.readLine();
+ // Assert.assertEquals(null, lineActual);
+ if (lineActual != null) {
+ throw new Exception("Result for " + scriptFile + " changed at line " + num + ":\n< \n> "
+ + lineActual);
+ }
+ } finally {
+ readerExpected.close();
+ readerActual.close();
+ }
+ }
+ AsterixTestHelper.deleteRec(actualDir);
+ }
+
+ private static class AdmFileFilter implements FileFilter {
+
+ public static final AdmFileFilter INSTANCE = new AdmFileFilter();
+
+ private AdmFileFilter() {
+ }
+
+ @Override
+ public boolean accept(File path) {
+ if (path.isHidden() || !path.isFile()) {
+ return false;
+ }
+ return path.getName().endsWith(".adm");
+ }
+ }
+
+}
diff --git a/asterix-tools/src/test/resources/adgts/dgscripts/events/event.adg b/asterix-tools/src/test/resources/adgts/dgscripts/events/event.adg
new file mode 100644
index 0000000..b757994
--- /dev/null
+++ b/asterix-tools/src/test/resources/adgts/dgscripts/events/event.adg
@@ -0,0 +1,51 @@
+drop dataverse events if exists;
+create dataverse events;
+use dataverse events;
+
+
+create type AddressType as closed {
+ /*+ val-files data/events/streets.txt */
+ street: string,
+ /*+ val-files data/events/cities.txt */
+ city: string,
+ /*+ interval long 100000 999999 */
+ zip: string,
+ /*+ val-file-same-idx data/events/latlongs.txt city */
+ latlong: point
+}
+
+/*+ dgen event.adm 5*/
+create type EventType as closed {
+ /*+ auto 100000 */
+ event_id: int64,
+ /*+ insert-rand-int event- - */
+ name: string,
+ location: AddressType ?,
+ /*+ list 1 3 */
+ organizers: {{
+ /*+ gen-fields int 0 2 other */
+ {
+ /*+ val-files data/uscensus/dist.all.first.cleaned data/uscensus/dist.all.last.cleaned */
+ name: string
+ }
+ }},
+ /*+ list 1 2 */
+ sponsoring_sigs: [
+ {
+ /*+ interval int 100 120 */
+ sig_id: int32,
+ /*+ val-files data/events/chapter_names.txt */
+ chapter_name: string
+ }
+ ],
+ /*+ list-val-file data/events/interests.txt 1 4 */
+ interest_keywords: {{string}},
+ /*+ interval double 5 20 */
+ price: double?,
+ /*+ datetime-between-years 1999 2012 */
+ start_time: datetime,
+ /*+ datetime-add-rand-hours 1 4 start_time */
+ end_time: datetime
+}
+
+
diff --git a/asterix-tools/src/test/resources/adgts/dgscripts/users/user.adg b/asterix-tools/src/test/resources/adgts/dgscripts/users/user.adg
new file mode 100644
index 0000000..8ad2ad5
--- /dev/null
+++ b/asterix-tools/src/test/resources/adgts/dgscripts/users/user.adg
@@ -0,0 +1,39 @@
+drop dataverse events if exists;
+create dataverse events;
+use dataverse events;
+
+
+create type AddressType as closed {
+ /*+ val-files data/events/streets.txt */
+ street: string,
+ /*+ val-files data/events/cities.txt */
+ city: string,
+ /*+ interval long 100000 999999 */
+ zip: string,
+ /*+ val-file-same-idx data/events/latlongs.txt city */
+ latlong: point
+}
+
+/*+ dgen user.adm 3*/
+create type UserType as open /*+ gen-fields int 0 10 field */ {
+ /*+ val-files data/uscensus/dist.all.first.cleaned data/uscensus/dist.all.last.cleaned */
+ name: string,
+ /*+ insert-rand-int user @example.com */
+ email: string,
+ /*+ list-val-file data/events/interests.txt 0 7 */
+ interests: {{string}},
+ address: AddressType,
+ /*+ list 1 10 */
+ member_of: {{
+ /*+ gen-fields int 0 2 other */
+ {
+ /*+ interval int 100000 999999 */
+ sig_id: int32,
+ /*+ val-files data/events/chapter_names.txt */
+ chapter_name: string,
+ /*+ date-between-years 1970 1998 */
+ member_since: date
+ }
+ }}
+}
+
diff --git a/asterix-tools/src/test/resources/adgts/only.txt b/asterix-tools/src/test/resources/adgts/only.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-tools/src/test/resources/adgts/only.txt
diff --git a/asterix-tools/src/test/resources/adgts/results/events/event.adm b/asterix-tools/src/test/resources/adgts/results/events/event.adm
new file mode 100644
index 0000000..dd5b654
--- /dev/null
+++ b/asterix-tools/src/test/resources/adgts/results/events/event.adm
@@ -0,0 +1,5 @@
+{ "event_id": 100000i64, "name": "event-1161255810-", "location": { "street": "Washington St.", "city": "Irvine", "zip": "245704", "latlong": point("33,-117") }, "organizers": {{{ "name": "Ruth Sasala", "other_1": 928991271, "other_0": 824112980 }, { "name": "Bo Sanchz" }, { "name": "Karan Eye", "other_0": 647271456 }}}, "sponsoring_sigs": [{ "sig_id": 115, "chapter_name": "Los Angeles" }, { "sig_id": 115, "chapter_name": "Seattle" }], "interest_keywords": {{"tennis", "running", "bass"}}, "price": 5.729624060793583, "start_time": datetime("2004-01-20T02:00:00"), "end_time": datetime("2004-01-20T05:00:00") }
+{ "event_id": 100001i64, "name": "event-571555944-", "location": { "street": "Main St.", "city": "Irvine", "zip": "589756", "latlong": point("33,-117") }, "organizers": {{{ "name": "Tabatha Lammel", "other_0": 1064393481, "other_1": 811017688 }, { "name": "Magda Eska" }, { "name": "Joye Wholey", "other_1": 1465062491, "other_0": 675875773 }}}, "sponsoring_sigs": [{ "sig_id": 112, "chapter_name": "Los Angeles" }, { "sig_id": 112, "chapter_name": "Seattle" }], "interest_keywords": {{"squash"}}, "price": 5.783702367883383, "start_time": datetime("2005-02-11T23:00:00"), "end_time": datetime("2005-02-11T23:00:00") }
+{ "event_id": 100002i64, "name": "event-928991271-", "organizers": {{{ "name": "Grady Miyashiro", "other_0": 1753250755 }}}, "sponsoring_sigs": [{ "sig_id": 105, "chapter_name": "Kirkland" }, { "sig_id": 119, "chapter_name": "San Clemente" }], "interest_keywords": {{"wine", "cooking"}}, "start_time": datetime("2009-07-20T03:00:00"), "end_time": datetime("2009-07-20T07:00:00") }
+{ "event_id": 100003i64, "name": "event-1620216540-", "organizers": {{{ "name": "Ione Janiszewski", "other_0": 1449157330 }, { "name": "Ilse Hurse" }, { "name": "Mason Freudenthal", "other_0": 172885567 }}}, "sponsoring_sigs": [{ "sig_id": 117, "chapter_name": "Kirkland" }, { "sig_id": 104, "chapter_name": "Irvine" }], "interest_keywords": {{"fishing", "music", "skiing"}}, "start_time": datetime("2009-05-23T14:00:00"), "end_time": datetime("2009-05-23T17:00:00") }
+{ "event_id": 100004i64, "name": "event-824112980-", "organizers": {{{ "name": "Soo Raul" }, { "name": "Leif Henshall" }}}, "sponsoring_sigs": [{ "sig_id": 106, "chapter_name": "Portland" }], "interest_keywords": {{"wine", "walking", "basketball", "running"}}, "price": 5.191878754151207, "start_time": datetime("2002-12-23T01:00:00"), "end_time": datetime("2002-12-23T02:00:00") }
diff --git a/asterix-tools/src/test/resources/adgts/results/users/user.adm b/asterix-tools/src/test/resources/adgts/results/users/user.adm
new file mode 100644
index 0000000..35889dd
--- /dev/null
+++ b/asterix-tools/src/test/resources/adgts/results/users/user.adm
@@ -0,0 +1,3 @@
+{ "name": "Ruth Sasala", "email": "user1161255810@example.com", "interests": {{"tennis", "running", "bass", "bass", "squash"}}, "address": { "street": "Washington St.", "city": "Irvine", "zip": "245704", "latlong": point("33,-117") }, "member_of": {{{ "sig_id": 355810, "chapter_name": "Los Angeles", "member_since": date("1981-01-20"), "other_1": 928991271, "other_0": 824112980 }, { "sig_id": 155944, "chapter_name": "Seattle", "member_since": date("1980-09-06") }, { "sig_id": 291271, "chapter_name": "Los Angeles", "member_since": date("1973-11-21"), "other_0": 647271456 }, { "sig_id": 316540, "chapter_name": "Seattle", "member_since": date("1988-08-26"), "other_0": 1064393481, "other_1": 811017688 }}}, "field_1": 928991271, "field_0": 824112980 }
+{ "name": "Bo Sanchz", "email": "user571555944@example.com", "interests": {{"wine", "cooking"}}, "address": { "street": "Main St.", "city": "Irvine", "zip": "589756", "latlong": point("33,-117") }, "member_of": {{{ "sig_id": 712980, "chapter_name": "Kirkland", "member_since": date("1997-05-23") }, { "sig_id": 746825, "chapter_name": "San Clemente", "member_since": date("1980-03-12"), "other_1": 1465062491, "other_0": 675875773 }, { "sig_id": 409158, "chapter_name": "Kirkland", "member_since": date("1976-02-22"), "other_0": 1753250755 }, { "sig_id": 806510, "chapter_name": "Irvine", "member_since": date("1980-08-24"), "other_0": 1449157330 }, { "sig_id": 271456, "chapter_name": "Portland", "member_since": date("1997-11-27") }, { "sig_id": 167494, "chapter_name": "Los Angeles", "member_since": date("1976-08-16"), "other_0": 172885567 }, { "sig_id": 363383, "chapter_name": "Newport Beach", "member_since": date("1982-11-08") }}}, "field_0": 1051906510 }
+{ "name": "Karan Eye", "email": "user928991271@example.com", "interests": {{"fishing", "music", "skiing", "movies", "wine"}}, "address": { "street": "Washington St.", "city": "Huntington Beach", "zip": "248905", "latlong": point("33,-117") }, "member_of": {{{ "sig_id": 693481, "chapter_name": "Los Angeles", "member_since": date("1988-01-03") }, { "sig_id": 354593, "chapter_name": "Irvine", "member_since": date("1995-03-21"), "other_0": 1721620164 }, { "sig_id": 217688, "chapter_name": "San Clemente", "member_since": date("1982-01-19"), "other_0": 646860650 }}}, "field_5": 1832663383, "field_6": 270254593, "field_3": 1332296190, "field_2": 432209174, "field_0": 520792154, "field_7": 464970681, "field_4": 1753250755, "field_8": 1539455368, "field_1": 1167889770 }
diff --git a/asterix-tools/src/test/resources/logging.properties b/asterix-tools/src/test/resources/logging.properties
new file mode 100644
index 0000000..a205eca
--- /dev/null
+++ b/asterix-tools/src/test/resources/logging.properties
@@ -0,0 +1,65 @@
+############################################################
+# Default Logging Configuration File
+#
+# You can use a different file by specifying a filename
+# with the java.util.logging.config.file system property.
+# For example java -Djava.util.logging.config.file=myfile
+############################################################
+
+############################################################
+# Global properties
+############################################################
+
+# "handlers" specifies a comma separated list of log Handler
+# classes. These handlers will be installed during VM startup.
+# Note that these classes must be on the system classpath.
+# By default we only configure a ConsoleHandler, which will only
+# show messages at the INFO and above levels.
+
+handlers= java.util.logging.ConsoleHandler
+
+# To also add the FileHandler, use the following line instead.
+
+# handlers= java.util.logging.FileHandler, java.util.logging.ConsoleHandler
+
+# Default global logging level.
+# This specifies which kinds of events are logged across
+# all loggers. For any given facility this global level
+# can be overriden by a facility specific level
+# Note that the ConsoleHandler also has a separate level
+# setting to limit messages printed to the console.
+
+.level= WARNING
+# .level= INFO
+# .level= FINE
+# .level = FINEST
+
+############################################################
+# Handler specific properties.
+# Describes specific configuration info for Handlers.
+############################################################
+
+# default file output is in user's home directory.
+
+# java.util.logging.FileHandler.pattern = %h/java%u.log
+# java.util.logging.FileHandler.limit = 50000
+# java.util.logging.FileHandler.count = 1
+# java.util.logging.FileHandler.formatter = java.util.logging.XMLFormatter
+
+# Limit the message that are printed on the console to FINE and above.
+
+java.util.logging.ConsoleHandler.level = FINE
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter
+
+
+############################################################
+# Facility specific properties.
+# Provides extra control for each logger.
+############################################################
+
+# For example, set the com.xyz.foo logger to only log SEVERE
+# messages:
+
+# edu.uci.ics.asterix.level = FINE
+# edu.uci.ics.algebricks.level = FINE
+# edu.uci.ics.hyracks.level = FINE