add valuewritable
git-svn-id: https://hyracks.googlecode.com/svn/branches/fullstack_genomix@2954 123451ca-8445-de46-9d55-352943316053
diff --git a/genomix/genomix-hadoop/actual/conf.xml b/genomix/genomix-hadoop/actual/conf.xml
index d544e83..8d2522e 100755
--- a/genomix/genomix-hadoop/actual/conf.xml
+++ b/genomix/genomix-hadoop/actual/conf.xml
@@ -12,7 +12,7 @@
<property><name>dfs.namenode.logging.level</name><value>info</value></property>
<property><name>dfs.datanode.address</name><value>127.0.0.1:0</value></property>
<property><name>io.skip.checksum.errors</name><value>false</value></property>
-<property><name>fs.default.name</name><value>hdfs://localhost:58914</value></property>
+<property><name>fs.default.name</name><value>hdfs://localhost:49270</value></property>
<property><name>mapred.child.tmp</name><value>./tmp</value></property>
<property><name>fs.har.impl.disable.cache</name><value>true</value></property>
<property><name>dfs.safemode.threshold.pct</name><value>0.999f</value></property>
@@ -43,7 +43,7 @@
<property><name>fs.ramfs.impl</name><value>org.apache.hadoop.fs.InMemoryFileSystem</value></property>
<property><name>dfs.name.edits.dir</name><value>${dfs.name.dir}</value></property>
<property><name>mapred.skip.map.max.skip.records</name><value>0</value></property>
-<property><name>hadoop.job.ugi</name><value>yingyib,staff,com.apple.access_ssh,com.apple.access_screensharing,com.apple.access_remote_ae,everyone,_appstore,localaccounts,_appserverusr,admin,_appserveradm,_lpadmin,_lpoperator,_developer</value></property>
+<property><name>hadoop.job.ugi</name><value>hadoop,staff,everyone,localaccounts</value></property>
<property><name>mapred.job.tracker.persist.jobstatus.dir</name><value>/jobtracker/jobsInfo</value></property>
<property><name>dfs.block.size</name><value>67108864</value></property>
<property><name>fs.s3.buffer.dir</name><value>${hadoop.tmp.dir}/s3</value></property>
@@ -125,7 +125,7 @@
<property><name>mapred.map.output.compression.codec</name><value>org.apache.hadoop.io.compress.DefaultCodec</value></property>
<property><name>mapred.tasktracker.dns.interface</name><value>default</value></property>
<property><name>dfs.namenode.decommission.interval</name><value>3</value></property>
-<property><name>dfs.http.address</name><value>localhost:58915</value></property>
+<property><name>dfs.http.address</name><value>localhost:49271</value></property>
<property><name>dfs.heartbeat.interval</name><value>3</value></property>
<property><name>mapred.job.tracker</name><value>local</value></property>
<property><name>io.seqfile.sorter.recordlimit</name><value>1000000</value></property>
diff --git a/genomix/genomix-hadoop/actual/result2/.part-00000.crc b/genomix/genomix-hadoop/actual/result2/.part-00000.crc
index 1bb3d3d..d254c47 100755
--- a/genomix/genomix-hadoop/actual/result2/.part-00000.crc
+++ b/genomix/genomix-hadoop/actual/result2/.part-00000.crc
Binary files differ
diff --git a/genomix/genomix-hadoop/actual/result2/part-00000 b/genomix/genomix-hadoop/actual/result2/part-00000
index 122fd34..4c44896 100755
--- a/genomix/genomix-hadoop/actual/result2/part-00000
+++ b/genomix/genomix-hadoop/actual/result2/part-00000
Binary files differ
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixCombiner.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixCombiner.java
index b66ee15..a72d740 100755
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixCombiner.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixCombiner.java
@@ -1,5 +1,3 @@
-package edu.uci.ics.graphbuilding;
-
/*
* Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+package edu.uci.ics.graphbuilding;
+
import java.io.IOException;
import java.util.Iterator;
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixDriver.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixDriver.java
index 71a675e..4699a7d 100755
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixDriver.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixDriver.java
@@ -1,5 +1,3 @@
-package edu.uci.ics.graphbuilding;
-
/*
* Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+package edu.uci.ics.graphbuilding;
+
import java.io.IOException;
import org.apache.hadoop.fs.FileSystem;
@@ -73,16 +74,11 @@
conf.setMapOutputValueClass(ValueWritable.class);
conf.setInputFormat(TextInputFormat.class);
-// conf.setOutputFormat(TextOutputFormat.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
conf.setOutputKeyClass(ValueBytesWritable.class);
conf.setOutputValueClass(ValueWritable.class);
FileInputFormat.setInputPaths(conf, new Path(inputPath));
FileOutputFormat.setOutputPath(conf, new Path(outputPath));
-// SequenceFileOutputFormat.setOutputPath(conf,new Path(outputPath));
-// SequenceFileOutputFormat.setCompressOutput(conf, true);
-// SequenceFileOutputFormat.setOutputCompressorClass(conf, GzipCodec.class);
-// SequenceFileOutputFormat.setOutputCompressionType(conf, CompressionType.BLOCK);
conf.setNumReduceTasks(numReducers);
FileSystem dfs = FileSystem.get(conf);
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixMapper.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixMapper.java
index 7162c91..d711114 100755
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixMapper.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixMapper.java
@@ -1,5 +1,3 @@
-package edu.uci.ics.graphbuilding;
-
/*
* Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,16 +12,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+package edu.uci.ics.graphbuilding;
+
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.IntWritable;
+//import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.VLongWritable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
@@ -33,6 +31,7 @@
/**
* This class implement mapper operator of mapreduce model
*/
+@SuppressWarnings("deprecation")
public class GenomixMapper extends MapReduceBase implements
Mapper<LongWritable, Text, ValueBytesWritable, ValueWritable> {
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixReducer.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixReducer.java
index a9584f6..951f7b1 100755
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixReducer.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/GenomixReducer.java
@@ -1,5 +1,3 @@
-package edu.uci.ics.graphbuilding;
-
/*
* Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +12,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package edu.uci.ics.graphbuilding;
+
import java.io.IOException;
import java.util.Iterator;
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/ValueBytesWritable.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/ValueBytesWritable.java
new file mode 100644
index 0000000..331d5c7
--- /dev/null
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/ValueBytesWritable.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.graphbuilding;
+
+import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
+import org.apache.hadoop.io.BinaryComparable;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.WritableComparator;
+
+public class ValueBytesWritable extends BinaryComparable implements WritableComparable<BinaryComparable> {
+ private static final int LENGTH_BYTES = 4;
+ private static final byte[] EMPTY_BYTES = {};
+ private byte size;
+ private byte[] bytes;
+
+ public ValueBytesWritable() {
+ this(EMPTY_BYTES);
+ }
+
+ public ValueBytesWritable(byte[] bytes) {
+ this.bytes = bytes;
+ this.size = (byte) bytes.length;
+ }
+
+ @Override
+ public byte[] getBytes() {
+ return bytes;
+ }
+
+ @Deprecated
+ public byte[] get() {
+ return getBytes();
+ }
+
+ @Override
+ public int getLength() {
+ return (int) size;
+ }
+
+ @Deprecated
+ public int getSize() {
+ return getLength();
+ }
+
+ public void setSize(byte size) {
+ if ((int) size > getCapacity()) {
+ setCapacity((byte) (size * 3 / 2));
+ }
+ this.size = size;
+ }
+
+ public int getCapacity() {
+ return bytes.length;
+ }
+
+ public void setCapacity(byte new_cap) {
+ if (new_cap != getCapacity()) {
+ byte[] new_data = new byte[new_cap];
+ if (new_cap < size) {
+ size = new_cap;
+ }
+ if (size != 0) {
+ System.arraycopy(bytes, 0, new_data, 0, size);
+ }
+ bytes = new_data;
+ }
+ }
+
+ public void set(ValueBytesWritable newData) {
+ set(newData.bytes, (byte) 0, newData.size);
+ }
+
+ public void set(byte[] newData, byte offset, byte length) {
+ setSize((byte) 0);
+ setSize(length);
+ System.arraycopy(newData, offset, bytes, 0, size);
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ setSize((byte) 0); // clear the old data
+ setSize(in.readByte());
+ in.readFully(bytes, 0, size);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeByte(size);
+ out.write(bytes, 0, size);
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object right_obj) {
+ if (right_obj instanceof ValueBytesWritable)
+ return super.equals(right_obj);
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer(3 * size);
+ for (int idx = 0; idx < (int) size; idx++) {
+ // if not the first, put a blank separator in
+ if (idx != 0) {
+ sb.append(' ');
+ }
+ String num = Integer.toHexString(0xff & bytes[idx]);
+ // if it is only one digit, add a leading 0.
+ if (num.length() < 2) {
+ sb.append('0');
+ }
+ sb.append(num);
+ }
+ return sb.toString();
+ }
+
+ public static class Comparator extends WritableComparator {
+ public Comparator() {
+ super(ValueBytesWritable.class);
+ }
+
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ return compareBytes(b1, s1 + LENGTH_BYTES, l1 - LENGTH_BYTES, b2, s2 + LENGTH_BYTES, l2 - LENGTH_BYTES);
+ }
+ }
+
+ static { // register this comparator
+ WritableComparator.define(ValueBytesWritable.class, new Comparator());
+ }
+}
diff --git a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/ValueWritable.java b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/ValueWritable.java
index 2989ae6..14fec79 100755
--- a/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/ValueWritable.java
+++ b/genomix/genomix-hadoop/src/main/java/edu/uci/ics/graphbuilding/ValueWritable.java
@@ -1,5 +1,3 @@
-package edu.uci.ics.graphbuilding;
-
/*
* Copyright 2009-2012 by The Regents of the University of California
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +12,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package edu.uci.ics.graphbuilding;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
diff --git a/genomix/genomix-hadoop/src/test/java/edu/uci/ics/graphbuilding/GraphBuildingTest.java b/genomix/genomix-hadoop/src/test/java/edu/uci/ics/graphbuilding/GraphBuildingTest.java
index 754f99d..3b6c377 100755
--- a/genomix/genomix-hadoop/src/test/java/edu/uci/ics/graphbuilding/GraphBuildingTest.java
+++ b/genomix/genomix-hadoop/src/test/java/edu/uci/ics/graphbuilding/GraphBuildingTest.java
@@ -81,7 +81,7 @@
bw.newLine();
}
bw.close();
-
+
dumpResult();
TestUtils.compareWithResult(new File(TEST_SOURCE_DIR), new File(EXPECTED_PATH));