Merge branch 'master' into jarodwen/hotfix/issue425
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
index d6e0f42..282d26b 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
@@ -188,8 +188,12 @@
// if already rewritten, the required type is not null
if (TypeComputerUtilities.getRequiredType(funcExpr) != null)
return false;
- TypeComputerUtilities.setRequiredAndInputTypes(funcExpr, requiredRecordType, inputRecordType);
- return staticRecordTypeCast(funcExpr, requiredRecordType, inputRecordType, env);
+ boolean casted = staticRecordTypeCast(funcExpr, requiredRecordType, inputRecordType, env);
+ if (casted) {
+ //enforce the required type if it is statically casted
+ TypeComputerUtilities.setRequiredAndInputTypes(funcExpr, requiredRecordType, inputRecordType);
+ }
+ return casted;
}
/**
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java
index 9fe5750..792ff25 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AbstractAqlTranslator.java
@@ -30,6 +30,7 @@
import edu.uci.ics.asterix.metadata.entities.AsterixBuiltinTypeMap;
import edu.uci.ics.asterix.metadata.entities.Dataverse;
import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
/**
@@ -41,6 +42,12 @@
protected static final Map<String, BuiltinType> builtinTypeMap = AsterixBuiltinTypeMap.getBuiltinTypes();
public void validateOperation(Dataverse defaultDataverse, Statement stmt) throws AsterixException {
+
+ if (AsterixClusterProperties.INSTANCE.getState().equals(AsterixClusterProperties.State.UNUSABLE)) {
+ throw new AsterixException(" Asterix Cluster is in " + AsterixClusterProperties.State.UNUSABLE + " state."
+ + "\n One or more Node Controllers have left.\n");
+ }
+
boolean invalidOperation = false;
String message = null;
String dataverse = defaultDataverse != null ? defaultDataverse.getDataverseName() : null;
diff --git a/asterix-app/data/tinysocial/twm.adm b/asterix-app/data/tinysocial/twm.adm
new file mode 100644
index 0000000..d18c70f
--- /dev/null
+++ b/asterix-app/data/tinysocial/twm.adm
@@ -0,0 +1,12 @@
+{"tweetid":"1","user":{"screen-name":"NathanGiesen@211","lang":"en","friends_count":39339,"statuses_count":473,"name":"Nathan Giesen","followers_count":49416},"sender-location":point("47.44,80.65"),"send-time":datetime("2008-04-26T10:10:00"),"referred-topics":{{"t-mobile","customization"}},"message-text":" love t-mobile its customization is good:)"}
+{"tweetid":"2","user":{"screen-name":"ColineGeyer@63","lang":"en","friends_count":121,"statuses_count":362,"name":"Coline Geyer","followers_count":17159},"sender-location":point("32.84,67.14"),"send-time":datetime("2010-05-13T10:10:00"),"referred-topics":{{"verizon","shortcut-menu"}},"message-text":" like verizon its shortcut-menu is awesome:)"}
+{"tweetid":"3","user":{"screen-name":"NathanGiesen@211","lang":"en","friends_count":39339,"statuses_count":473,"name":"Nathan Giesen","followers_count":49416},"sender-location":point("29.72,75.8"),"send-time":datetime("2006-11-04T10:10:00"),"referred-topics":{{"motorola","speed"}},"message-text":" like motorola the speed is good:)"}
+{"tweetid":"4","user":{"screen-name":"NathanGiesen@211","lang":"en","friends_count":39339,"statuses_count":473,"name":"Nathan Giesen","followers_count":49416},"sender-location":point("39.28,70.48"),"send-time":datetime("2011-12-26T10:10:00"),"referred-topics":{{"sprint","voice-command"}},"message-text":" like sprint the voice-command is mind-blowing:)"}
+{"tweetid":"5","user":{"screen-name":"NathanGiesen@211","lang":"en","friends_count":39339,"statuses_count":473,"name":"Nathan Giesen","followers_count":49416},"sender-location":point("40.09,92.69"),"send-time":datetime("2006-08-04T10:10:00"),"referred-topics":{{"motorola","speed"}},"message-text":" can't stand motorola its speed is terrible:("}
+{"tweetid":"6","user":{"screen-name":"ColineGeyer@63","lang":"en","friends_count":121,"statuses_count":362,"name":"Coline Geyer","followers_count":17159},"sender-location":point("47.51,83.99"),"send-time":datetime("2010-05-07T10:10:00"),"referred-topics":{{"iphone","voice-clarity"}},"message-text":" like iphone the voice-clarity is good:)"}
+{"tweetid":"7","user":{"screen-name":"ChangEwing_573","lang":"en","friends_count":182,"statuses_count":394,"name":"Chang Ewing","followers_count":32136},"sender-location":point("36.21,72.6"),"send-time":datetime("2011-08-25T10:10:00"),"referred-topics":{{"samsung","platform"}},"message-text":" like samsung the platform is good"}
+{"tweetid":"8","user":{"screen-name":"NathanGiesen@211","lang":"en","friends_count":39339,"statuses_count":473,"name":"Nathan Giesen","followers_count":49416},"sender-location":point("46.05,93.34"),"send-time":datetime("2005-10-14T10:10:00"),"referred-topics":{{"t-mobile","shortcut-menu"}},"message-text":" like t-mobile the shortcut-menu is awesome:)"}
+{"tweetid":"9","user":{"screen-name":"NathanGiesen@211","lang":"en","friends_count":39339,"statuses_count":473,"name":"Nathan Giesen","followers_count":49416},"sender-location":point("36.86,74.62"),"send-time":datetime("2012-07-21T10:10:00"),"referred-topics":{{"verizon","voicemail-service"}},"message-text":" love verizon its voicemail-service is awesome"}
+{"tweetid":"10","user":{"screen-name":"ColineGeyer@63","lang":"en","friends_count":121,"statuses_count":362,"name":"Coline Geyer","followers_count":17159},"sender-location":point("29.15,76.53"),"send-time":datetime("2008-01-26T10:10:00"),"referred-topics":{{"verizon","voice-clarity"}},"message-text":" hate verizon its voice-clarity is OMG:("}
+{"tweetid":"11","user":{"screen-name":"NilaMilliron_tw","lang":"en","friends_count":445,"statuses_count":164,"name":"Nila Milliron","followers_count":22649},"sender-location":point("37.59,68.42"),"send-time":datetime("2008-03-09T10:10:00"),"referred-topics":{{"iphone","platform"}},"message-text":" can't stand iphone its platform is terrible"}
+{"tweetid":"12","user":{"screen-name":"OliJackson_512","lang":"en","friends_count":445,"statuses_count":164,"name":"Oli Jackson","followers_count":22649},"sender-location":point("24.82,94.63"),"send-time":datetime("2010-02-13T10:10:00"),"referred-topics":{{"samsung","voice-command"}},"message-text":" like samsung the voice-command is amazing:)"}
\ No newline at end of file
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
index 65eae64..e64f68f 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixAppRuntimeContext.java
@@ -49,10 +49,11 @@
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactory;
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactoryProvider;
-public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAsterixPropertiesProvider {
+public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAsterixPropertiesProvider {
private static final int DEFAULT_BUFFER_CACHE_PAGE_SIZE = 32768;
private static final int DEFAULT_LIFECYCLEMANAGER_MEMORY_BUDGET = 1024 * 1024 * 1024; // 1GB
private static final int DEFAULT_MAX_OPEN_FILES = Integer.MAX_VALUE;
+ private static final int METADATA_IO_DEVICE_ID = 0;
private final INCApplicationContext ncApplicationContext;
@@ -188,6 +189,10 @@
return ioManager;
}
+ public int getMetaDataIODeviceId() {
+ return METADATA_IO_DEVICE_ID;
+ }
+
@Override
public AsterixStorageProperties getStorageProperties() {
return storageProperties;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 272050f..fb87a8e1 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -1,5 +1,6 @@
package edu.uci.ics.asterix.api.common;
+import java.io.File;
import java.util.EnumSet;
import edu.uci.ics.asterix.common.config.GlobalConfig;
@@ -50,6 +51,8 @@
ncConfig1.datasetIPAddress = "127.0.0.1";
ncConfig1.resultHistorySize = 1000;
ncConfig1.nodeId = NC1_ID;
+ ncConfig1.ioDevices = System.getProperty("java.io.tmpdir") + File.separator + "nc1/iodevice0" + ","
+ + System.getProperty("java.io.tmpdir") + File.separator + "nc1/iodevice1";
ncConfig1.appNCMainClass = NCApplicationEntryPoint.class.getName();
nc1 = new NodeControllerService(ncConfig1);
nc1.start();
@@ -62,6 +65,8 @@
ncConfig2.datasetIPAddress = "127.0.0.1";
ncConfig2.resultHistorySize = 1000;
ncConfig2.nodeId = NC2_ID;
+ ncConfig2.ioDevices = System.getProperty("java.io.tmpdir") + File.separator + "nc2/iodevice0" + ","
+ + System.getProperty("java.io.tmpdir") + File.separator + "nc2/iodevice1";
ncConfig2.appNCMainClass = NCApplicationEntryPoint.class.getName();
nc2 = new NodeControllerService(ncConfig2);
nc2.start();
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index 777ca6f..e22c215 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -171,7 +171,7 @@
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
comparatorFactories, blooFilterKeyFields, true, storageProperties.getMemoryComponentPageSize(),
- storageProperties.getMemoryComponentNumPages());
+ storageProperties.getMemoryComponentNumPages(), fs);
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
index fb83871..ebd1a9c 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
@@ -40,7 +40,8 @@
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields, false,
- storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages(),
+ secondaryFileSplitProvider.getFileSplits());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
index 677eb23..b79102b 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
@@ -76,7 +76,8 @@
import edu.uci.ics.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeDataflowHelperFactory;
@SuppressWarnings("rawtypes")
-// TODO: We should eventually have a hierarchy of classes that can create all possible index job specs,
+// TODO: We should eventually have a hierarchy of classes that can create all
+// possible index job specs,
// not just for creation.
public abstract class SecondaryIndexCreator {
protected final PhysicalOptimizationConfig physOptConf;
@@ -336,16 +337,13 @@
for (int i = 0; i < numSecondaryKeyFields + numPrimaryKeys; i++) {
fieldPermutation[i] = i;
}
- Pair<IFileSplitProvider, AlgebricksPartitionConstraint> secondarySplitsAndConstraint = metadataProvider
- .splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName,
- secondaryIndexName);
TreeIndexBulkLoadOperatorDescriptor treeIndexBulkLoadOp = new TreeIndexBulkLoadOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- secondarySplitsAndConstraint.first, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
+ secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
secondaryBloomFilterKeyFields, fieldPermutation, fillFactor, false, numElementsHint,
dataflowHelperFactory, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, treeIndexBulkLoadOp,
- secondarySplitsAndConstraint.second);
+ secondaryPartitionConstraint);
return treeIndexBulkLoadOp;
}
@@ -365,7 +363,8 @@
}
ICopyEvaluatorFactory selectCond = null;
if (numSecondaryKeyFields > 1) {
- // Create conjunctive condition where all secondary index keys must satisfy 'is not null'.
+ // Create conjunctive condition where all secondary index keys must
+ // satisfy 'is not null'.
AndDescriptor andDesc = new AndDescriptor();
selectCond = andDesc.createEvaluatorFactory(andArgsEvalFactories);
} else {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index a2e28da..ef94ec2 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -157,7 +157,7 @@
ILocalResourceMetadata localResourceMetadata = new LSMInvertedIndexLocalResourceMetadata(invListsTypeTraits,
primaryComparatorFactories, tokenTypeTraits, tokenComparatorFactories, tokenizerFactory,
storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages(),
- isPartitioned);
+ isPartitioned, secondaryFileSplitProvider.getFileSplits());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMInvertedIndexResource);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
index ad61cbe..b7e4886 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
@@ -65,7 +65,7 @@
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties.getMemoryComponentPageSize(),
- storageProperties.getMemoryComponentNumPages());
+ storageProperties.getMemoryComponentNumPages(), secondaryFileSplitProvider.getFileSplits());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 8019a41..5d4f6ec 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -54,6 +54,7 @@
webServer.start();
setupJSONAPIServer(externalProperties);
jsonAPIServer.start();
+ ccAppCtx.addClusterLifecycleListener(ClusterLifecycleListener.INSTANCE);
}
@Override
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ClusterLifecycleListener.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ClusterLifecycleListener.java
new file mode 100644
index 0000000..e8a1ad2
--- /dev/null
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/ClusterLifecycleListener.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.hyracks.bootstrap;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
+import edu.uci.ics.hyracks.api.application.IClusterLifecycleListener;
+
+public class ClusterLifecycleListener implements IClusterLifecycleListener {
+
+ public static ClusterLifecycleListener INSTANCE = new ClusterLifecycleListener();
+
+ private ClusterLifecycleListener() {
+ }
+
+ private static final Logger LOGGER = Logger.getLogger(ClusterLifecycleListener.class.getName());
+
+ @Override
+ public void notifyNodeJoin(String nodeId, Map<String, String> ncConfiguration) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("NC: " + nodeId + " joined");
+ }
+ AsterixClusterProperties.INSTANCE.addNCConfiguration(nodeId, ncConfiguration);
+ }
+
+ public void notifyNodeFailure(Set<String> deadNodeIds) {
+ for (String deadNode : deadNodeIds) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("NC: " + deadNode + " left");
+ }
+ AsterixClusterProperties.INSTANCE.removeNCConfiguration(deadNode);
+ }
+
+ }
+
+}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java b/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
index 32f17b2..02cf6a4 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/result/ResultUtils.java
@@ -91,11 +91,10 @@
public static String extractErrorMessage(Throwable e) {
StringBuilder errorMessageBuilder = new StringBuilder();
Throwable cause = e;
+ errorMessageBuilder.append(cause.getLocalizedMessage());
while (cause != null) {
- StackTraceElement[] stackTraceElements = e.getStackTrace();
- errorMessageBuilder.append(cause.toString());
- errorMessageBuilder.append(stackTraceElements.length > 0 ? "\n at " + stackTraceElements[0] : "");
- errorMessageBuilder.append("\n");
+ StackTraceElement[] stackTraceElements = cause.getStackTrace();
+ errorMessageBuilder.append(stackTraceElements.length > 0 ? "\n caused by: " + stackTraceElements[0] : "");
cause = cause.getCause();
}
return errorMessageBuilder.toString();
diff --git a/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.1.ddl.aql
new file mode 100644
index 0000000..17da24b
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.1.ddl.aql
@@ -0,0 +1,13 @@
+drop dataverse TinySocial if exists;
+create dataverse TinySocial;
+use dataverse TinySocial;
+
+create type TweetMessageType as open {
+ tweetid: string
+}
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid
+hints(cardinality=100);
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.2.update.aql
new file mode 100644
index 0000000..627623a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.2.update.aql
@@ -0,0 +1,5 @@
+use dataverse TinySocial;
+
+load dataset TweetMessages
+using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/tinysocial/twm.adm"),("format"="adm"));
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.3.query.aql
new file mode 100644
index 0000000..f40e884
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/fuzzyjoin/opentype/opentype.3.query.aql
@@ -0,0 +1,15 @@
+use dataverse TinySocial;
+
+set simfunction "jaccard";
+set simthreshold "0.3";
+
+for $t in dataset TweetMessages
+order by $t.tweetid
+return {
+ "tweet": $t,
+ "similar-tweets": for $t2 in dataset TweetMessages
+ order by $t2.tweetid
+ where $t2.referred-topics ~= $t.referred-topics
+ and $t2.tweetid != $t.tweetid
+ return $t2.referred-topics
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.1.ddl.aql
new file mode 100644
index 0000000..139597f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.1.ddl.aql
@@ -0,0 +1,2 @@
+
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.3.query.aql
new file mode 100644
index 0000000..825b269
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/numeric/caret0/caret0.3.query.aql
@@ -0,0 +1,5 @@
+let $n1 := 2.0
+let $n2 := 4096.0
+let $n3 := 3
+let $n4 := 2
+return { "c1": $n1^$n2, "c2": $n2^$n1, "c3": $n3^$n4 }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.1.ddl.aql
new file mode 100644
index 0000000..4d74b83
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.1.ddl.aql
@@ -0,0 +1,18 @@
+/*
+ * Description : This test case is to verify the fix for issue487
+ : https://code.google.com/p/asterixdb/issues/detail?id=487
+ * Expected Res : FAIL
+ * Date : 30th May 2013
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type EmpType as open {
+id : int32,
+name : string
+}
+
+create dataset Employee(EmpType) primary key id;
+
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.2.update.aql
new file mode 100644
index 0000000..6913b4f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.2.update.aql
@@ -0,0 +1,10 @@
+/*
+ * Description : This test case is to verify the fix for issue487
+ : https://code.google.com/p/asterixdb/issues/detail?id=487
+ * Expected Res : FAIL
+ * Date : 30th May 2013
+ */
+
+use dataverse test;
+
+insert into dataset Employee ({ "id":123});
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.3.query.aql
new file mode 100644
index 0000000..d53aba9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue487/query-issue487.3.query.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : This test case is to verify the fix for issue487
+ : https://code.google.com/p/asterixdb/issues/detail?id=487
+ * Expected Res : FAIL
+ * Date : 30th May 2013
+ */
+
+use dataverse test;
+
+for $l in dataset Employee
+return $l;
+
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.1.ddl.aql
new file mode 100644
index 0000000..754ea81
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.1.ddl.aql
@@ -0,0 +1,3 @@
+drop dataverse test if exists;
+create dataverse test;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.3.query.aql
new file mode 100644
index 0000000..88f0dfa
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/string/string-to-codepoint2/string-to-codepoint2.3.query.aql
@@ -0,0 +1,6 @@
+use dataverse test;
+
+let $x := "欢迎"
+let $c := string-to-codepoint($x)
+
+return {"result1": $c}
diff --git a/asterix-app/src/test/resources/runtimets/queries/tpch/q15_top_supplier/q15_top_supplier.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/tpch/q15_top_supplier/q15_top_supplier.3.query.aql
index f61ceb5..9f1080d 100644
--- a/asterix-app/src/test/resources/runtimets/queries/tpch/q15_top_supplier/q15_top_supplier.3.query.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/tpch/q15_top_supplier/q15_top_supplier.3.query.aql
@@ -17,7 +17,7 @@
for $s in dataset('Supplier')
for $r in revenue()
- where $s.s_suppkey = $r.supplier_no and $r.total_revenue=$m
+ where $s.s_suppkey = $r.supplier_no and $r.total_revenue<$m+0.000000001 and $r.total_revenue>$m-0.000000001
return {
"s_suppkey": $s.s_suppkey,
"s_name": $s.s_name,
diff --git a/asterix-app/src/test/resources/runtimets/results/fuzzyjoin/opentype/opentype.1.adm b/asterix-app/src/test/resources/runtimets/results/fuzzyjoin/opentype/opentype.1.adm
new file mode 100644
index 0000000..ae0c4cb
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/fuzzyjoin/opentype/opentype.1.adm
@@ -0,0 +1,12 @@
+{ "tweet": { "tweetid": "1", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("47.44,80.65"), "send-time": datetime("2008-04-26T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "customization" }}, "message-text": " love t-mobile its customization is good:)" }, "similar-tweets": [ {{ "t-mobile", "shortcut-menu" }} ] }
+{ "tweet": { "tweetid": "10", "user": { "screen-name": "ColineGeyer@63", "lang": "en", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("29.15,76.53"), "send-time": datetime("2008-01-26T10:10:00.000Z"), "referred-topics": {{ "verizon", "voice-clarity" }}, "message-text": " hate verizon its voice-clarity is OMG:(" }, "similar-tweets": [ {{ "verizon", "shortcut-menu" }}, {{ "iphone", "voice-clarity" }}, {{ "verizon", "voicemail-service" }} ] }
+{ "tweet": { "tweetid": "11", "user": { "screen-name": "NilaMilliron_tw", "lang": "en", "friends_count": 445, "statuses_count": 164, "name": "Nila Milliron", "followers_count": 22649 }, "sender-location": point("37.59,68.42"), "send-time": datetime("2008-03-09T10:10:00.000Z"), "referred-topics": {{ "iphone", "platform" }}, "message-text": " can't stand iphone its platform is terrible" }, "similar-tweets": [ {{ "iphone", "voice-clarity" }}, {{ "samsung", "platform" }} ] }
+{ "tweet": { "tweetid": "12", "user": { "screen-name": "OliJackson_512", "lang": "en", "friends_count": 445, "statuses_count": 164, "name": "Oli Jackson", "followers_count": 22649 }, "sender-location": point("24.82,94.63"), "send-time": datetime("2010-02-13T10:10:00.000Z"), "referred-topics": {{ "samsung", "voice-command" }}, "message-text": " like samsung the voice-command is amazing:)" }, "similar-tweets": [ {{ "sprint", "voice-command" }}, {{ "samsung", "platform" }} ] }
+{ "tweet": { "tweetid": "2", "user": { "screen-name": "ColineGeyer@63", "lang": "en", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("32.84,67.14"), "send-time": datetime("2010-05-13T10:10:00.000Z"), "referred-topics": {{ "verizon", "shortcut-menu" }}, "message-text": " like verizon its shortcut-menu is awesome:)" }, "similar-tweets": [ {{ "verizon", "voice-clarity" }}, {{ "t-mobile", "shortcut-menu" }}, {{ "verizon", "voicemail-service" }} ] }
+{ "tweet": { "tweetid": "3", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("29.72,75.8"), "send-time": datetime("2006-11-04T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " like motorola the speed is good:)" }, "similar-tweets": [ {{ "motorola", "speed" }} ] }
+{ "tweet": { "tweetid": "4", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("39.28,70.48"), "send-time": datetime("2011-12-26T10:10:00.000Z"), "referred-topics": {{ "sprint", "voice-command" }}, "message-text": " like sprint the voice-command is mind-blowing:)" }, "similar-tweets": [ {{ "samsung", "voice-command" }} ] }
+{ "tweet": { "tweetid": "5", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("40.09,92.69"), "send-time": datetime("2006-08-04T10:10:00.000Z"), "referred-topics": {{ "motorola", "speed" }}, "message-text": " can't stand motorola its speed is terrible:(" }, "similar-tweets": [ {{ "motorola", "speed" }} ] }
+{ "tweet": { "tweetid": "6", "user": { "screen-name": "ColineGeyer@63", "lang": "en", "friends_count": 121, "statuses_count": 362, "name": "Coline Geyer", "followers_count": 17159 }, "sender-location": point("47.51,83.99"), "send-time": datetime("2010-05-07T10:10:00.000Z"), "referred-topics": {{ "iphone", "voice-clarity" }}, "message-text": " like iphone the voice-clarity is good:)" }, "similar-tweets": [ {{ "verizon", "voice-clarity" }}, {{ "iphone", "platform" }} ] }
+{ "tweet": { "tweetid": "7", "user": { "screen-name": "ChangEwing_573", "lang": "en", "friends_count": 182, "statuses_count": 394, "name": "Chang Ewing", "followers_count": 32136 }, "sender-location": point("36.21,72.6"), "send-time": datetime("2011-08-25T10:10:00.000Z"), "referred-topics": {{ "samsung", "platform" }}, "message-text": " like samsung the platform is good" }, "similar-tweets": [ {{ "iphone", "platform" }}, {{ "samsung", "voice-command" }} ] }
+{ "tweet": { "tweetid": "8", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("46.05,93.34"), "send-time": datetime("2005-10-14T10:10:00.000Z"), "referred-topics": {{ "t-mobile", "shortcut-menu" }}, "message-text": " like t-mobile the shortcut-menu is awesome:)" }, "similar-tweets": [ {{ "t-mobile", "customization" }}, {{ "verizon", "shortcut-menu" }} ] }
+{ "tweet": { "tweetid": "9", "user": { "screen-name": "NathanGiesen@211", "lang": "en", "friends_count": 39339, "statuses_count": 473, "name": "Nathan Giesen", "followers_count": 49416 }, "sender-location": point("36.86,74.62"), "send-time": datetime("2012-07-21T10:10:00.000Z"), "referred-topics": {{ "verizon", "voicemail-service" }}, "message-text": " love verizon its voicemail-service is awesome" }, "similar-tweets": [ {{ "verizon", "voice-clarity" }}, {{ "verizon", "shortcut-menu" }} ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/numeric/caret0/caret0.1.adm b/asterix-app/src/test/resources/runtimets/results/numeric/caret0/caret0.1.adm
new file mode 100644
index 0000000..1edec8a
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/numeric/caret0/caret0.1.adm
@@ -0,0 +1 @@
+{ "c1": Infinityd, "c2": 1.6777216E7d, "c3": 9 }
diff --git a/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue487/query-issue487.1.adm b/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue487/query-issue487.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue487/query-issue487.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/string/string-to-codepoint2/string-to-codepoint2.1.adm b/asterix-app/src/test/resources/runtimets/results/string/string-to-codepoint2/string-to-codepoint2.1.adm
new file mode 100644
index 0000000..eacdb11
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/string/string-to-codepoint2/string-to-codepoint2.1.adm
@@ -0,0 +1 @@
+{ "result1": [ 27426, 36814 ] }
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index ed7b9db..4419671 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -1755,6 +1755,11 @@
</compilation-unit>
</test-case>
-->
+ <test-case FilePath="fuzzyjoin">
+ <compilation-unit name="opentype">
+ <output-dir compare="Text">opentype</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="index-join">
<test-case FilePath="index-join">
@@ -2160,6 +2165,11 @@
</test-group>
<test-group name="numeric">
<test-case FilePath="numeric">
+ <compilation-unit name="caret0">
+ <output-dir compare="Text">caret0</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="numeric">
<compilation-unit name="abs0">
<output-dir compare="Text">abs0</output-dir>
</compilation-unit>
@@ -2732,6 +2742,12 @@
<expected-error>edu.uci.ics.asterix.common.exceptions.AsterixException</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue487">
+ <output-dir compare="Text">query-issue487</output-dir>
+ <expected-error>edu.uci.ics.asterix.common.exceptions.AsterixException</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="quantifiers">
<test-case FilePath="quantifiers">
@@ -3478,6 +3494,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="string">
+ <compilation-unit name="string-to-codepoint2">
+ <output-dir compare="Text">string-to-codepoint2</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="string">
<compilation-unit name="strlen02">
<output-dir compare="Text">strlen02</output-dir>
</compilation-unit>
diff --git a/asterix-aql/pom.xml b/asterix-aql/pom.xml
index 249299c..f8c652f 100644
--- a/asterix-aql/pom.xml
+++ b/asterix-aql/pom.xml
@@ -30,6 +30,7 @@
</goals>
<configuration>
<isStatic>false</isStatic>
+ <javaUnicodeEscape>true</javaUnicodeEscape>
</configuration>
</execution>
<execution>
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java
index deb51d0..e053989 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -23,6 +23,8 @@
public ILSMIOOperationScheduler getLSMIOScheduler();
+ public int getMetaDataIODeviceId();
+
public ILSMMergePolicy getLSMMergePolicy();
public IBufferCache getBufferCache();
@@ -42,8 +44,7 @@
public void setShuttingdown(boolean b);
public void deinitialize() throws HyracksDataException;
-
+
public double getBloomFilterFalsePositiveRate();
-
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
index d34e4ac..91bdca6 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
@@ -15,7 +15,7 @@
private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT = (32 << 10); // 32KB
private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.memorycomponent.numpages";
- private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 4096; // ... so 128MB components
+ private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 2048; // ... so 64MB components
private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY = "storage.memorycomponent.globalbudget";
private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = (1 << 30); // 1GB
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/dataflow/IAsterixApplicationContextInfo.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/dataflow/IAsterixApplicationContextInfo.java
index ee05c7a..a057f91 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/dataflow/IAsterixApplicationContextInfo.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/dataflow/IAsterixApplicationContextInfo.java
@@ -19,17 +19,18 @@
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
/**
- * Provides methods for obtaining the IIndexRegistryProvider, IStorageManager and
+ * Provides methods for obtaining the IIndexLifecycleManagerProvider, IStorageManagerInterface and
* ICCApplicationContext implementation.
*/
-public interface IAsterixApplicationContextInfo{
+public interface IAsterixApplicationContextInfo {
/**
+ * Returns an instance of the implementation for IIndexLifecycleManagerProvider.
*
+ * @return IIndexLifecycleManagerProvider implementation instance
*/
public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider();
-
/**
* Returns an instance of the implementation for IStorageManagerInterface.
*
@@ -43,5 +44,4 @@
* @return ICCApplicationContext implementation instance
*/
public ICCApplicationContext getCCApplicationContext();
-
}
diff --git a/asterix-events/pom.xml b/asterix-events/pom.xml
index 94042e1..108bde2 100644
--- a/asterix-events/pom.xml
+++ b/asterix-events/pom.xml
@@ -173,5 +173,10 @@
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>1.2.9</version>
+ </dependency>
</dependencies>
</project>
diff --git a/asterix-external-data/pom.xml b/asterix-external-data/pom.xml
index d534716..842cd67 100644
--- a/asterix-external-data/pom.xml
+++ b/asterix-external-data/pom.xml
@@ -117,6 +117,12 @@
<version>1.0.0</version>
<type>jar</type>
<scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <artifactId>rome</artifactId>
+ <groupId>net.java.dev.rome</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>rome</groupId>
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index 791666d..422fdfd 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -73,6 +73,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManager;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.util.IndexFileNameUtil;
import edu.uci.ics.hyracks.storage.am.lsm.btree.impls.LSMBTree;
import edu.uci.ics.hyracks.storage.am.lsm.btree.util.LSMBTreeUtils;
import edu.uci.ics.hyracks.storage.am.lsm.common.api.IInMemoryBufferCache;
@@ -333,7 +334,9 @@
}
private static void enlistMetadataDataset(IMetadataIndex index, boolean create) throws Exception {
- String filePath = metadataStore + File.separator + index.getFileNameRelativePath();
+ String filePath = IndexFileNameUtil.prepareFileName(
+ metadataStore + File.separator + index.getFileNameRelativePath(),
+ runtimeContext.getMetaDataIODeviceId());
FileReference file = new FileReference(new File(filePath));
IInMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), DEFAULT_MEM_PAGE_SIZE,
DEFAULT_MEM_NUM_PAGES, new TransientFileMapManager());
@@ -350,7 +353,7 @@
fileMapProvider, typeTraits, comparatorFactories, bloomFilterKeyFields,
runtimeContext.getBloomFilterFalsePositiveRate(), runtimeContext.getLSMMergePolicy(),
runtimeContext.getLSMBTreeOperationTrackerFactory(), runtimeContext.getLSMIOScheduler(),
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, runtimeContext.getMetaDataIODeviceId());
lsmBtree.create();
resourceID = runtimeContext.getResourceIdFactory().createId();
indexLifecycleManager.register(resourceID, lsmBtree);
@@ -358,12 +361,14 @@
AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(),
- storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages(),
+ runtimeContext.getMetaDataIODeviceId());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory();
- localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, file.getFile()
- .getPath(), 0));
+ localResourceRepository.insert(
+ localResourceFactory.createLocalResource(resourceID, file.getFile().getPath(), 0),
+ runtimeContext.getMetaDataIODeviceId());
} else {
resourceID = localResourceRepository.getResourceByName(file.getFile().getPath()).getResourceId();
lsmBtree = (LSMBTree) indexLifecycleManager.getIndex(resourceID);
@@ -372,7 +377,7 @@
bufferCache, fileMapProvider, typeTraits, comparatorFactories, bloomFilterKeyFields,
runtimeContext.getBloomFilterFalsePositiveRate(), runtimeContext.getLSMMergePolicy(),
runtimeContext.getLSMBTreeOperationTrackerFactory(), runtimeContext.getLSMIOScheduler(),
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, runtimeContext.getMetaDataIODeviceId());
indexLifecycleManager.register(resourceID, lsmBtree);
}
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 3128153..117f492 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -66,6 +66,7 @@
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
import edu.uci.ics.asterix.runtime.base.AsterixTupleFilterFactory;
import edu.uci.ics.asterix.runtime.formats.FormatUtils;
@@ -418,7 +419,8 @@
adapterFactoryClassname = adapterFactoryMapping.get(adapterName);
if (adapterFactoryClassname != null) {
} else {
- // adapterName has been provided as a fully qualified classname
+ // adapterName has been provided as a fully qualified
+ // classname
adapterFactoryClassname = adapterName;
}
adapterFactory = (IAdapterFactory) Class.forName(adapterFactoryClassname).newInstance();
@@ -769,9 +771,11 @@
numElementsHint = Long.parseLong(dataset.getHints().get("CARDINALITY"));
}
- //TODO
- //figure out the right behavior of the bulkload and then give the right callback
- //(ex. what's the expected behavior when there is an error during bulkload?)
+ // TODO
+ // figure out the right behavior of the bulkload and then give the
+ // right callback
+ // (ex. what's the expected behavior when there is an error during
+ // bulkload?)
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
appContext.getStorageManagerInterface(), appContext.getIndexLifecycleManagerProvider(),
splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields, fieldPermutation,
@@ -830,7 +834,7 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForInternalOrFeedDataset(
dataSource.getId().getDataverseName(), datasetName, indexName);
- //prepare callback
+ // prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
int[] primaryKeyFields = new int[numKeys];
@@ -1023,7 +1027,7 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForInternalOrFeedDataset(
dataverseName, datasetName, indexName);
- //prepare callback
+ // prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
int[] primaryKeyFields = new int[primaryKeys.size()];
@@ -1138,7 +1142,8 @@
tokenComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
tokenTypeTraits[0] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
if (isPartitioned) {
- // The partitioning field is hardcoded to be a short *without* an Asterix type tag.
+ // The partitioning field is hardcoded to be a short *without*
+ // an Asterix type tag.
tokenComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
tokenTypeTraits[1] = ShortPointable.TYPE_TRAITS;
}
@@ -1149,7 +1154,7 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForInternalOrFeedDataset(
dataverseName, datasetName, indexName);
- //prepare callback
+ // prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
int[] primaryKeyFields = new int[primaryKeys.size()];
@@ -1244,7 +1249,7 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForInternalOrFeedDataset(
dataverseName, datasetName, indexName);
- //prepare callback
+ // prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
int datasetId = dataset.getDatasetId();
int[] primaryKeyFields = new int[numPrimaryKeys];
@@ -1338,8 +1343,13 @@
continue;
}
for (int i = 0; i < nodeStores.length; i++) {
- File f = new File(nodeStores[i] + File.separator + relPathFile);
- splits.add(new FileSplit(node, new FileReference(f)));
+ int numIODevices = AsterixClusterProperties.INSTANCE.getNumberOfIODevices(node);
+ for (int j = 0; j < nodeStores.length; j++) {
+ for (int k = 0; k < numIODevices; k++) {
+ File f = new File(nodeStores[j] + File.separator + relPathFile);
+ splits.add(new FileSplit(node, new FileReference(f), k));
+ }
+ }
}
}
return splits.toArray(new FileSplit[] {});
@@ -1368,9 +1378,17 @@
LOGGER.warning("Node " + nd + " has no stores.");
throw new AlgebricksException("Node " + nd + " has no stores.");
} else {
+ int numIODevices;
+ if (datasetDetails.getNodeGroupName().compareTo(MetadataConstants.METADATA_NODEGROUP_NAME) == 0) {
+ numIODevices = 1;
+ } else {
+ numIODevices = AsterixClusterProperties.INSTANCE.getNumberOfIODevices(nd);
+ }
for (int j = 0; j < nodeStores.length; j++) {
- File f = new File(nodeStores[j] + File.separator + relPathFile);
- splitArray.add(new FileSplit(nd, new FileReference(f)));
+ for (int k = 0; k < numIODevices; k++) {
+ File f = new File(nodeStores[j] + File.separator + relPathFile);
+ splitArray.add(new FileSplit(nd, new FileReference(f), k));
+ }
}
}
}
@@ -1417,21 +1435,6 @@
return dataverseName + File.separator + fileName;
}
- public Pair<IFileSplitProvider, IFileSplitProvider> getInvertedIndexFileSplitProviders(
- IFileSplitProvider splitProvider) {
- int numSplits = splitProvider.getFileSplits().length;
- FileSplit[] btreeSplits = new FileSplit[numSplits];
- FileSplit[] invListsSplits = new FileSplit[numSplits];
- for (int i = 0; i < numSplits; i++) {
- String nodeName = splitProvider.getFileSplits()[i].getNodeName();
- String path = splitProvider.getFileSplits()[i].getLocalFile().getFile().getPath();
- btreeSplits[i] = new FileSplit(nodeName, path + "_$btree");
- invListsSplits[i] = new FileSplit(nodeName, path + "_$invlists");
- }
- return new Pair<IFileSplitProvider, IFileSplitProvider>(new ConstantFileSplitProvider(btreeSplits),
- new ConstantFileSplitProvider(invListsSplits));
- }
-
public Dataset findDataset(String dataverse, String dataset) throws AlgebricksException {
try {
return MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverse, dataset);
@@ -1463,10 +1466,13 @@
public AlgebricksPartitionConstraint getClusterLocations() {
ArrayList<String> locs = new ArrayList<String>();
- for (String k : stores.keySet()) {
- String[] nodeStores = stores.get(k);
+ for (String i : stores.keySet()) {
+ String[] nodeStores = stores.get(i);
+ int numIODevices = AsterixClusterProperties.INSTANCE.getNumberOfIODevices(i);
for (int j = 0; j < nodeStores.length; j++) {
- locs.add(k);
+ for (int k = 0; k < numIODevices; k++) {
+ locs.add(i);
+ }
}
}
String[] cluster = new String[locs.size()];
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/ListItemBinaryComparatorFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/ListItemBinaryComparatorFactory.java
new file mode 100644
index 0000000..6e80fe6
--- /dev/null
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/ListItemBinaryComparatorFactory.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.dataflow.data.nontagged.comparators;
+
+import edu.uci.ics.asterix.formats.nontagged.UTF8StringLowercasePointable;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.EnumDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.FloatPointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+
+public class ListItemBinaryComparatorFactory implements IBinaryComparatorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final ListItemBinaryComparatorFactory INSTANCE = new ListItemBinaryComparatorFactory();
+
+ private ListItemBinaryComparatorFactory() {
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ return createBinaryComparator(ATypeTag.NULL, ATypeTag.NULL, false);
+ }
+
+ public IBinaryComparator createBinaryComparator(final ATypeTag firstItemTypeTag, final ATypeTag secondItemTypeTag,
+ final boolean ignoreCase) {
+ return new IBinaryComparator() {
+ final IBinaryComparator ascBoolComp = BooleanBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ final IBinaryComparator ascIntComp = new PointableBinaryComparatorFactory(IntegerPointable.FACTORY)
+ .createBinaryComparator();
+ final IBinaryComparator ascLongComp = LongBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ final IBinaryComparator ascStrComp = new PointableBinaryComparatorFactory(UTF8StringPointable.FACTORY)
+ .createBinaryComparator();
+ final IBinaryComparator ascLowerCaseStrComp = new PointableBinaryComparatorFactory(
+ UTF8StringLowercasePointable.FACTORY).createBinaryComparator();
+ final IBinaryComparator ascFloatComp = new PointableBinaryComparatorFactory(FloatPointable.FACTORY)
+ .createBinaryComparator();
+ final IBinaryComparator ascDoubleComp = new PointableBinaryComparatorFactory(DoublePointable.FACTORY)
+ .createBinaryComparator();
+ final IBinaryComparator ascRectangleComp = ARectanglePartialBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ final IBinaryComparator ascCircleComp = ACirclePartialBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ final IBinaryComparator ascDurationComp = ADurationPartialBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ final IBinaryComparator ascIntervalComp = AIntervalPartialBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ final IBinaryComparator ascLineComp = ALinePartialBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ final IBinaryComparator ascPointComp = APointPartialBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ final IBinaryComparator ascPoint3DComp = APoint3DPartialBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ final IBinaryComparator ascPolygonComp = APolygonPartialBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+ final IBinaryComparator rawComp = RawBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+
+ if (b1[s1] == ATypeTag.NULL.serialize()) {
+ if (b2[s2] == ATypeTag.NULL.serialize())
+ return 0;
+ else
+ return -1;
+ } else {
+ if (b2[s2] == ATypeTag.NULL.serialize())
+ return 1;
+ }
+
+ ATypeTag tag1 = firstItemTypeTag;
+ int skip1 = 0;
+ if (firstItemTypeTag == ATypeTag.ANY) {
+ tag1 = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(b1[s1]);
+ skip1 = 1;
+ }
+
+ ATypeTag tag2 = secondItemTypeTag;
+ int skip2 = 0;
+ if (secondItemTypeTag == ATypeTag.ANY) {
+ tag2 = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(b2[s2]);
+ skip2 = 1;
+ }
+
+ if (tag1 != tag2) {
+ return rawComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+
+ switch (tag1) {
+ case BOOLEAN: {
+ return ascBoolComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case TIME:
+ case DATE:
+ case YEARMONTHDURATION:
+ case INT32: {
+ return ascIntComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case DATETIME:
+ case DAYTIMEDURATION:
+ case INT64: {
+ return ascLongComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case FLOAT: {
+ return ascFloatComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case DOUBLE: {
+ return ascDoubleComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case STRING: {
+ if (ignoreCase) {
+ return ascLowerCaseStrComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ } else {
+ return ascStrComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ }
+ case RECTANGLE: {
+ return ascRectangleComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case CIRCLE: {
+ return ascCircleComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case POINT: {
+ return ascPointComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case POINT3D: {
+ return ascPoint3DComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case LINE: {
+ return ascLineComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case POLYGON: {
+ return ascPolygonComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case DURATION: {
+ return ascDurationComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ case INTERVAL: {
+ return ascIntervalComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ default: {
+ return rawComp.compare(b1, s1 + skip1, l1 - skip1, b2, s2 + skip2, l2 - skip2);
+ }
+ }
+ }
+ };
+ }
+}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/ListItemBinaryHashFunctionFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/ListItemBinaryHashFunctionFactory.java
new file mode 100644
index 0000000..0fab7de
--- /dev/null
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/ListItemBinaryHashFunctionFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.dataflow.data.nontagged.hash;
+
+import java.io.IOException;
+
+import edu.uci.ics.asterix.formats.nontagged.UTF8StringLowercasePointable;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.EnumDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.MurmurHash3BinaryHashFunctionFamily;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.util.GrowableArray;
+
+/**
+ * This hash function factory is introduced to be able to hash heterogeneous list items.
+ * The item type tag is also included in the hash computation to distinguish the different
+ * types with the same raw bytes.
+ */
+public class ListItemBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final ListItemBinaryHashFunctionFactory INSTANCE = new ListItemBinaryHashFunctionFactory();
+
+ private ListItemBinaryHashFunctionFactory() {
+ }
+
+ @Override
+ public IBinaryHashFunction createBinaryHashFunction() {
+ return createBinaryHashFunction(ATypeTag.ANY, false);
+ }
+
+ public IBinaryHashFunction createBinaryHashFunction(final ATypeTag itemTypeTag, final boolean ignoreCase) {
+ return new IBinaryHashFunction() {
+
+ private IBinaryHashFunction lowerCaseStringHash = new PointableBinaryHashFunctionFactory(
+ UTF8StringLowercasePointable.FACTORY).createBinaryHashFunction();
+ private IBinaryHashFunction genericBinaryHash = MurmurHash3BinaryHashFunctionFamily.INSTANCE
+ .createBinaryHashFunction(0);
+ private GrowableArray taggedBytes = new GrowableArray();
+
+ @Override
+ public int hash(byte[] bytes, int offset, int length) {
+ ATypeTag tag = itemTypeTag;
+ int skip = 0;
+ if (itemTypeTag == ATypeTag.ANY) {
+ tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[offset]);
+ skip = 1;
+ }
+ switch (tag) {
+ case STRING: {
+ if (ignoreCase) {
+ return lowerCaseStringHash.hash(bytes, offset + skip, length - skip);
+ }
+ }
+ default: {
+ if (itemTypeTag != ATypeTag.ANY) {
+ // add the itemTypeTag in front of the data
+ try {
+ resetTaggedBytes(bytes, offset, length);
+ return genericBinaryHash.hash(taggedBytes.getByteArray(), 0, length + 1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ return genericBinaryHash.hash(bytes, offset, length);
+ }
+ }
+ }
+ }
+
+ private void resetTaggedBytes(byte[] data, int offset, int length) throws IOException {
+ taggedBytes.reset();
+ taggedBytes.getDataOutput().writeByte(itemTypeTag.serialize());
+ taggedBytes.getDataOutput().write(data, offset, length);
+ }
+ };
+ }
+}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
index e85a72d..81100bf 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixAppContextInfo.java
@@ -57,7 +57,6 @@
INSTANCE.metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
INSTANCE.storageProperties = new AsterixStorageProperties(propertiesAccessor);
INSTANCE.txnProperties = new AsterixTransactionProperties(propertiesAccessor);
-
Logger.getLogger("edu.uci.ics").setLevel(INSTANCE.externalProperties.getLogLevel());
}
@@ -108,6 +107,4 @@
public IStorageManagerInterface getStorageManagerInterface() {
return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
}
-
-
}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixClusterProperties.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixClusterProperties.java
new file mode 100644
index 0000000..bc2a6af
--- /dev/null
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixClusterProperties.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.om.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * A holder class for properties related to the Asterix cluster.
+ */
+
+public class AsterixClusterProperties {
+
+ private static final Logger LOGGER = Logger.getLogger(AsterixClusterProperties.class.getName());
+
+ private static final String IO_DEVICES = "iodevices";
+
+ public static final AsterixClusterProperties INSTANCE = new AsterixClusterProperties();
+
+ private Map<String, Map<String, String>> ncConfiguration = new HashMap<String, Map<String, String>>();
+
+ private AsterixClusterProperties() {
+ }
+
+ public enum State {
+ ACTIVE,
+ UNUSABLE
+ }
+
+ private State state = State.UNUSABLE;
+
+ public void removeNCConfiguration(String nodeId) {
+ state = State.UNUSABLE;
+ ncConfiguration.remove(nodeId);
+ }
+
+ public void addNCConfiguration(String nodeId, Map<String, String> configuration) {
+ ncConfiguration.put(nodeId, configuration);
+ if (ncConfiguration.keySet().size() == AsterixAppContextInfo.getInstance().getMetadataProperties()
+ .getNodeNames().size()) {
+ state = State.ACTIVE;
+ }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(" Registering configuration parameters for node id" + nodeId);
+ }
+ }
+
+ /**
+ * Returns the number of IO devices configured for a Node Controller
+ *
+ * @param nodeId
+ * unique identifier of the Node Controller
+ * @return number of IO devices. -1 if the node id is not valid. A node id is not valid
+ * if it does not correspond to the set of registered Node Controllers.
+ */
+ public int getNumberOfIODevices(String nodeId) {
+ Map<String, String> ncConfig = ncConfiguration.get(nodeId);
+ if (ncConfig == null) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Configuration parameters for nodeId" + nodeId
+ + " not found. The node has not joined yet or has left.");
+ }
+ return -1;
+ }
+ return ncConfig.get(IO_DEVICES).split(",").length;
+ }
+
+ public State getState() {
+ return state;
+ }
+
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java
index efae5f9..269d363 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java
@@ -13,7 +13,10 @@
protected byte[] data;
protected int count = 0;
protected int pos = -1;
- protected int size = -1;
+ protected int nextPos = -1;
+ protected int itemLen = -1;
+ protected int numberOfItems = -1;
+ protected int listLength = -1;
protected int startOff = -1;
protected IBinaryComparator cmp;
@@ -27,12 +30,12 @@
@Override
public boolean hasNext() {
- return count < size;
+ return count < numberOfItems;
}
@Override
public int size() {
- return size;
+ return numberOfItems;
}
@Override
@@ -45,10 +48,20 @@
return pos;
}
+ public int getItemLen() {
+ return itemLen;
+ }
+
@Override
public void next() {
try {
- pos = getItemOffset(data, startOff, ++count);
+ pos = nextPos;
+ ++count;
+ nextPos = startOff + listLength;
+ if (count + 1 < numberOfItems) {
+ nextPos = getItemOffset(data, startOff, count + 1);
+ }
+ itemLen = nextPos - pos;
} catch (AsterixException e) {
throw new AsterixRuntimeException(e);
}
@@ -59,6 +72,11 @@
count = 0;
try {
pos = getItemOffset(data, startOff, count);
+ nextPos = startOff + listLength;
+ if (count + 1 < numberOfItems) {
+ nextPos = getItemOffset(data, startOff, count + 1);
+ }
+ itemLen = nextPos - pos;
} catch (AsterixException e) {
throw new AsterixRuntimeException(e);
}
@@ -67,7 +85,8 @@
public void reset(byte[] data, int startOff) {
this.data = data;
this.startOff = startOff;
- size = getNumberOfItems(data, startOff);
+ this.numberOfItems = getNumberOfItems(data, startOff);
+ this.listLength = getListLength(data, startOff);
ATypeTag tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(data[startOff + 1]);
switch (tag) {
case INT32: {
@@ -102,4 +121,6 @@
protected abstract int getItemOffset(byte[] serOrderedList, int offset, int itemIndex) throws AsterixException;
protected abstract int getNumberOfItems(byte[] serOrderedList, int offset);
+
+ protected abstract int getListLength(byte[] serOrderedList, int offset);
}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixOrderedListIterator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixOrderedListIterator.java
index d3714c1..fc92875 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixOrderedListIterator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixOrderedListIterator.java
@@ -14,4 +14,9 @@
protected int getNumberOfItems(byte[] serOrderedList, int offset) {
return AOrderedListSerializerDeserializer.getNumberOfItems(serOrderedList, offset);
}
+
+ @Override
+ protected int getListLength(byte[] serOrderedList, int offset) {
+ return AOrderedListSerializerDeserializer.getOrderedListLength(serOrderedList, offset + 1);
+ }
}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixUnorderedListIterator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixUnorderedListIterator.java
index de7742b..5f01581 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixUnorderedListIterator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AsterixUnorderedListIterator.java
@@ -14,4 +14,9 @@
protected int getNumberOfItems(byte[] serOrderedList, int offset) {
return AUnorderedListSerializerDeserializer.getNumberOfItems(serOrderedList, offset);
}
+
+ @Override
+ protected int getListLength(byte[] serOrderedList, int offset) {
+ return AUnorderedListSerializerDeserializer.getUnorderedListLength(serOrderedList, offset + 1);
+ }
}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
index ab73df2..b732f40 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
@@ -62,8 +62,8 @@
probeListCount++;
byte[] buf = probeIter.getData();
int off = probeIter.getPos();
- int len = getItemLen(buf, off);
- keyEntry.set(buf, off, len);
+ int len = probeIter.getItemLen();
+ keyEntry.set(buf, off, len);
BinaryEntry entry = hashMap.get(keyEntry);
if (entry != null) {
// Increment second value.
@@ -94,7 +94,7 @@
}
return intersectionSize;
}
-
+
@Override
protected void writeResult(float jacc) throws IOException {
listBuilder.reset(listType);
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
index 9f5c9c8..391fb30 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
@@ -4,8 +4,6 @@
import java.io.IOException;
import java.util.Arrays;
-import edu.uci.ics.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
-import edu.uci.ics.asterix.formats.nontagged.AqlBinaryHashFunctionFactoryProvider;
import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import edu.uci.ics.asterix.om.base.AFloat;
import edu.uci.ics.asterix.om.base.AMutableFloat;
@@ -22,9 +20,10 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
-import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.ListItemBinaryComparatorFactory;
+import edu.uci.ics.asterix.dataflow.data.nontagged.hash.ListItemBinaryHashFunctionFactory;
public class SimilarityJaccardEvaluator implements ICopyEvaluator {
@@ -58,7 +57,8 @@
protected int firstStart = -1;
protected int secondStart = -1;
protected float jaccSim = 0.0f;
- protected ATypeTag itemTypeTag;
+ protected ATypeTag firstItemTypeTag;
+ protected ATypeTag secondItemTypeTag;
protected BinaryHashMap hashMap;
protected BinaryEntry keyEntry = new BinaryEntry();
@@ -105,6 +105,9 @@
firstTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[firstStart]);
secondTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[secondStart]);
+
+ firstItemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[firstStart + 1]);
+ secondItemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argOut.getByteArray()[secondStart + 1]);
}
protected boolean prepareLists(byte[] bytes, int firstStart, int secondStart, ATypeTag argType)
@@ -116,17 +119,12 @@
if (firstListIter.size() == 0 || secondListIter.size() == 0) {
return false;
}
- if (firstTypeTag == ATypeTag.ANY || secondTypeTag == ATypeTag.ANY) {
- throw new AlgebricksException("\n Jaccard can only be called on homogenous lists");
- }
// TODO: Check item types are compatible.
- itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[firstStart + 1]);
return true;
}
protected float computeResult(byte[] bytes, int firstStart, int secondStart, ATypeTag argType)
throws AlgebricksException {
- setHashMap(bytes, firstStart, secondStart);
// We will subtract the intersection size later to get the real union size.
int firstListSize = firstListIter.size();
int secondListSize = secondListIter.size();
@@ -136,7 +134,10 @@
AbstractAsterixListIterator probeList = (buildList == firstListIter) ? secondListIter : firstListIter;
int buildListSize = (buildList == firstListIter) ? firstListSize : secondListSize;
int probeListSize = (probeList == firstListIter) ? firstListSize : secondListSize;
+ ATypeTag buildItemTypeTag = (buildList == firstListIter) ? firstItemTypeTag : secondItemTypeTag;
+ ATypeTag probeItemTypeTag = (probeList == firstListIter) ? firstItemTypeTag : secondItemTypeTag;
+ setHashMap(bytes, buildItemTypeTag, probeItemTypeTag);
buildHashMap(buildList);
int intersectionSize = probeHashMap(probeList, buildListSize, probeListSize);
// Special indicator for the "check" version of jaccard.
@@ -154,7 +155,7 @@
while (buildIter.hasNext()) {
byte[] buf = buildIter.getData();
int off = buildIter.getPos();
- int len = getItemLen(buf, off);
+ int len = buildIter.getItemLen();
keyEntry.set(buf, off, len);
BinaryEntry entry = hashMap.put(keyEntry, valEntry);
if (entry != null) {
@@ -172,7 +173,7 @@
while (probeIter.hasNext()) {
byte[] buf = probeIter.getData();
int off = probeIter.getPos();
- int len = getItemLen(buf, off);
+ int len = probeIter.getItemLen();
keyEntry.set(buf, off, len);
BinaryEntry entry = hashMap.get(keyEntry);
if (entry != null) {
@@ -195,69 +196,19 @@
return intersectionSize;
}
- protected void setHashMap(byte[] bytes, int firstStart, int secondStart) {
+ protected void setHashMap(byte[] bytes, ATypeTag buildItemTypeTag, ATypeTag probeItemTypeTag) {
if (hashMap != null) {
hashMap.clear();
return;
}
- IBinaryHashFunction hashFunc = null;
- IBinaryComparator cmp = null;
- switch (itemTypeTag) {
- case INT32: {
- hashFunc = AqlBinaryHashFunctionFactoryProvider.INTEGER_POINTABLE_INSTANCE.createBinaryHashFunction();
- cmp = AqlBinaryComparatorFactoryProvider.INTEGER_POINTABLE_INSTANCE.createBinaryComparator();
- break;
- }
- case FLOAT: {
- hashFunc = AqlBinaryHashFunctionFactoryProvider.FLOAT_POINTABLE_INSTANCE.createBinaryHashFunction();
- cmp = AqlBinaryComparatorFactoryProvider.FLOAT_POINTABLE_INSTANCE.createBinaryComparator();
- break;
- }
- case DOUBLE: {
- hashFunc = AqlBinaryHashFunctionFactoryProvider.DOUBLE_POINTABLE_INSTANCE.createBinaryHashFunction();
- cmp = AqlBinaryComparatorFactoryProvider.DOUBLE_POINTABLE_INSTANCE.createBinaryComparator();
- break;
- }
- case STRING: {
- if (ignoreCase) {
- // Ignore case in comparisons and hashing.
- hashFunc = AqlBinaryHashFunctionFactoryProvider.UTF8STRING_LOWERCASE_POINTABLE_INSTANCE
- .createBinaryHashFunction();
- cmp = AqlBinaryComparatorFactoryProvider.UTF8STRING_LOWERCASE_POINTABLE_INSTANCE
- .createBinaryComparator();
- } else {
- hashFunc = AqlBinaryHashFunctionFactoryProvider.UTF8STRING_POINTABLE_INSTANCE
- .createBinaryHashFunction();
- cmp = AqlBinaryComparatorFactoryProvider.UTF8STRING_POINTABLE_INSTANCE.createBinaryComparator();
- }
- break;
- }
- default: {
- break;
- }
- }
- hashMap = new BinaryHashMap(TABLE_SIZE, TABLE_FRAME_SIZE, hashFunc, cmp);
- }
- protected int getItemLen(byte[] bytes, int itemOff) {
- switch (itemTypeTag) {
- case INT32: {
- return 4;
- }
- case FLOAT: {
- return 4;
- }
- case DOUBLE: {
- return 8;
- }
- case STRING: {
- // 2 bytes for the UTF8 len, plus the string data.
- return 2 + UTF8StringPointable.getUTFLength(bytes, itemOff);
- }
- default: {
- return -1;
- }
- }
+ IBinaryHashFunction putHashFunc = ListItemBinaryHashFunctionFactory.INSTANCE.createBinaryHashFunction(
+ buildItemTypeTag, ignoreCase);
+ IBinaryHashFunction getHashFunc = ListItemBinaryHashFunctionFactory.INSTANCE.createBinaryHashFunction(
+ probeItemTypeTag, ignoreCase);
+ IBinaryComparator cmp = ListItemBinaryComparatorFactory.INSTANCE.createBinaryComparator(buildItemTypeTag,
+ probeItemTypeTag, ignoreCase);
+ hashMap = new BinaryHashMap(TABLE_SIZE, TABLE_FRAME_SIZE, putHashFunc, getHashFunc, cmp);
}
protected boolean checkArgTypes(ATypeTag typeTag1, ATypeTag typeTag2) throws AlgebricksException {
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/BinaryHashMap.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/BinaryHashMap.java
index 240f8c7..6367996 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/BinaryHashMap.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/BinaryHashMap.java
@@ -20,150 +20,154 @@
* Intended to work with binary data and be able to map arbitrary key types to
* arbitrary value types, given that they have implementations of
* IBinaryHashFunction and IBinaryComparator.
- *
* Uses 2 bytes each to indicate the length of the key and the value.
* Uses 8 byte pointers for the linked list (4 bytes frame index, 4 bytes frame offset).
- *
* This class is NOT thread safe.
- *
*/
public class BinaryHashMap {
- // Special value to indicate an empty "bucket" in the header array.
- private static final long NULL_PTR = -1;
- private static final int PTR_SIZE = 8;
- private static final int SLOT_SIZE = 2;
- private static final int ENTRY_HEADER_SIZE = PTR_SIZE + 2 * SLOT_SIZE;
- private final IBinaryHashFunction hashFunc;
- private final IBinaryComparator cmp;
- private final BinaryEntry returnValue = new BinaryEntry();
-
- private final long[] listHeads;
- private final int frameSize;
- private final List<ByteBuffer> frames = new ArrayList<ByteBuffer>();
- private int currFrameIndex;
- private int nextOff;
- private int size;
-
- // Can be used for key or value.
- public static class BinaryEntry {
- public byte[] buf;
- public int off;
- public int len;
-
- public void set(byte[] buf, int off, int len) {
- this.buf = buf;
- this.off = off;
- this.len = len;
- }
-
- // Inefficient. Just for debugging.
- @SuppressWarnings("rawtypes")
- public String print(ISerializerDeserializer serde) throws HyracksDataException {
- ByteArrayInputStream inStream = new ByteArrayInputStream(buf, off, len);
+ // Special value to indicate an empty "bucket" in the header array.
+ private static final long NULL_PTR = -1;
+ private static final int PTR_SIZE = 8;
+ private static final int SLOT_SIZE = 2;
+ private static final int ENTRY_HEADER_SIZE = PTR_SIZE + 2 * SLOT_SIZE;
+ private final IBinaryHashFunction putHashFunc;
+ private final IBinaryHashFunction getHashFunc;
+ private final IBinaryComparator cmp;
+ private final BinaryEntry returnValue = new BinaryEntry();
+
+ private final long[] listHeads;
+ private final int frameSize;
+ private final List<ByteBuffer> frames = new ArrayList<ByteBuffer>();
+ private int currFrameIndex;
+ private int nextOff;
+ private int size;
+
+ // Can be used for key or value.
+ public static class BinaryEntry {
+ public byte[] buf;
+ public int off;
+ public int len;
+
+ public void set(byte[] buf, int off, int len) {
+ this.buf = buf;
+ this.off = off;
+ this.len = len;
+ }
+
+ // Inefficient. Just for debugging.
+ @SuppressWarnings("rawtypes")
+ public String print(ISerializerDeserializer serde) throws HyracksDataException {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(buf, off, len);
DataInput dataIn = new DataInputStream(inStream);
return serde.deserialize(dataIn).toString();
- }
- }
-
- public BinaryHashMap(int tableSize, int frameSize, IBinaryHashFunction hashFunc, IBinaryComparator cmp) {
- listHeads = new long[tableSize];
- this.frameSize = frameSize;
- this.hashFunc = hashFunc;
- this.cmp = cmp;
- frames.add(ByteBuffer.allocate(frameSize));
- clear();
- }
-
- /**
- * Inserts key, value into the hash map. If key already exists, returns
- * existing entry. Otherwise, returns null.
- *
- * @param key
- * @param value
- * @return
- */
- public BinaryEntry put(BinaryEntry key, BinaryEntry value) {
- return getPutInternal(key, value, true);
- }
-
- /**
- * Retrieves value for given key. Returns null if key doesn't exist.
- *
- * @param key
- * @param value
- * @return
- */
- public BinaryEntry get(BinaryEntry key) {
- return getPutInternal(key, null, false);
- }
-
- private BinaryEntry getPutInternal(BinaryEntry key, BinaryEntry value, boolean put) {
- int bucket = Math.abs(hashFunc.hash(key.buf, key.off, key.len) % listHeads.length);
- long headPtr = listHeads[bucket];
- if (headPtr == NULL_PTR) {
- // Key definitely doesn't exist yet.
- if (put) {
- listHeads[bucket] = appendEntry(key, value);
- }
- return null;
- }
- // Follow the chain until we found an entry matching the given key.
- int frameOff;
- ByteBuffer frame;
- do {
- int frameIndex = getFrameIndex(headPtr);
- frameOff = getFrameOffset(headPtr);
- frame = frames.get(frameIndex);
- int entryKeyOff = frameOff + ENTRY_HEADER_SIZE;
- int entryKeyLen = frame.getShort(frameOff);
- if (cmp.compare(frame.array(), entryKeyOff, entryKeyLen, key.buf,
- key.off, key.len) == 0) {
- // Key found, set values and return.
- int entryValOff = frameOff + ENTRY_HEADER_SIZE + entryKeyLen;
- int entryValLen = frame.getShort(frameOff + SLOT_SIZE);
- returnValue.set(frame.array(), entryValOff, entryValLen);
- return returnValue;
- }
- headPtr = frame.getLong(frameOff + 2 * SLOT_SIZE);
- } while (headPtr != NULL_PTR);
- // We've followed the chain to its end, and didn't find the key.
- if (put) {
- // Append the new entry, and set a pointer to it in the last entry we've checked.
- long newPtr = appendEntry(key, value);
- frame.putLong(frameOff + 2 * SLOT_SIZE, newPtr);
- }
- return null;
- }
-
- public long appendEntry(BinaryEntry key, BinaryEntry value) {
- ByteBuffer frame = frames.get(currFrameIndex);
- int requiredSpace = key.len + value.len + ENTRY_HEADER_SIZE;
- if (nextOff + requiredSpace >= frameSize) {
- // Entry doesn't fit on frame, allocate a new one.
- if (requiredSpace > frameSize) {
- throw new IllegalStateException("Key and value greater than framesize.");
- }
- frames.add(ByteBuffer.allocate(frameSize));
- currFrameIndex++;
- nextOff = 0;
- frame = frames.get(currFrameIndex);
- }
- writeEntryHeader(frame, nextOff, key.len, value.len, NULL_PTR);
- System.arraycopy(key.buf, key.off, frame.array(), nextOff + ENTRY_HEADER_SIZE, key.len);
- System.arraycopy(value.buf, value.off, frame.array(), nextOff + ENTRY_HEADER_SIZE + key.len, value.len);
- long entryPtr = getEntryPtr(currFrameIndex, nextOff);
- nextOff += requiredSpace;
- size++;
- return entryPtr;
- }
-
- private void writeEntryHeader(ByteBuffer frame, int targetOff, int keyLen, int valLen, long ptr) {
- frame.putShort(targetOff, (short) keyLen);
- frame.putShort(targetOff + SLOT_SIZE, (short) valLen);
- frame.putLong(targetOff + 2 * SLOT_SIZE, ptr);
- }
+ }
+ }
- private long getEntryPtr(int frameIndex, int frameOff) {
+ public BinaryHashMap(int tableSize, int frameSize, IBinaryHashFunction putHashFunc,
+ IBinaryHashFunction getHashFunc, IBinaryComparator cmp) {
+ listHeads = new long[tableSize];
+ this.frameSize = frameSize;
+ this.putHashFunc = putHashFunc;
+ this.getHashFunc = getHashFunc;
+ this.cmp = cmp;
+ frames.add(ByteBuffer.allocate(frameSize));
+ clear();
+ }
+
+ /**
+ * Inserts key, value into the hash map. If key already exists, returns
+ * existing entry. Otherwise, returns null.
+ *
+ * @param key
+ * @param value
+ * @return
+ */
+ public BinaryEntry put(BinaryEntry key, BinaryEntry value) {
+ return getPutInternal(key, value, true);
+ }
+
+ /**
+ * Retrieves value for given key. Returns null if key doesn't exist.
+ *
+ * @param key
+ * @param value
+ * @return
+ */
+ public BinaryEntry get(BinaryEntry key) {
+ return getPutInternal(key, null, false);
+ }
+
+ private BinaryEntry getPutInternal(BinaryEntry key, BinaryEntry value, boolean put) {
+ int bucket;
+ if (put) {
+ bucket = Math.abs(putHashFunc.hash(key.buf, key.off, key.len) % listHeads.length);
+ } else {
+ bucket = Math.abs(getHashFunc.hash(key.buf, key.off, key.len) % listHeads.length);
+ }
+ long headPtr = listHeads[bucket];
+ if (headPtr == NULL_PTR) {
+ // Key definitely doesn't exist yet.
+ if (put) {
+ listHeads[bucket] = appendEntry(key, value);
+ }
+ return null;
+ }
+ // Follow the chain until we found an entry matching the given key.
+ int frameOff;
+ ByteBuffer frame;
+ do {
+ int frameIndex = getFrameIndex(headPtr);
+ frameOff = getFrameOffset(headPtr);
+ frame = frames.get(frameIndex);
+ int entryKeyOff = frameOff + ENTRY_HEADER_SIZE;
+ int entryKeyLen = frame.getShort(frameOff);
+ if (cmp.compare(frame.array(), entryKeyOff, entryKeyLen, key.buf, key.off, key.len) == 0) {
+ // Key found, set values and return.
+ int entryValOff = frameOff + ENTRY_HEADER_SIZE + entryKeyLen;
+ int entryValLen = frame.getShort(frameOff + SLOT_SIZE);
+ returnValue.set(frame.array(), entryValOff, entryValLen);
+ return returnValue;
+ }
+ headPtr = frame.getLong(frameOff + 2 * SLOT_SIZE);
+ } while (headPtr != NULL_PTR);
+ // We've followed the chain to its end, and didn't find the key.
+ if (put) {
+ // Append the new entry, and set a pointer to it in the last entry we've checked.
+ long newPtr = appendEntry(key, value);
+ frame.putLong(frameOff + 2 * SLOT_SIZE, newPtr);
+ }
+ return null;
+ }
+
+ public long appendEntry(BinaryEntry key, BinaryEntry value) {
+ ByteBuffer frame = frames.get(currFrameIndex);
+ int requiredSpace = key.len + value.len + ENTRY_HEADER_SIZE;
+ if (nextOff + requiredSpace >= frameSize) {
+ // Entry doesn't fit on frame, allocate a new one.
+ if (requiredSpace > frameSize) {
+ throw new IllegalStateException("Key and value greater than framesize.");
+ }
+ frames.add(ByteBuffer.allocate(frameSize));
+ currFrameIndex++;
+ nextOff = 0;
+ frame = frames.get(currFrameIndex);
+ }
+ writeEntryHeader(frame, nextOff, key.len, value.len, NULL_PTR);
+ System.arraycopy(key.buf, key.off, frame.array(), nextOff + ENTRY_HEADER_SIZE, key.len);
+ System.arraycopy(value.buf, value.off, frame.array(), nextOff + ENTRY_HEADER_SIZE + key.len, value.len);
+ long entryPtr = getEntryPtr(currFrameIndex, nextOff);
+ nextOff += requiredSpace;
+ size++;
+ return entryPtr;
+ }
+
+ private void writeEntryHeader(ByteBuffer frame, int targetOff, int keyLen, int valLen, long ptr) {
+ frame.putShort(targetOff, (short) keyLen);
+ frame.putShort(targetOff + SLOT_SIZE, (short) valLen);
+ frame.putLong(targetOff + 2 * SLOT_SIZE, ptr);
+ }
+
+ private long getEntryPtr(int frameIndex, int frameOff) {
return (((long) frameIndex) << 32) + frameOff;
}
@@ -175,93 +179,94 @@
return (int) (ptr & 0xffffffff);
}
- public int size() {
- return size;
- }
+ public int size() {
+ return size;
+ }
- public boolean isEmpty() {
- return size > 0;
- }
+ public boolean isEmpty() {
+ return size > 0;
+ }
- public void clear() {
- // Initialize all entries to point to nothing.
- Arrays.fill(listHeads, NULL_PTR);
- currFrameIndex = 0;
- nextOff = 0;
- size = 0;
- }
-
- public Iterator<Pair<BinaryEntry, BinaryEntry>> iterator() {
- return new BinaryHashMapIterator();
- }
-
- public class BinaryHashMapIterator implements Iterator<Pair<BinaryEntry, BinaryEntry> > {
- private final Pair<BinaryEntry, BinaryEntry> val = new Pair<BinaryEntry, BinaryEntry>(new BinaryEntry(), new BinaryEntry());
- private int listHeadIndex;
- private ByteBuffer frame;
- private int frameIndex;
- private int frameOff;
-
- public BinaryHashMapIterator() {
- listHeadIndex = 0;
- frame = null;
- frameIndex = -1;
- frameOff = -1;
- }
-
- @Override
- public boolean hasNext() {
- if (frame != null) {
- long nextPtr = frame.getLong(frameOff + 2 * SLOT_SIZE);
- if (nextPtr == NULL_PTR) {
- // End of current list.
- listHeadIndex++;
- return nextListHead();
- } else {
- // Follow pointer.
- setValue(nextPtr);
- return true;
- }
- }
- return nextListHead();
- }
+ public void clear() {
+ // Initialize all entries to point to nothing.
+ Arrays.fill(listHeads, NULL_PTR);
+ currFrameIndex = 0;
+ nextOff = 0;
+ size = 0;
+ }
- private boolean nextListHead() {
- // Position to first non-null list-head pointer.
- while(listHeadIndex < listHeads.length && listHeads[listHeadIndex] == NULL_PTR) {
- listHeadIndex++;
- }
- if (listHeadIndex < listHeads.length) {
- // Positioned to first non-null list head.
- setValue(listHeads[listHeadIndex]);
- return true;
- } else {
- // No more lists.
- frame = null;
- return false;
- }
- }
-
- private void setValue(long ptr) {
- frameIndex = getFrameIndex(ptr);
- frameOff = getFrameOffset(ptr);
- frame = frames.get(frameIndex);
- int entryKeyOff = frameOff + ENTRY_HEADER_SIZE;
- int entryKeyLen = frame.getShort(frameOff);
- int entryValOff = frameOff + ENTRY_HEADER_SIZE + entryKeyLen;
- int entryValLen = frame.getShort(frameOff + SLOT_SIZE);
- val.first.set(frame.array(), entryKeyOff, entryKeyLen);
- val.second.set(frame.array(), entryValOff, entryValLen);
- }
-
- @Override
- public Pair<BinaryEntry, BinaryEntry> next() {
- return val;
- }
+ public Iterator<Pair<BinaryEntry, BinaryEntry>> iterator() {
+ return new BinaryHashMapIterator();
+ }
- @Override
- public void remove() {
- throw new UnsupportedOperationException("Remove not implemented");
- }
- }
+ public class BinaryHashMapIterator implements Iterator<Pair<BinaryEntry, BinaryEntry>> {
+ private final Pair<BinaryEntry, BinaryEntry> val = new Pair<BinaryEntry, BinaryEntry>(new BinaryEntry(),
+ new BinaryEntry());
+ private int listHeadIndex;
+ private ByteBuffer frame;
+ private int frameIndex;
+ private int frameOff;
+
+ public BinaryHashMapIterator() {
+ listHeadIndex = 0;
+ frame = null;
+ frameIndex = -1;
+ frameOff = -1;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (frame != null) {
+ long nextPtr = frame.getLong(frameOff + 2 * SLOT_SIZE);
+ if (nextPtr == NULL_PTR) {
+ // End of current list.
+ listHeadIndex++;
+ return nextListHead();
+ } else {
+ // Follow pointer.
+ setValue(nextPtr);
+ return true;
+ }
+ }
+ return nextListHead();
+ }
+
+ private boolean nextListHead() {
+ // Position to first non-null list-head pointer.
+ while (listHeadIndex < listHeads.length && listHeads[listHeadIndex] == NULL_PTR) {
+ listHeadIndex++;
+ }
+ if (listHeadIndex < listHeads.length) {
+ // Positioned to first non-null list head.
+ setValue(listHeads[listHeadIndex]);
+ return true;
+ } else {
+ // No more lists.
+ frame = null;
+ return false;
+ }
+ }
+
+ private void setValue(long ptr) {
+ frameIndex = getFrameIndex(ptr);
+ frameOff = getFrameOffset(ptr);
+ frame = frames.get(frameIndex);
+ int entryKeyOff = frameOff + ENTRY_HEADER_SIZE;
+ int entryKeyLen = frame.getShort(frameOff);
+ int entryValOff = frameOff + ENTRY_HEADER_SIZE + entryKeyLen;
+ int entryValLen = frame.getShort(frameOff + SLOT_SIZE);
+ val.first.set(frame.array(), entryKeyOff, entryKeyLen);
+ val.second.set(frame.array(), entryValOff, entryValLen);
+ }
+
+ @Override
+ public Pair<BinaryEntry, BinaryEntry> next() {
+ return val;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException("Remove not implemented");
+ }
+ }
}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/NumericCaretDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/NumericCaretDescriptor.java
new file mode 100644
index 0000000..208c454
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/NumericCaretDescriptor.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.runtime.evaluators.functions;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class NumericCaretDescriptor extends AbstractNumericArithmeticEval {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new NumericCaretDescriptor();
+ }
+ };
+
+ /* (non-Javadoc)
+ * @see edu.uci.ics.asterix.runtime.evaluators.functions.AbstractNumericArithmeticEval#evaluateInteger(long, long)
+ */
+ @Override
+ protected long evaluateInteger(long lhs, long rhs) throws HyracksDataException {
+ double result = Math.pow(lhs, rhs);
+ if (result > Long.MAX_VALUE) {
+ throw new ArithmeticException("Overflow of caret operation: " + lhs + " ^ " + rhs);
+ }
+ if (result < Long.MIN_VALUE) {
+ throw new ArithmeticException("Underflow of caret operation: " + lhs + " ^ " + rhs);
+ }
+ return (long) result;
+ }
+
+ /* (non-Javadoc)
+ * @see edu.uci.ics.asterix.runtime.evaluators.functions.AbstractNumericArithmeticEval#evaluateDouble(double, double)
+ */
+ @Override
+ protected double evaluateDouble(double lhs, double rhs) throws HyracksDataException {
+ return Math.pow(lhs, rhs);
+ }
+
+ /* (non-Javadoc)
+ * @see edu.uci.ics.asterix.om.functions.AbstractFunctionDescriptor#getIdentifier()
+ */
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.CARET;
+ }
+
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
index 2c81b7c..af647e9 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
@@ -149,6 +149,7 @@
import edu.uci.ics.asterix.runtime.evaluators.functions.NotNullDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.NumericAbsDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.NumericAddDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.NumericCaretDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.NumericCeilingDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.NumericDivideDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.NumericFloorDescriptor;
@@ -347,6 +348,7 @@
temp.add(NumericMultiplyDescriptor.FACTORY);
temp.add(NumericSubtractDescriptor.FACTORY);
temp.add(NumericModuloDescriptor.FACTORY);
+ temp.add(NumericCaretDescriptor.FACTORY);
temp.add(IsNullDescriptor.FACTORY);
temp.add(NotDescriptor.FACTORY);
temp.add(LenDescriptor.FACTORY);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
index e06b5df..c3f9747 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMBTreeLocalResourceMetadata.java
@@ -6,6 +6,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
@@ -27,14 +28,27 @@
private final int[] bloomFilterKeyFields;
private final int memPageSize;
private final int memNumPages;
+ private FileSplit[] fileSplits;
+ private int ioDeviceID;
public LSMBTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
- int[] bloomFilterKeyFields, boolean isPrimary, int memPageSize, int memNumPages) {
+ int[] bloomFilterKeyFields, boolean isPrimary, int memPageSize, int memNumPages, FileSplit[] fileSplits) {
this.typeTraits = typeTraits;
this.cmpFactories = cmpFactories;
this.bloomFilterKeyFields = bloomFilterKeyFields;
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
+ this.fileSplits = fileSplits;
+ }
+
+ public LSMBTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
+ int[] bloomFilterKeyFields, boolean isPrimary, int memPageSize, int memNumPages, int ioDeviceID) {
+ this.typeTraits = typeTraits;
+ this.cmpFactories = cmpFactories;
+ this.bloomFilterKeyFields = bloomFilterKeyFields;
+ this.memPageSize = memPageSize;
+ this.memNumPages = memNumPages;
+ this.ioDeviceID = ioDeviceID;
}
@Override
@@ -45,13 +59,13 @@
memNumPages, new TransientFileMapManager());
ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
IInMemoryFreePageManager memFreePageManager = new InMemoryFreePageManager(memNumPages, metaDataFrameFactory);
- LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager,
- runtimeContextProvider.getIOManager(), file, runtimeContextProvider.getBufferCache(),
- runtimeContextProvider.getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields,
- runtimeContextProvider.getBloomFilterFalsePositiveRate(), runtimeContextProvider.getLSMMergePolicy(),
- runtimeContextProvider.getLSMBTreeOperationTrackerFactory(),
- runtimeContextProvider.getLSMIOScheduler(),
- runtimeContextProvider.getLSMBTreeIOOperationCallbackProvider(), partition);
+ LSMBTree lsmBTree = LSMBTreeUtils.createLSMTree(memBufferCache, memFreePageManager, runtimeContextProvider
+ .getIOManager(), file, runtimeContextProvider.getBufferCache(), runtimeContextProvider
+ .getFileMapManager(), typeTraits, cmpFactories, bloomFilterKeyFields, runtimeContextProvider
+ .getBloomFilterFalsePositiveRate(), runtimeContextProvider.getLSMMergePolicy(), runtimeContextProvider
+ .getLSMBTreeOperationTrackerFactory(), runtimeContextProvider.getLSMIOScheduler(),
+ runtimeContextProvider.getLSMBTreeIOOperationCallbackProvider(), fileSplits == null ? ioDeviceID
+ : fileSplits[partition].getIODeviceId());
return lsmBTree;
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
index d56c6ee..89a32c5 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMInvertedIndexLocalResourceMetadata.java
@@ -4,6 +4,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IndexException;
@@ -28,11 +29,12 @@
private final int memPageSize;
private final int memNumPages;
private final boolean isPartitioned;
+ private final FileSplit[] fileSplits;
public LSMInvertedIndexLocalResourceMetadata(ITypeTraits[] invListTypeTraits,
IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, int memPageSize,
- int memNumPages, boolean isPartitioned) {
+ int memNumPages, boolean isPartitioned, FileSplit[] fileSplits) {
this.invListTypeTraits = invListTypeTraits;
this.invListCmpFactories = invListCmpFactories;
this.tokenTypeTraits = tokenTypeTraits;
@@ -41,6 +43,7 @@
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
this.isPartitioned = isPartitioned;
+ this.fileSplits = fileSplits;
}
@Override
@@ -62,7 +65,8 @@
runtimeContextProvider.getLSMMergePolicy(),
runtimeContextProvider.getLSMInvertedIndexOperationTrackerFactory(),
runtimeContextProvider.getLSMIOScheduler(),
- runtimeContextProvider.getLSMInvertedIndexIOOperationCallbackProvider(), partition);
+ runtimeContextProvider.getLSMInvertedIndexIOOperationCallbackProvider(),
+ fileSplits[partition].getIODeviceId());
} else {
return InvertedIndexUtils.createLSMInvertedIndex(memBufferCache, memFreePageManager,
runtimeContextProvider.getFileMapManager(), invListTypeTraits, invListCmpFactories,
@@ -72,7 +76,8 @@
runtimeContextProvider.getLSMMergePolicy(),
runtimeContextProvider.getLSMInvertedIndexOperationTrackerFactory(),
runtimeContextProvider.getLSMIOScheduler(),
- runtimeContextProvider.getLSMInvertedIndexIOOperationCallbackProvider(), partition);
+ runtimeContextProvider.getLSMInvertedIndexIOOperationCallbackProvider(),
+ fileSplits[partition].getIODeviceId());
}
} catch (IndexException e) {
throw new HyracksDataException(e);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index 95fab45..2506697 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -8,6 +8,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.storage.am.common.api.IInMemoryFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
@@ -33,11 +34,12 @@
private final ILinearizeComparatorFactory linearizeCmpFactory;
private final int memPageSize;
private final int memNumPages;
+ private final FileSplit[] fileSplits;
public LSMRTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int memPageSize,
- int memNumPages) {
+ int memNumPages, FileSplit[] fileSplits) {
this.typeTraits = typeTraits;
this.rtreeCmpFactories = rtreeCmpFactories;
this.btreeCmpFactories = btreeCmpFactories;
@@ -46,6 +48,7 @@
this.linearizeCmpFactory = linearizeCmpFactory;
this.memPageSize = memPageSize;
this.memNumPages = memNumPages;
+ this.fileSplits = fileSplits;
}
@Override
@@ -66,7 +69,8 @@
runtimeContextProvider.getLSMMergePolicy(),
runtimeContextProvider.getLSMRTreeOperationTrackerFactory(),
runtimeContextProvider.getLSMIOScheduler(),
- runtimeContextProvider.getLSMRTreeIOOperationCallbackProvider(), linearizeCmpFactory, partition);
+ runtimeContextProvider.getLSMRTreeIOOperationCallbackProvider(), linearizeCmpFactory,
+ fileSplits[partition].getIODeviceId());
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 191bc8a..2361bac 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -29,6 +29,7 @@
import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
import edu.uci.ics.hyracks.storage.common.file.LocalResource;
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactory;
@@ -36,26 +37,34 @@
public class PersistentLocalResourceRepository implements ILocalResourceRepository {
private static final Logger LOGGER = Logger.getLogger(PersistentLocalResourceRepository.class.getName());
- private final String mountPoint;
- private static final String ROOT_METADATA_DIRECTORY = "asterix_root_metadata/";
- private static final String ROOT_METADATA_FILE_NAME_PREFIX = ".asterix_root_metadata_";
+ private final String[] mountPoints;
+ private static final String ROOT_METADATA_DIRECTORY = "asterix_root_metadata";
+ private static final String ROOT_METADATA_FILE_NAME_PREFIX = ".asterix_root_metadata";
private static final long ROOT_LOCAL_RESOURCE_ID = -4321;
private static final String METADATA_FILE_NAME = ".metadata";
private Map<String, LocalResource> name2ResourceMap = new HashMap<String, LocalResource>();
private Map<Long, LocalResource> id2ResourceMap = new HashMap<Long, LocalResource>();
- private String rootMetadataFileName;
- private String rootDir;
+ private final int numIODevices;
- public PersistentLocalResourceRepository(String mountPoint) throws HyracksDataException {
- File mountPointDir = new File(mountPoint);
- if (!mountPointDir.exists()) {
- throw new HyracksDataException(mountPointDir.getAbsolutePath() + "doesn't exist.");
+ public PersistentLocalResourceRepository(List<IODeviceHandle> devices) throws HyracksDataException {
+ numIODevices = devices.size();
+ this.mountPoints = new String[numIODevices];
+ for (int i = 0; i < numIODevices; i++) {
+ String mountPoint = devices.get(i).getPath().getPath();
+ File mountPointDir = new File(mountPoint);
+ if (!mountPointDir.exists()) {
+ throw new HyracksDataException(mountPointDir.getAbsolutePath() + "doesn't exist.");
+ }
+ if (!mountPoint.endsWith(System.getProperty("file.separator"))) {
+ mountPoints[i] = new String(mountPoint + System.getProperty("file.separator"));
+ } else {
+ mountPoints[i] = new String(mountPoint);
+ }
}
- if (!mountPoint.endsWith(System.getProperty("file.separator"))) {
- this.mountPoint = new String(mountPoint + System.getProperty("file.separator"));
- } else {
- this.mountPoint = new String(mountPoint);
- }
+ }
+
+ private String prepareRootMetaDataFileName(String mountPoint, String nodeId, int ioDeviceId) {
+ return mountPoint + ROOT_METADATA_DIRECTORY + "_" + nodeId + "_" + "iodevice" + ioDeviceId;
}
public void initialize(String nodeId, String rootDir, boolean isNewUniverse, ResourceIdFactory resourceIdFactory)
@@ -63,53 +72,40 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Initializing local resource repository ... ");
}
- LocalResource rootLocalResource = null;
- //#. if the rootMetadataFile doesn't exist, create it and return.
- rootMetadataFileName = new String(mountPoint + ROOT_METADATA_DIRECTORY + ROOT_METADATA_FILE_NAME_PREFIX
- + nodeId);
- File rootMetadataFile = new File(rootMetadataFileName);
if (isNewUniverse) {
- File rootMetadataDir = new File(mountPoint + ROOT_METADATA_DIRECTORY);
- if (!rootMetadataDir.exists()) {
- rootMetadataDir.mkdir();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("created the root-metadata-file's directory: " + rootMetadataDir.getAbsolutePath());
+ //#. if the rootMetadataFile doesn't exist, create it and return.
+ for (int i = 0; i < numIODevices; i++) {
+ String rootMetadataFileName = prepareRootMetaDataFileName(mountPoints[i], nodeId, i) + File.separator
+ + ROOT_METADATA_FILE_NAME_PREFIX;
+ File rootMetadataFile = new File(rootMetadataFileName);
+
+ File rootMetadataDir = new File(prepareRootMetaDataFileName(mountPoints[i], nodeId, i));
+ if (!rootMetadataDir.exists()) {
+ rootMetadataDir.mkdir();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("created the root-metadata-file's directory: " + rootMetadataDir.getAbsolutePath());
+ }
}
- }
- rootMetadataFile.delete();
- if (rootDir.startsWith(System.getProperty("file.separator"))) {
- this.rootDir = new String(mountPoint + rootDir.substring(System.getProperty("file.separator").length()));
- } else {
- this.rootDir = new String(mountPoint + rootDir);
- }
- rootLocalResource = new LocalResource(ROOT_LOCAL_RESOURCE_ID, rootMetadataFileName, 0, 0, this.rootDir);
- insert(rootLocalResource);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("created the root-metadata-file: " + rootMetadataFileName);
- }
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Completed the initialization of the local resource repository");
- }
- return;
- }
+ rootMetadataFile.delete();
+ String mountedRootDir;
+ if (rootDir.startsWith(System.getProperty("file.separator"))) {
+ mountedRootDir = new String(mountPoints[i]
+ + rootDir.substring(System.getProperty("file.separator").length()));
+ } else {
+ mountedRootDir = new String(mountPoints[i] + rootDir);
+ }
+ LocalResource rootLocalResource = new LocalResource(ROOT_LOCAL_RESOURCE_ID, rootMetadataFileName, 0, 0,
+ mountedRootDir);
+ insert(rootLocalResource, i);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("created the root-metadata-file: " + rootMetadataFileName);
+ }
- //#. if the rootMetadataFile exists, read it and set this.rootDir.
- rootLocalResource = readLocalResource(rootMetadataFile);
- this.rootDir = (String) rootLocalResource.getResourceObject();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("The root directory of the local resource repository is " + this.rootDir);
- }
-
- //#. load all local resources.
- File rootDirFile = new File(this.rootDir);
- if (!rootDirFile.exists()) {
- //rootDir may not exist if this node is not the metadata node and doesn't have any user data.
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("The root directory of the local resource repository doesn't exist: there is no local resource.");
- LOGGER.info("Completed the initialization of the local resource repository");
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Completed the initialization of the local resource repository");
+ }
}
return;
}
@@ -124,27 +120,50 @@
}
};
- long maxResourceId = 0;
- File[] dataverseFileList = rootDirFile.listFiles();
- if (dataverseFileList == null) {
- throw new HyracksDataException("Metadata dataverse doesn't exist.");
- }
- for (File dataverseFile : dataverseFileList) {
- if (dataverseFile.isDirectory()) {
- File[] indexFileList = dataverseFile.listFiles();
- if (indexFileList != null) {
- for (File indexFile : indexFileList) {
- if (indexFile.isDirectory()) {
- File[] metadataFiles = indexFile.listFiles(filter);
- if (metadataFiles != null) {
- for (File metadataFile : metadataFiles) {
- LocalResource localResource = readLocalResource(metadataFile);
- id2ResourceMap.put(localResource.getResourceId(), localResource);
- name2ResourceMap.put(localResource.getResourceName(), localResource);
- maxResourceId = Math.max(localResource.getResourceId(), maxResourceId);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("loaded local resource - [id: " + localResource.getResourceId()
- + ", name: " + localResource.getResourceName() + "]");
+ for (int i = 0; i < numIODevices; i++) {
+ String rootMetadataFileName = prepareRootMetaDataFileName(mountPoints[i], nodeId, i) + File.separator
+ + ROOT_METADATA_FILE_NAME_PREFIX;
+ File rootMetadataFile = new File(rootMetadataFileName);
+ //#. if the rootMetadataFile exists, read it and set this.rootDir.
+ LocalResource rootLocalResource = readLocalResource(rootMetadataFile);
+ String mountedRootDir = (String) rootLocalResource.getResourceObject();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("The root directory of the local resource repository is " + mountedRootDir);
+ }
+
+ //#. load all local resources.
+ File rootDirFile = new File(mountedRootDir);
+ if (!rootDirFile.exists()) {
+ //rootDir may not exist if this node is not the metadata node and doesn't have any user data.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("The root directory of the local resource repository doesn't exist: there is no local resource.");
+ LOGGER.info("Completed the initialization of the local resource repository");
+ }
+ continue;
+ }
+
+ long maxResourceId = 0;
+ File[] dataverseFileList = rootDirFile.listFiles();
+ if (dataverseFileList == null) {
+ throw new HyracksDataException("Metadata dataverse doesn't exist.");
+ }
+ for (File dataverseFile : dataverseFileList) {
+ if (dataverseFile.isDirectory()) {
+ File[] indexFileList = dataverseFile.listFiles();
+ if (indexFileList != null) {
+ for (File indexFile : indexFileList) {
+ if (indexFile.isDirectory()) {
+ File[] metadataFiles = indexFile.listFiles(filter);
+ if (metadataFiles != null) {
+ for (File metadataFile : metadataFiles) {
+ LocalResource localResource = readLocalResource(metadataFile);
+ id2ResourceMap.put(localResource.getResourceId(), localResource);
+ name2ResourceMap.put(localResource.getResourceName(), localResource);
+ maxResourceId = Math.max(localResource.getResourceId(), maxResourceId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("loaded local resource - [id: " + localResource.getResourceId()
+ + ", name: " + localResource.getResourceName() + "]");
+ }
}
}
}
@@ -152,11 +171,11 @@
}
}
}
- }
- resourceIdFactory.initId(maxResourceId + 1);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("The resource id factory is intialized with the value: " + (maxResourceId + 1));
- LOGGER.info("Completed the initialization of the local resource repository");
+ resourceIdFactory.initId(maxResourceId + 1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("The resource id factory is intialized with the value: " + (maxResourceId + 1));
+ LOGGER.info("Completed the initialization of the local resource repository");
+ }
}
}
@@ -171,7 +190,7 @@
}
@Override
- public synchronized void insert(LocalResource resource) throws HyracksDataException {
+ public synchronized void insert(LocalResource resource, int ioDeviceId) throws HyracksDataException {
long id = resource.getResourceId();
if (id2ResourceMap.containsKey(id)) {
@@ -185,8 +204,10 @@
FileOutputStream fos = null;
ObjectOutputStream oosToFos = null;
+
try {
- fos = new FileOutputStream(getFileName(mountPoint, resource.getResourceName(), resource.getResourceId()));
+ fos = new FileOutputStream(getFileName(mountPoints[ioDeviceId], resource.getResourceName(),
+ resource.getResourceId()));
oosToFos = new ObjectOutputStream(fos);
oosToFos.writeObject(resource);
oosToFos.flush();
@@ -211,26 +232,26 @@
}
@Override
- public synchronized void deleteResourceById(long id) throws HyracksDataException {
+ public synchronized void deleteResourceById(long id, int ioDeviceId) throws HyracksDataException {
LocalResource resource = id2ResourceMap.get(id);
if (resource == null) {
throw new HyracksDataException("Resource doesn't exist");
}
id2ResourceMap.remove(id);
name2ResourceMap.remove(resource.getResourceName());
- File file = new File(getFileName(mountPoint, resource.getResourceName(), resource.getResourceId()));
+ File file = new File(getFileName(mountPoints[ioDeviceId], resource.getResourceName(), resource.getResourceId()));
file.delete();
}
@Override
- public synchronized void deleteResourceByName(String name) throws HyracksDataException {
+ public synchronized void deleteResourceByName(String name, int ioDeviceId) throws HyracksDataException {
LocalResource resource = name2ResourceMap.get(name);
if (resource == null) {
throw new HyracksDataException("Resource doesn't exist");
}
id2ResourceMap.remove(resource.getResourceId());
name2ResourceMap.remove(name);
- File file = new File(getFileName(mountPoint, resource.getResourceName(), resource.getResourceId()));
+ File file = new File(getFileName(mountPoints[ioDeviceId], resource.getResourceName(), resource.getResourceId()));
file.delete();
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
index c7efca5..f6847f9 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepositoryFactory.java
@@ -14,11 +14,8 @@
*/
package edu.uci.ics.asterix.transaction.management.resource;
-import java.util.List;
-
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.IIOManager;
-import edu.uci.ics.hyracks.api.io.IODeviceHandle;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepository;
import edu.uci.ics.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
@@ -31,7 +28,6 @@
@Override
public ILocalResourceRepository createRepository() throws HyracksDataException {
- List<IODeviceHandle> devices = ioManager.getIODevices();
- return new PersistentLocalResourceRepository(devices.get(0).getPath().getPath());
+ return new PersistentLocalResourceRepository(ioManager.getIODevices());
}
-}
+}
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index c313bd7..aa0bc82 100644
--- a/pom.xml
+++ b/pom.xml
@@ -135,14 +135,6 @@
<url>http://obelix.ics.uci.edu/nexus/content/repositories/algebricks-snapshots/</url>
</repository>
</repositories>
- <dependencies>
- <dependency>
- <groupId>net.sourceforge.cobertura</groupId>
- <artifactId>cobertura</artifactId>
- <version>1.9.4</version>
- <optional>true</optional>
- </dependency>
- </dependencies>
<dependencyManagement>
<dependencies>
<dependency>