1) added support to collect logs 2) modified naming convention for asterix_root_metadata
diff --git a/asterix-events/src/main/resources/events/events.xml b/asterix-events/src/main/resources/events/events.xml
index f85e3ea..068367e 100644
--- a/asterix-events/src/main/resources/events/events.xml
+++ b/asterix-events/src/main/resources/events/events.xml
@@ -70,6 +70,13 @@
<daemon>false</daemon>
</event>
<event>
+ <type>directory_copy</type>
+ <script>file/dir_copy.sh</script>
+ <description>Copies a directory (and its contents) from the remove file system to the local file system</description>
+ <args>destination_node destination_path local_source_path</args>
+ <daemon>false</daemon>
+ </event>
+ <event>
<type>file_delete</type>
<script>file/delete.sh</script>
<description>Deletes a file on the local file system to a remote node</description>
diff --git a/asterix-events/src/main/resources/events/file/dir_copy.sh b/asterix-events/src/main/resources/events/file/dir_copy.sh
new file mode 100755
index 0000000..6015dca
--- /dev/null
+++ b/asterix-events/src/main/resources/events/file/dir_copy.sh
@@ -0,0 +1,7 @@
+USERNAME=$1
+SRC_HOST=$2
+SRC_DIR=$3
+DEST_DIR=$4
+mkdir -p $DEST_DIR
+echo "scp -r $USERNAME@$SRC_HOST:$SRC_DIR $DEST_DIR"
+scp -r $USERNAME@$SRC_HOST:$SRC_DIR $DEST_DIR
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/HelpCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/HelpCommand.java
index 3bc700d..76e40fa 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/HelpCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/HelpCommand.java
@@ -58,6 +58,9 @@
case ALTER:
helpMessage = new AlterCommand().getUsageDescription();
break;
+ case LOG:
+ helpMessage = new LogCommand().getUsageDescription();
+ break;
default:
helpMessage = "Unknown command " + command;
}
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java
index 08c3039..3f99ab0 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/command/LogCommand.java
@@ -15,7 +15,10 @@
package edu.uci.ics.asterix.installer.command;
import java.io.File;
+import java.io.FilenameFilter;
+import java.util.Date;
+import org.apache.commons.io.FileUtils;
import org.kohsuke.args4j.Option;
import edu.uci.ics.asterix.event.management.EventrixClient;
@@ -23,34 +26,53 @@
import edu.uci.ics.asterix.installer.driver.InstallerDriver;
import edu.uci.ics.asterix.installer.driver.InstallerUtil;
import edu.uci.ics.asterix.installer.error.InstallerException;
-import edu.uci.ics.asterix.installer.error.VerificationUtil;
import edu.uci.ics.asterix.installer.events.PatternCreator;
import edu.uci.ics.asterix.installer.model.AsterixInstance;
-import edu.uci.ics.asterix.installer.model.AsterixRuntimeState;
import edu.uci.ics.asterix.installer.model.AsterixInstance.State;
-import edu.uci.ics.asterix.installer.service.ServiceProvider;
public class LogCommand extends AbstractCommand {
@Override
protected void execCommand() throws Exception {
- InstallerDriver.initConfig();
+ InstallerDriver.initConfig(true);
String asterixInstanceName = ((LogConfig) config).name;
- AsterixInstance instance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE);
+ AsterixInstance instance = InstallerUtil.validateAsterixInstanceExists(asterixInstanceName, State.INACTIVE,
+ State.UNUSABLE, State.ACTIVE);
PatternCreator pc = new PatternCreator();
EventrixClient client = InstallerUtil.getEventrixClient(instance.getCluster());
- String outputDir = ((LogConfig) config).outputDir == null ? instance.getCluster().getWorkingDir().getDir()
+ String outputDir = ((LogConfig) config).outputDir == null ? InstallerDriver.getManagixHome() + File.separator + "logdump"
: ((LogConfig) config).outputDir;
File f = new File(outputDir);
+ String outputDirPath = f.getAbsolutePath();
if (!f.exists()) {
boolean success = f.mkdirs();
if (!success) {
- throw new InstallerException("Unable to create output directory:" + outputDir);
+ throw new InstallerException("Unable to create output directory:" + outputDirPath);
}
}
- Patterns transferLogPattern = pc.getTransferLogPattern(asterixInstanceName, instance.getCluster(), outputDir);
+ Patterns transferLogPattern = pc.getGenerateLogPattern(asterixInstanceName, instance.getCluster(), outputDirPath);
client.submit(transferLogPattern);
- LOGGER.info("Log tar ball located at " + outputDir);
+ File outputDirFile = new File(outputDirPath);
+ final String destFileName = "log_" + new Date().toString().replace(' ', '_') + ".zip";
+ File destFile = new File(outputDirFile, destFileName);
+ InstallerUtil.zipDir(outputDirFile, destFile);
+
+ String[] filesToDelete = outputDirFile.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return !name.equals(destFileName);
+ }
+
+ });
+ for (String fileS : filesToDelete) {
+ f = new File(outputDirFile, fileS);
+ if (f.isDirectory()) {
+ FileUtils.deleteDirectory(f);
+ } else {
+ f.delete();
+ }
+ }
+ LOGGER.info("Log zip archive created at " + destFile.getAbsolutePath());
}
@Override
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
index 5014ff7..90c797d 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
@@ -68,6 +68,10 @@
public static final String TXN_LOG_DIR_KEY_SUFFIX = "txnLogDir";
public static final String ASTERIX_CONFIGURATION_FILE = "asterix-configuration.xml";
public static final String TXN_LOG_CONFIGURATION_FILE = "log.properties";
+ public static final int CLUSTER_NET_PORT_DEFAULT = 1098;
+ public static final int CLIENT_NET_PORT_DEFAULT = 1099;
+ public static final int HTTP_PORT_DEFAULT = 8888;
+ public static final int WEB_INTERFACE_PORT_DEFAULT = 19001;
public static AsterixInstance createAsterixInstance(String asterixInstanceName, Cluster cluster,
AsterixConfiguration asterixConfiguration) throws FileNotFoundException, IOException {
@@ -104,15 +108,22 @@
}
}
clusterProperties.add(new Property("ASTERIX_HOME", cluster.getWorkingDir().getDir() + File.separator
- + "asterix"));
- clusterProperties.add(new Property("CLUSTER_NET_IP", cluster.getMasterNode().getClusterIp()));
- clusterProperties.add(new Property("CLUSTER_NET_PORT", cluster.getPorts().getClusterPort()));
- clusterProperties.add(new Property("CLIENT_NET_IP", cluster.getMasterNode().getClientIp()));
- clusterProperties.add(new Property("CLIENT_NET_PORT", cluster.getPorts().getClientPort()));
+ + "asterix"));
clusterProperties.add(new Property("LOG_DIR", cluster.getLogDir()));
clusterProperties.add(new Property("JAVA_HOME", cluster.getJavaHome()));
clusterProperties.add(new Property("WORKING_DIR", cluster.getWorkingDir().getDir()));
- clusterProperties.add(new Property("HTTP_PORT", cluster.getPorts().getHttpPort()));
+ clusterProperties.add(new Property("CLIENT_NET_IP", cluster.getMasterNode().getClientIp()));
+ clusterProperties.add(new Property("CLUSTER_NET_IP", cluster.getMasterNode().getClusterIp()));
+
+ int clusterNetPort = cluster.getPorts() != null && cluster.getPorts().getClusterPort() != null ? Integer
+ .parseInt(cluster.getPorts().getClusterPort()) : CLUSTER_NET_PORT_DEFAULT;
+ int clientNetPort = cluster.getPorts() != null && cluster.getPorts().getClientPort() != null ? Integer
+ .parseInt(cluster.getPorts().getClientPort()) : CLIENT_NET_PORT_DEFAULT;
+ int httpPort = cluster.getPorts() != null && cluster.getPorts().getHttpPort() != null ? Integer
+ .parseInt(cluster.getPorts().getHttpPort()) : HTTP_PORT_DEFAULT;
+ clusterProperties.add(new Property("CLIENT_NET_PORT", "" + clientNetPort));
+ clusterProperties.add(new Property("CLUSTER_NET_PORT", "" + clusterNetPort));
+ clusterProperties.add(new Property("HTTP_PORT", "" + httpPort));
cluster.setEnv(new Env(clusterProperties));
}
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
index c2d6c04..7330c67 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/events/PatternCreator.java
@@ -16,8 +16,10 @@
import java.io.File;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import edu.uci.ics.asterix.event.driver.EventDriver;
@@ -183,7 +185,7 @@
txnLogDir = node.getTxnLogDir() == null ? instance.getCluster().getTxnLogDir() : node.getTxnLogDir();
store = node.getStore() == null ? cluster.getStore() : node.getStore();
pargs = workingDir + " " + instance.getName() + " " + iodevices + " " + store + " "
- + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + txnLogDir + " " + backupId + " " + backupDir
+ + BackupCommand.ASTERIX_ROOT_METADATA_DIR + " " + txnLogDir + " " + backupId + " " + backupDir
+ " " + "local" + " " + node.getId();
Event event = new Event("backup", nodeid, pargs);
patternList.add(new Pattern(null, 1, null, event));
@@ -479,17 +481,31 @@
return new Pattern(null, 1, null, event);
}
- public Patterns getTransferLogPattern(String asterixInstanceName, Cluster cluster, String outputDir) {
+ public Patterns getGenerateLogPattern(String asterixInstanceName, Cluster cluster, String outputDir) {
List<Pattern> patternList = new ArrayList<Pattern>();
+ Map<String,String> nodeLogs = new HashMap<String,String>();
+
String username = cluster.getUsername() == null ? System.getProperty("user.name") : cluster.getUsername();
- String destHost = cluster.getMasterNode().getClusterIp();
+ String srcHost = cluster.getMasterNode().getClientIp();
+ Nodeid nodeid = new Nodeid(new Value(null, EventDriver.CLIENT_NODE.getId()));
+ String srcDir = cluster.getMasterNode().getLogDir() == null ? cluster.getLogDir() : cluster.getMasterNode()
+ .getLogDir();
+ String destDir = outputDir + File.separator + "cc";
+ String pargs = username + " " + srcHost + " " + srcDir + " " + destDir;
+ Event event = new Event("directory_copy", nodeid, pargs);
+ Pattern p = new Pattern(null, 1, null, event);
+ patternList.add(p);
+ nodeLogs.put(cluster.getMasterNode().getClusterIp(),srcDir);
for (Node node : cluster.getNode()) {
- Nodeid nodeid = new Nodeid(new Value(null, node.getId()));
- String srcDir = node.getLogDir();
- String destDir = outputDir + File.separator + node.getId();
- String pargs = username + " " + srcDir + " " + destHost + " " + destDir;
- Event event = new Event("directory_transfer", nodeid, pargs);
- Pattern p = new Pattern(null, 1, null, event);
+ srcHost = node.getClusterIp();
+ srcDir = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir();
+ if(nodeLogs.get(node.getClusterIp()) != null && nodeLogs.get(node.getClusterIp()).equals(srcDir)){
+ continue;
+ }
+ destDir = outputDir + File.separator + node.getId();
+ pargs = username + " " + srcHost + " " + srcDir + " " + destDir;
+ event = new Event("directory_copy", nodeid, pargs);
+ p = new Pattern(null, 1, null, event);
patternList.add(p);
}
Patterns patterns = new Patterns(patternList);
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/EventList.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/EventList.java
index b605889..33f489e 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/EventList.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/EventList.java
@@ -28,6 +28,7 @@
FILE_TRANSFER,
FILE_CREATE,
DIRECTORY_TRANSFER,
+ DIRECTORY_COPY,
NODE_INFO
}
}
diff --git a/asterix-installer/src/main/resources/clusters/local/local.xml b/asterix-installer/src/main/resources/clusters/local/local.xml
index ef06822..3f739de 100644
--- a/asterix-installer/src/main/resources/clusters/local/local.xml
+++ b/asterix-installer/src/main/resources/clusters/local/local.xml
@@ -1,24 +1,26 @@
<cluster xmlns="cluster">
- <name>local</name>
- <working_dir>
- <dir>/tmp/asterix-installer</dir>
- <NFS>true</NFS>
- </working_dir>
- <log_dir>/tmp/asterix/logs</log_dir>
- <txn_log_dir>/tmp/asterix/logs</txn_log_dir>
- <cluster_port>1099</cluster_port>
- <client_port>1098</client_port>
- <http_port>8888</http_port>
- <iodevices>/tmp</iodevices>
- <store>asterix/storage</store>
- <java_home></java_home>
- <master_node>
- <id>master</id>
- <client_ip>127.0.0.1</client_ip>
- <cluster_ip>127.0.0.1</cluster_ip>
- </master_node>
- <node>
- <id>node1</id>
- <cluster_ip>127.0.0.1</cluster_ip>
- </node>
+ <name>local</name>
+ <working_dir>
+ <dir>/tmp/asterix-installer</dir>
+ <NFS>true</NFS>
+ </working_dir>
+ <log_dir>/tmp/asterix/logs</log_dir>
+ <txn_log_dir>/tmp/asterix/logs</txn_log_dir>
+ <ports>
+ <cluster_port>1099</cluster_port>
+ <client_port>1098</client_port>
+ <http_port>8888</http_port>
+ </ports>
+ <iodevices>/tmp</iodevices>
+ <store>asterix/storage</store>
+ <java_home></java_home>
+ <master_node>
+ <id>master</id>
+ <client_ip>127.0.0.1</client_ip>
+ <cluster_ip>127.0.0.1</cluster_ip>
+ </master_node>
+ <node>
+ <id>node1</id>
+ <cluster_ip>127.0.0.1</cluster_ip>
+ </node>
</cluster>
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index a8f129a..d99fa8b 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -64,7 +64,8 @@
}
private String prepareRootMetaDataFileName(String mountPoint, String nodeId, int ioDeviceId) {
- return mountPoint + ROOT_METADATA_DIRECTORY + "_" + nodeId + "_" + "iodevice" + ioDeviceId;
+ return mountPoint + File.separator + ROOT_METADATA_DIRECTORY + File.separator + ROOT_METADATA_DIRECTORY + "_"
+ + nodeId + "_" + "iodevice" + ioDeviceId;
}
public void initialize(String nodeId, String rootDir, boolean isNewUniverse, ResourceIdFactory resourceIdFactory)
@@ -82,7 +83,13 @@
File rootMetadataDir = new File(prepareRootMetaDataFileName(mountPoints[i], nodeId, i));
if (!rootMetadataDir.exists()) {
- rootMetadataDir.mkdir();
+ boolean success = rootMetadataDir.mkdirs();
+ if (!success) {
+ if (LOGGER.isLoggable(Level.SEVERE)) {
+ LOGGER.severe("Unable to create root metadata directory"
+ + rootMetadataDir.getAbsolutePath());
+ }
+ }
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("created the root-metadata-file's directory: " + rootMetadataDir.getAbsolutePath());
}