Basis for Cluster integration testing
This branch adds cluster testing via Vagrant.
Requires my branch of the vagrant-maven plugin to work,
which can be sourced here:
https://github.com/parshimers/vagrant-maven-plugin
It is enabled with -DclusterTest=true in mvn verify.
A virtualized cluster with 4 nodes is started, and
then Asterix is started via managix on this cluster,
and then stopped.
Change-Id: I7e3cdcd4162ada19ee1e15f532be7447b4f34367
Reviewed-on: http://fulliautomatix.ics.uci.edu:8443/31
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <imaxon@uci.edu>
Reviewed-by: Zachary Heilbron <zheilbron@gmail.com>
Reviewed-by: Till Westmann <westmann@gmail.com>
Reviewed-by: Chris Hillery <ceej@lambda.nu>
diff --git a/asterix-installer/pom.xml b/asterix-installer/pom.xml
index 96316bb..ad25bed 100644
--- a/asterix-installer/pom.xml
+++ b/asterix-installer/pom.xml
@@ -1,12 +1,17 @@
-<!-- ! Copyright 2009-2013 by The Regents of the University of California
- ! Licensed under the Apache License, Version 2.0 (the "License"); ! you may
- not use this file except in compliance with the License. ! you may obtain
- a copy of the License from ! ! http://www.apache.org/licenses/LICENSE-2.0
- ! ! Unless required by applicable law or agreed to in writing, software !
- distributed under the License is distributed on an "AS IS" BASIS, ! WITHOUT
- WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ! See the
- License for the specific language governing permissions and ! limitations
- under the License. ! -->
+<!--
+ ! Copyright 2009-2013 by The Regents of the University of California
+ ! Licensed under the Apache License, Version 2.0 (the "License");
+ ! you may not use this file except in compliance with the License.
+ ! you may obtain a copy of the License from
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing, software
+ ! distributed under the License is distributed on an "AS IS" BASIS,
+ ! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ! See the License for the specific language governing permissions and
+ ! limitations under the License.
+ !-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -18,6 +23,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<failsafe.test.excludes>**/DmlRecoveryIT.java</failsafe.test.excludes>
+ <cluster.test.excludes>**/AsterixClusterLifeCycleIT.java</cluster.test.excludes>
</properties>
<build>
@@ -109,6 +115,7 @@
<forkMode>pertest</forkMode>
<excludes>
<exclude>${failsafe.test.excludes}</exclude>
+ <exclude>${cluster.test.excludes}</exclude>
</excludes>
</configuration>
<executions>
@@ -123,6 +130,79 @@
</plugins>
</build>
+
+ <profiles>
+ <profile>
+ <id>cluster-tests</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>clusterTest</name>
+ <value>true</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>net.ju-n.maven.plugins</groupId>
+ <artifactId>vagrant-maven-plugin</artifactId>
+ <version>1.1.0-SNAPSHOT</version>
+ <executions>
+
+ <!-- Before tests: import box, start VM -->
+ <execution>
+ <id>setup-and-start-vagrant-vm</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>up</goal>
+ </goals>
+ <configuration>
+ <box>chef/centos-6.5</box>
+ <url>https://vagrantcloud.com/chef/centos-6.5</url>
+ <vagrantHome>${project.basedir}/src/test/resources/clusterts</vagrantHome>
+ </configuration>
+ </execution>
+
+ <!-- After tests: shut down, destroy VM -->
+ <execution>
+ <id>destroy-vagrant-vm</id>
+ <phase>post-integration-test</phase>
+ <goals>
+ <goal>destroy</goal>
+ </goals>
+ <configuration>
+ <box>chef/centos-6.5</box>
+ <url>https://vagrantcloud.com/chef/centos-6.5</url>
+ <vagrantHome>${project.basedir}/src/test/resources/clusterts</vagrantHome>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>2.6</version>
+ <configuration>
+ <runOrder>alphabetical</runOrder>
+ <forkMode>pertest</forkMode>
+ <excludes>
+ <exclude>${failsafe.test.excludes}</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
<dependencies>
<dependency>
<groupId>args4j</groupId>
diff --git a/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixClusterLifeCycleIT.java b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixClusterLifeCycleIT.java
new file mode 100644
index 0000000..117d7cb
--- /dev/null
+++ b/asterix-installer/src/test/java/edu/uci/ics/asterix/installer/test/AsterixClusterLifeCycleIT.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.installer.test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.logging.Logger;
+import java.util.List;
+import java.io.InputStream;
+import java.io.FilenameFilter;
+import java.lang.ProcessBuilder;
+import java.nio.charset.StandardCharsets;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.Assert;
+import org.junit.runners.Parameterized.Parameters;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import edu.uci.ics.asterix.test.aql.TestsUtils;
+import edu.uci.ics.asterix.testframework.context.TestCaseContext;
+
+public class AsterixClusterLifeCycleIT {
+
+ private static final String PATH_BASE = StringUtils.join(new String[] { "src", "test", "resources",
+ "integrationts", "lifecycle" }, File.separator);
+ private static final String CLUSTER_BASE = StringUtils.join(
+ new String[] { "src", "test", "resources", "clusterts" }, File.separator);
+ private static final String PATH_ACTUAL = "ittest" + File.separator;
+ private static String managixFolderName;
+ private static final Logger LOGGER = Logger.getLogger(AsterixClusterLifeCycleIT.class.getName());
+ private static List<TestCaseContext> testCaseCollection;
+ private static File asterixProjectDir = new File(System.getProperty("user.dir"));
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ //testcase setup
+ TestCaseContext.Builder b = new TestCaseContext.Builder();
+ testCaseCollection = b.build(new File(PATH_BASE));
+ File outdir = new File(PATH_ACTUAL);
+ outdir.mkdirs();
+
+ //vagrant setup
+ File installerTargetDir = new File(asterixProjectDir, "target");
+ System.out.println(managixFolderName);
+ managixFolderName = installerTargetDir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return new File(dir, name).isDirectory() && name.startsWith("asterix-installer")
+ && name.endsWith("binary-assembly");
+ }
+
+ })[0];
+ invoke("cp", "-r", installerTargetDir.toString() + "/" + managixFolderName, asterixProjectDir + "/"
+ + CLUSTER_BASE);
+
+ logOutput(remoteInvoke("cp -r /vagrant/" + managixFolderName + " /tmp/asterix").getInputStream());
+
+ logOutput(managixInvoke("configure").getInputStream());
+ logOutput(managixInvoke("validate").getInputStream());
+
+ Process p = managixInvoke("create -n vagrant-ssh -c /vagrant/cluster.xml");
+ String pout = processOut(p);
+ LOGGER.info(pout);
+ Assert.assertTrue(checkOutput(pout, "ACTIVE"));
+ //TODO: I should check for 'WARNING' here, but issue 764 stops this from being reliable
+ LOGGER.info("Test start active cluster instance PASSED");
+
+ Process stop = managixInvoke("stop -n vagrant-ssh");
+ Assert.assertTrue(checkOutput(stop.getInputStream(), "Stopped Asterix instance"));
+ LOGGER.info("Test stop active cluster instance PASSED");
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ Process p = managixInvoke("delete -n vagrant-ssh");
+ managixInvoke("rm -rf /vagrant/managix-working");
+ Assert.assertTrue(checkOutput(p.getInputStream(), "Deleted Asterix instance"));
+ LOGGER.info("Test delete active instance PASSED");
+ }
+
+ @Parameters
+ public static Collection<Object[]> tests() throws Exception {
+ Collection<Object[]> testArgs = new ArrayList<Object[]>();
+ return testArgs;
+ }
+
+ public static boolean checkOutput(InputStream input, String requiredSubString) {
+ //right now im just going to look at the output, which is wholly inadequate
+ //TODO: try using cURL to actually poke the instance to see if it is more alive
+ String candidate;
+ try {
+ candidate = IOUtils.toString(input, StandardCharsets.UTF_8.name());
+ } catch (IOException e) {
+ LOGGER.warning("Could not check output of subprocess");
+ return false;
+ }
+ return candidate.contains(requiredSubString);
+ }
+
+ public static boolean checkOutput(String candidate, String requiredSubString) {
+ return candidate.contains(requiredSubString);
+ }
+
+ public static String processOut(Process p) throws IOException {
+ InputStream input = p.getInputStream();
+ return IOUtils.toString(input, StandardCharsets.UTF_8.name());
+ }
+
+ public static void logOutput(InputStream input) {
+ try {
+ LOGGER.info(IOUtils.toString(input, StandardCharsets.UTF_8.name()));
+ } catch (IOException e) {
+ LOGGER.warning("Could not print output of subprocess");
+ }
+ }
+
+ private static Process invoke(String... args) throws Exception {
+ ProcessBuilder pb = new ProcessBuilder(args);
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+ return p;
+ }
+
+ private static Process remoteInvoke(String cmd) throws Exception {
+ ProcessBuilder pb = new ProcessBuilder("vagrant", "ssh", "cc", "-c", "MANAGIX_HOME=/tmp/asterix/ " + cmd);
+ File cwd = new File(asterixProjectDir.toString() + "/" + CLUSTER_BASE);
+ pb.directory(cwd);
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+ return p;
+ }
+
+ private static Process managixInvoke(String cmd) throws Exception {
+ return remoteInvoke("/tmp/asterix/bin/managix " + cmd);
+ }
+
+ @Test
+ public void StartStopActiveInstance() throws Exception {
+ //TODO: is the instance actually live?
+ //TODO: is ZK still running?
+ try {
+ Process start = managixInvoke("start -n vagrant-ssh");
+ Assert.assertTrue(checkOutput(start.getInputStream(), "ACTIVE"));
+ Process stop = managixInvoke("stop -n vagrant-ssh");
+ Assert.assertTrue(checkOutput(stop.getInputStream(), "Stopped Asterix instance"));
+ LOGGER.info("Test start/stop active cluster instance PASSED");
+ } catch (Exception e) {
+ throw new Exception("Test start/stop FAILED!", e);
+ }
+ }
+
+ public void test() throws Exception {
+ for (TestCaseContext testCaseCtx : testCaseCollection) {
+ TestsUtils.executeTest(PATH_ACTUAL, testCaseCtx, null, false);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ try {
+ setUp();
+ new AsterixClusterLifeCycleIT().test();
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.severe("TEST CASE(S) FAILED");
+ } finally {
+ tearDown();
+ }
+ }
+
+}
diff --git a/asterix-installer/src/test/resources/clusterts/Vagrantfile b/asterix-installer/src/test/resources/clusterts/Vagrantfile
new file mode 100644
index 0000000..fdd1852
--- /dev/null
+++ b/asterix-installer/src/test/resources/clusterts/Vagrantfile
@@ -0,0 +1,52 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+ config.vm.provider "virtualbox" do |v|
+ v.memory =2048
+ v.cpus = 2
+ end
+ config.vm.provision "file", source: "id_rsa", destination: "/home/vagrant/.ssh/id_rsa"
+ config.vm.provision "file", source: "known_hosts", destination: "/home/vagrant/.ssh/known_hosts"
+ config.vm.provision "file", source: "hosts", destination: "/home/vagrant/hosts"
+
+ config.vm.provision "shell", inline: "mv /home/vagrant/hosts /etc/hosts"
+ config.vm.provision "shell", privileged: false, inline: "chmod 400 /home/vagrant/.ssh/id_rsa"
+ $java_inst = <<-END
+ wget -q --no-cookies --no-check-certificate \
+ --header 'Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie' \
+ 'http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.rpm' \
+ -O /tmp/jdk.rpm;
+
+ sudo yum -y localinstall /tmp/jdk.rpm;
+ sudo yum -y install unzip;
+
+ END
+ config.vm.provision "shell", inline: $java_inst
+
+ config.vm.define "nc3" do |nc|
+ nc.vm.box = "chef/centos-6.5"
+ nc.vm.hostname = "nc3"
+ nc.vm.network "private_network", ip: "10.10.0.5"
+ end
+ config.vm.define "nc2" do |nc|
+ nc.vm.box = "chef/centos-6.5"
+ nc.vm.hostname = "nc2"
+ nc.vm.network "private_network", ip: "10.10.0.4"
+ end
+ config.vm.define "nc1" do |nc|
+ nc.vm.box = "chef/centos-6.5"
+ nc.vm.hostname = "nc1"
+ nc.vm.network "private_network", ip: "10.10.0.3"
+ end
+
+ config.vm.define "cc" do |cc|
+ cc.vm.box = "chef/centos-6.5"
+ cc.vm.hostname = "cc"
+ cc.vm.network "private_network", ip: "10.10.0.2"
+ end
+
+end
diff --git a/asterix-installer/src/test/resources/clusterts/cluster.xml b/asterix-installer/src/test/resources/clusterts/cluster.xml
new file mode 100644
index 0000000..78e8e7e
--- /dev/null
+++ b/asterix-installer/src/test/resources/clusterts/cluster.xml
@@ -0,0 +1,45 @@
+ <cluster xmlns="cluster">
+
+ <name>vagrant</name>
+
+ <username>vagrant</username>
+
+ <working_dir>
+ <dir>/vagrant/managix-working</dir>
+ <NFS>true</NFS>
+ </working_dir>
+
+ <log_dir>/home/vagrant/logs/</log_dir>
+ <txn_log_dir>/home/vagrant/tx_logs</txn_log_dir>
+
+ <iodevices>/home/vagrant</iodevices>
+
+ <store>storage</store>
+
+ <java_home>/usr/java/jdk1.7.0_51</java_home>
+
+ <master_node>
+ <id>cc</id>
+ <client_ip>10.10.0.2</client_ip>
+ <cluster_ip>10.10.0.2</cluster_ip>
+ <client_port>1098</client_port>
+ <cluster_port>1099</cluster_port>
+ <http_port>8888</http_port>
+ </master_node>
+ <node>
+ <id>nc0</id>
+ <cluster_ip>10.10.0.2</cluster_ip>
+ </node>
+ <node>
+ <id>nc1</id>
+ <cluster_ip>10.10.0.3</cluster_ip>
+ </node>
+ <node>
+ <id>nc2</id>
+ <cluster_ip>10.10.0.4</cluster_ip>
+ </node>
+ <node>
+ <id>nc3</id>
+ <cluster_ip>10.10.0.5</cluster_ip>
+ </node>
+</cluster>
diff --git a/asterix-installer/src/test/resources/clusterts/hosts b/asterix-installer/src/test/resources/clusterts/hosts
new file mode 100644
index 0000000..3a5e4bd
--- /dev/null
+++ b/asterix-installer/src/test/resources/clusterts/hosts
@@ -0,0 +1,6 @@
+127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
+10.10.0.2 cc
+10.10.0.3 nc1
+10.10.0.4 nc2
+10.10.0.5 nc3
diff --git a/asterix-installer/src/test/resources/clusterts/id_rsa b/asterix-installer/src/test/resources/clusterts/id_rsa
new file mode 100644
index 0000000..7d6a083
--- /dev/null
+++ b/asterix-installer/src/test/resources/clusterts/id_rsa
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzI
+w+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoP
+kcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2
+hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NO
+Td0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcW
+yLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQIBIwKCAQEA4iqWPJXtzZA68mKd
+ELs4jJsdyky+ewdZeNds5tjcnHU5zUYE25K+ffJED9qUWICcLZDc81TGWjHyAqD1
+Bw7XpgUwFgeUJwUlzQurAv+/ySnxiwuaGJfhFM1CaQHzfXphgVml+fZUvnJUTvzf
+TK2Lg6EdbUE9TarUlBf/xPfuEhMSlIE5keb/Zz3/LUlRg8yDqz5w+QWVJ4utnKnK
+iqwZN0mwpwU7YSyJhlT4YV1F3n4YjLswM5wJs2oqm0jssQu/BT0tyEXNDYBLEF4A
+sClaWuSJ2kjq7KhrrYXzagqhnSei9ODYFShJu8UWVec3Ihb5ZXlzO6vdNQ1J9Xsf
+4m+2ywKBgQD6qFxx/Rv9CNN96l/4rb14HKirC2o/orApiHmHDsURs5rUKDx0f9iP
+cXN7S1uePXuJRK/5hsubaOCx3Owd2u9gD6Oq0CsMkE4CUSiJcYrMANtx54cGH7Rk
+EjFZxK8xAv1ldELEyxrFqkbE4BKd8QOt414qjvTGyAK+OLD3M2QdCQKBgQDtx8pN
+CAxR7yhHbIWT1AH66+XWN8bXq7l3RO/ukeaci98JfkbkxURZhtxV/HHuvUhnPLdX
+3TwygPBYZFNo4pzVEhzWoTtnEtrFueKxyc3+LjZpuo+mBlQ6ORtfgkr9gBVphXZG
+YEzkCD3lVdl8L4cw9BVpKrJCs1c5taGjDgdInQKBgHm/fVvv96bJxc9x1tffXAcj
+3OVdUN0UgXNCSaf/3A/phbeBQe9xS+3mpc4r6qvx+iy69mNBeNZ0xOitIjpjBo2+
+dBEjSBwLk5q5tJqHmy/jKMJL4n9ROlx93XS+njxgibTvU6Fp9w+NOFD/HvxB3Tcz
+6+jJF85D5BNAG3DBMKBjAoGBAOAxZvgsKN+JuENXsST7F89Tck2iTcQIT8g5rwWC
+P9Vt74yboe2kDT531w8+egz7nAmRBKNM751U/95P9t88EDacDI/Z2OwnuFQHCPDF
+llYOUI+SpLJ6/vURRbHSnnn8a/XG+nzedGH5JGqEJNQsz+xT2axM0/W/CRknmGaJ
+kda/AoGANWrLCz708y7VYgAtW2Uf1DPOIYMdvo6fxIB5i9ZfISgcJ/bbCUkFrhoH
++vq/5CIWxCPp0f85R4qxxQ5ihxJ0YDQT9Jpx4TMss4PSavPaBH3RXow5Ohe+bYoQ
+NE5OgEXk2wVfZczCZpigBKbKZHNYcelXtTt/nP3rsCuGcM4h53s=
+-----END RSA PRIVATE KEY-----
diff --git a/asterix-installer/src/test/resources/clusterts/known_hosts b/asterix-installer/src/test/resources/clusterts/known_hosts
new file mode 100644
index 0000000..a960382
--- /dev/null
+++ b/asterix-installer/src/test/resources/clusterts/known_hosts
@@ -0,0 +1,11 @@
+127.0.0.1 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+localhost ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+10.10.0.2 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+10.10.0.1 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+10.10.0.3 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+10.10.0.4 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+10.10.0.5 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+nc1 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+cc ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+nc2 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==
+nc3 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAv+VDpocINo7EnBb6PWeSCVvzxi3kNxo8uLpM04ZZUEKXYv8OkbRc+MLiBKuOHscXSdiwP016gMVCyVQmyrggUEQ1KvDSZFHDLcSSheuEiCTyxGdc0nFSG2E6AS+fth3zjL1mIj+G0U7FNHv4EVScNCFSpBQUsJwPpX9GRHtvpEZmtPgysnz5WCCv/7lCnKQubQRVffE2V24Kg2x4anUF23xaOJU75ZAi3uIZBrOOdQJKUqTLpUIcqE6BU3U0kqx+9hkPQYj/FhWyqn2GB9N+h8u6uvbEcsVtNPxWJEdE2/4rDjQBMHy6kXC6n2MG/q4gMVyLVy7IuP4vBccEOracew==