Change "samples" to "opt" in the distribution package.
This change also does the following things:
- make AWS script more robust: (1) can be run from anywhere,
(2) eliminate the need of customize_deployment.yml.
- cleanup documentation to reflect the directory structure
change.
- fix Managix zip to be able to directories deeper than 2 levels.
Change-Id: Ie7876376d089b9ebebba3d37ac90ac346a741be3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1482
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
Tested-by: Ian Maxon <imaxon@apache.org>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/README.md b/README.md
index f6a1c2b..7f28757 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,7 @@
* Start a single-machine AsterixDB instance:
$cd asterixdb/asterix-server/target/asterix-server-*-binary-assembly/
- $./samples/local/bin/start-sample-cluster.sh
+ $./opt/local/bin/start-sample-cluster.sh
* Good to go and run queries in your browser at:
diff --git a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java
index 9b22396..b6cc339 100644
--- a/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java
+++ b/asterixdb/asterix-events/src/main/java/org/apache/asterix/event/service/AsterixEventServiceUtil.java
@@ -71,6 +71,7 @@
public static final int CLUSTER_NET_PORT_DEFAULT = 1098;
public static final int CLIENT_NET_PORT_DEFAULT = 1099;
public static final int HTTP_PORT_DEFAULT = 8888;
+ private static final String EMPTY_STRING = "";
public static final String MANAGIX_CONF_XML = "conf" + File.separator + "managix-conf.xml";
@@ -345,25 +346,23 @@
public static void zipDir(File sourceDir, File destFile) throws IOException {
FileOutputStream fos = new FileOutputStream(destFile);
ZipOutputStream zos = new ZipOutputStream(fos);
- zipDir(sourceDir, destFile, zos);
+ zipDir(EMPTY_STRING, sourceDir, destFile, zos);
zos.close();
}
- private static void zipDir(File sourceDir, final File destFile, ZipOutputStream zos) throws IOException {
- File[] dirList = sourceDir.listFiles(new FileFilter() {
- public boolean accept(File f) {
- return !f.getName().endsWith(destFile.getName());
- }
- });
+ private static void zipDir(String prefix, File sourceDir, final File destFile, ZipOutputStream zos)
+ throws IOException {
+ File[] dirList = sourceDir.listFiles(f->!f.getName().endsWith(destFile.getName()));
+ String newPrefix = prefix.equals(EMPTY_STRING) ? prefix : prefix + File.separator;
for (int i = 0; i < dirList.length; i++) {
File f = dirList[i];
if (f.isDirectory()) {
- zipDir(f, destFile, zos);
+ zipDir(newPrefix + f.getName(), f, destFile, zos);
} else {
int bytesIn = 0;
byte[] readBuffer = new byte[2156];
FileInputStream fis = new FileInputStream(f);
- ZipEntry entry = new ZipEntry(sourceDir.getName() + File.separator + f.getName());
+ ZipEntry entry = new ZipEntry(newPrefix + f.getName());
zos.putNextEntry(entry);
while ((bytesIn = fis.read(readBuffer)) != -1) {
zos.write(readBuffer, 0, bytesIn);
diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
index 32a0180..48aa168 100644
--- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
+++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
@@ -35,33 +35,16 @@
</includes>
</fileSet>
<fileSet>
- <directory>src/main/samples</directory>
- <outputDirectory>samples</outputDirectory>
+ <directory>src/main/opt</directory>
+ <outputDirectory>opt</outputDirectory>
<excludes>
<exclude>**/*.sh</exclude>
</excludes>
<filtered>true</filtered>
</fileSet>
<fileSet>
- <directory>src/main/samples</directory>
- <outputDirectory>samples</outputDirectory>
- <includes>
- <include>**/*.sh</include>
- </includes>
- <filtered>true</filtered>
- <fileMode>0755</fileMode>
- </fileSet>
- <fileSet>
- <directory>src/main/aws</directory>
- <outputDirectory>aws</outputDirectory>
- <excludes>
- <exclude>**/*.sh</exclude>
- </excludes>
- <filtered>true</filtered>
- </fileSet>
- <fileSet>
- <directory>src/main/aws</directory>
- <outputDirectory>aws</outputDirectory>
+ <directory>src/main/opt</directory>
+ <outputDirectory>opt</outputDirectory>
<includes>
<include>**/*.sh</include>
</includes>
diff --git a/asterixdb/asterix-server/src/main/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
deleted file mode 100755
index fd2c85b..0000000
--- a/asterixdb/asterix-server/src/main/aws/bin/stop.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash -x
-# ------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ------------------------------------------------------------
-
-
-# Terminates an AWS cluster
-ansible-playbook -i "localhost," ansible/aws_stop.yml
diff --git a/asterixdb/asterix-server/src/main/aws/README b/asterixdb/asterix-server/src/main/opt/aws/README
similarity index 80%
rename from asterixdb/asterix-server/src/main/aws/README
rename to asterixdb/asterix-server/src/main/opt/aws/README
index ebd8fee..b071ca2 100644
--- a/asterixdb/asterix-server/src/main/aws/README
+++ b/asterixdb/asterix-server/src/main/opt/aws/README
@@ -20,12 +20,12 @@
To start an AWS cluster, you need to do the following steps:
1. Create an AWS account and an IAM user.
- Sets up a security group that you'd like to use for your AWS cluster. The security group should at least allow
+ Set up a security group that you'd like to use for your AWS cluster. The security group should at least allow
all TCP connection from anywhere.
-2. Retrieve your AWS key pair name and fill that after "keypair:" in conf/aws_settings.yml;
- retrieve your AWS "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml;
- retrieve your AWS "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml.
+2. Retrieve your AWS EC2 key pair name and fill that after "keypair:" in conf/aws_settings.yml;
+ retrieve your AWS IAM "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml;
+ retrieve your AWS IAM "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml.
Note that you can only read or download "access key ID" and "secret access key" once from your AWS console.
If you forget them, you have to create new keys again and delete the old ones.
@@ -41,8 +41,9 @@
Host *.amazonaws.com
IdentityFile ~/.ssh/id_rsa
-5. Install ansible and boto3:
- ansible: http://docs.ansible.com/ansible/intro_installation.html
+5. Install ansible, boto, and boto3:
+ ansible: pip install ansible
+ boto: pip install boto
boto3: pip install boto3
6. Launch your cluster instance on AWS:
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml
similarity index 100%
rename from asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
rename to asterixdb/asterix-server/src/main/opt/aws/ansible/aws_start.yml
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml
similarity index 100%
rename from asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
rename to asterixdb/asterix-server/src/main/opt/aws/ansible/aws_stop.yml
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml
similarity index 94%
rename from asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
rename to asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml
index 743b70e..27554a9 100644
--- a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/ansible/instance_start.yml
@@ -28,7 +28,10 @@
- name: Install JDK
shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm
- - include: customize_deployment.yml
+ - name: Deploy binaries
+ synchronize:
+ src: "{{ dist }}"
+ dest: "{{ binarydir }}"
- name: Ensure the log directory exists
file:
@@ -49,7 +52,7 @@
tasks:
- include_vars: ../conf/instance_settings.yml
- name: Copy cluster config to CC
- copy:
+ synchronize:
src: /tmp/asterixdb/cc.conf
dest: "{{ basedir }}/cc.conf"
@@ -60,4 +63,3 @@
shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" &
async: 10
poll: 0
-
diff --git a/asterixdb/asterix-server/src/main/aws/bin/start.sh b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh
similarity index 72%
rename from asterixdb/asterix-server/src/main/aws/bin/start.sh
rename to asterixdb/asterix-server/src/main/opt/aws/bin/start.sh
index 3779952..eebdb9c 100755
--- a/asterixdb/asterix-server/src/main/aws/bin/start.sh
+++ b/asterixdb/asterix-server/src/main/opt/aws/bin/start.sh
@@ -1,4 +1,4 @@
-#!/bin/bash -x
+#!/bin/bash
# ------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,15 +18,22 @@
# under the License.
# ------------------------------------------------------------
+# Gets the absolute path so that the script can work no matter where it is invoked.
+pushd `dirname $0` > /dev/null
+SCRIPT_PATH=`pwd -P`
+popd > /dev/null
+AWS_PATH=`dirname "${SCRIPT_PATH}"`
+OPT_PATH=`dirname "${AWS_PATH}"`
+DIST_PATH=`dirname "${OPT_PATH}"`
# Starts an AWS cluster.
-ansible-playbook -i "localhost," ansible/aws_start.yml
+ansible-playbook -i "localhost," $AWS_PATH/ansible/aws_start.yml
# Generates an Ansible inventory file and an AsterixDB configuration file.
temp=/tmp/asterixdb
inventory=$temp/inventory
conf=$temp/cc.conf
-java -cp "../repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf
+java -cp "${DIST_PATH}/repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf
# Waits a while so that all instances are up and running.
# TODO(yingyi) pull the "status check" field of each instance.
@@ -34,5 +41,5 @@
# Installs asterixdb on all AWS instances.
export ANSIBLE_HOST_KEY_CHECKING=false
-ansible-playbook -i $inventory ansible/instance_start.yml
+ansible-playbook -i $inventory $AWS_PATH/ansible/instance_start.yml
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml b/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh
old mode 100644
new mode 100755
similarity index 75%
rename from asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
rename to asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh
index bb69b68..0f8bf98
--- a/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/bin/stop.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
# ------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,12 +18,11 @@
# under the License.
# ------------------------------------------------------------
-- name: Ensure the working directory exists
- file:
- path: "{{ binarydir }}"
- state: directory
+# Gets the absolute path so that the script can work no matter where it is invoked.
+pushd `dirname $0` > /dev/null
+SCRIPT_PATH=`pwd -P`
+popd > /dev/null
+AWS_PATH=`dirname "${SCRIPT_PATH}"`
-- name: Unzip binary
- unarchive:
- src: "{{ lookup('pipe', 'ls -1 ' + srcpattern) }}"
- dest: "{{ binarydir }}"
+# Terminates an AWS cluster.
+ansible-playbook -i "localhost," $AWS_PATH/ansible/aws_stop.yml
diff --git a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/aws_settings.yml
similarity index 100%
rename from asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
rename to asterixdb/asterix-server/src/main/opt/aws/conf/aws_settings.yml
diff --git a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml
similarity index 88%
rename from asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
rename to asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml
index de83b7c..d2a9ace 100644
--- a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
+++ b/asterixdb/asterix-server/src/main/opt/aws/conf/instance_settings.yml
@@ -20,9 +20,6 @@
# The name of the product (or extension) being used.
product: asterixdb
-# The server binary zip.
-binary: asterix-server-*-binary-assembly.zip
-
# The script that starts a nc service.
ncsbin: "asterixncservice"
@@ -38,8 +35,8 @@
# The working directory.
binarydir: "{{ basedir }}/{{ product }}"
-# The pattern for retrieving the sever binary zip from the current build.
-srcpattern: "../../../{{ binary }}"
+# The full binary distribution directory.
+dist: "../../../"
# The nc service command (script).
ncservice: "{{ binarydir}}/bin/{{ ncsbin }}"
diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.bat b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
similarity index 100%
rename from asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.bat
rename to asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.bat
diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
similarity index 100%
rename from asterixdb/asterix-server/src/main/samples/local/bin/start-sample-cluster.sh
rename to asterixdb/asterix-server/src/main/opt/local/bin/start-sample-cluster.sh
diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.bat b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.bat
similarity index 100%
rename from asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.bat
rename to asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.bat
diff --git a/asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.sh b/asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
similarity index 100%
rename from asterixdb/asterix-server/src/main/samples/local/bin/stop-sample-cluster.sh
rename to asterixdb/asterix-server/src/main/opt/local/bin/stop-sample-cluster.sh
diff --git a/asterixdb/asterix-server/src/main/samples/local/conf/blue.conf b/asterixdb/asterix-server/src/main/opt/local/conf/blue.conf
similarity index 100%
rename from asterixdb/asterix-server/src/main/samples/local/conf/blue.conf
rename to asterixdb/asterix-server/src/main/opt/local/conf/blue.conf
diff --git a/asterixdb/asterix-server/src/main/samples/local/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
similarity index 100%
rename from asterixdb/asterix-server/src/main/samples/local/conf/cc.conf
rename to asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java
index 0991d4f..c926ec0 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/SampleLocalClusterIT.java
@@ -54,7 +54,7 @@
// src/test/resources/NCServiceExecutionIT/cc.conf.
private static final String OUTPUT_DIR = joinPath(TARGET_DIR, "sample-local-cluster");
- private static final String LOCAL_SAMPLES_DIR = joinPath(OUTPUT_DIR, "samples", "local");
+ private static final String LOCAL_SAMPLES_DIR = joinPath(OUTPUT_DIR, "opt", "local");
@Rule
public TestRule watcher = new TestMethodTracer();