Add AWS automation scripts to asterix-server.
- Allows users to customize an AWS-based instance by editing
conf/aws_settings.yml and conf/instance_settings.yml.
- Starts an AWS cluster, install JDK and AsterixDB automatically.
- Let the default value of storage.metadata.memorycomponent.numpages
adjust to the available JVM heap size.
Change-Id: If4061501e3561a649c3a2bb3068dc257f03c092d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1475
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
index 384d759..42fb7c3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.adm
@@ -104,7 +104,7 @@
"storage.memorycomponent.numcomponents" : 2,
"storage.memorycomponent.numpages" : 8,
"storage.memorycomponent.pagesize" : 131072,
- "storage.metadata.memorycomponent.numpages" : 256,
+ "storage.metadata.memorycomponent.numpages" : 85,
"transaction.log.dirs" : {
"asterix_nc1" : "target/txnLogDir/asterix_nc1",
"asterix_nc2" : "target/txnLogDir/asterix_nc2"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
index 2476197..75c4d3e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.adm
@@ -104,7 +104,7 @@
"storage.memorycomponent.numcomponents" : 2,
"storage.memorycomponent.numpages" : 8,
"storage.memorycomponent.pagesize" : 131072,
- "storage.metadata.memorycomponent.numpages" : 256,
+ "storage.metadata.memorycomponent.numpages" : 85,
"transaction.log.dirs" : {
"asterix_nc1" : "target/txnLogDir/asterix_nc1",
"asterix_nc2" : "target/txnLogDir/asterix_nc2"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
index 5d124e7..76219aa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.adm
@@ -104,7 +104,7 @@
"storage.memorycomponent.numcomponents" : 2,
"storage.memorycomponent.numpages" : 8,
"storage.memorycomponent.pagesize" : 131072,
- "storage.metadata.memorycomponent.numpages" : 256,
+ "storage.metadata.memorycomponent.numpages" : 85,
"transaction.log.dirs" : {
"asterix_nc1" : "target/txnLogDir/asterix_nc1",
"asterix_nc2" : "target/txnLogDir/asterix_nc2"
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index faacc64..68ad80c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -18,11 +18,11 @@
*/
package org.apache.asterix.common.config;
+import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
+
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.util.StorageUtil;
-import static org.apache.hyracks.util.StorageUtil.StorageUnit.KILOBYTE;
-
public class StorageProperties extends AbstractProperties {
private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize";
@@ -40,7 +40,6 @@
private static final String STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY =
"storage.metadata.memorycomponent.numpages";
- private static final int STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 256; // ... so 32MB components
private static final String STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY = "storage.memorycomponent.numcomponents";
private static final int STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_DEFAULT = 2; // 2 components
@@ -53,6 +52,7 @@
private final long storageBufferCacheSizeDefault;
private final int storageMemoryComponentNumPages;
+ private final int storageMetadataMemoryComponentNumPages;
private final long storageMemorycomponentGlobalbudgetDefault;
public StorageProperties(PropertiesAccessor accessor) {
@@ -68,6 +68,10 @@
// for a dataset, including data and indexes.
storageMemoryComponentNumPages = (int) (storageMemorycomponentGlobalbudgetDefault
/ (16 * getMemoryComponentPageSize()));
+ // By default, uses the min of 1/64 of the storageMemorycomponentGlobalbudgetDefault and 256 pages
+ // for the write buffer budget for a metadata dataset, including data and indexes.
+ storageMetadataMemoryComponentNumPages = Math
+ .min((int) (storageMemorycomponentGlobalbudgetDefault / (64 * getMemoryComponentPageSize())), 256);
}
@PropertyKey(STORAGE_BUFFERCACHE_PAGESIZE_KEY)
@@ -107,8 +111,7 @@
@PropertyKey(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY)
public int getMetadataMemoryComponentNumPages() {
return accessor.getProperty(STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_KEY,
- STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES_DEFAULT,
- PropertyInterpreters.getIntegerPropertyInterpreter());
+ storageMetadataMemoryComponentNumPages, PropertyInterpreters.getIntegerPropertyInterpreter());
}
@PropertyKey(STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS_KEY)
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index 3454428..f91ed64 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -493,5 +493,9 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
index b968ae1..32a0180 100644
--- a/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
+++ b/asterixdb/asterix-server/src/main/assembly/binary-assembly.xml
@@ -52,6 +52,23 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <excludes>
+ <exclude>**/*.sh</exclude>
+ </excludes>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/aws</directory>
+ <outputDirectory>aws</outputDirectory>
+ <includes>
+ <include>**/*.sh</include>
+ </includes>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>target/appassembler/repo</directory>
<outputDirectory>repo</outputDirectory>
</fileSet>
diff --git a/asterixdb/asterix-server/src/main/aws/README b/asterixdb/asterix-server/src/main/aws/README
new file mode 100644
index 0000000..ebd8fee
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/README
@@ -0,0 +1,54 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+To start an AWS cluster, you need to do the following steps:
+
+1. Create an AWS account and an IAM user.
+ Sets up a security group that you'd like to use for your AWS cluster. The security group should at least allow
+ all TCP connection from anywhere.
+
+2. Retrieve your AWS key pair name and fill that after "keypair:" in conf/aws_settings.yml;
+ retrieve your AWS "access key ID" and fill that after "access_key_id:" in conf/aws_settings.yml;
+ retrieve your AWS "secret access key" and fill that after "secret_access_key:" in conf/aws_settings.yml.
+ Note that you can only read or download "access key ID" and "secret access key" once from your AWS console.
+ If you forget them, you have to create new keys again and delete the old ones.
+
+3. Customize other settings in conf/aws_settings.yml and conf/instance_settings.yml to whatever you want.
+ Note that you have to make sure that the security group name (e.g., "group: default") matches the one you setup
+ in step 1.
+
+4. Configure your ssh setting by editing ~/.ssh/config and adding the following entry:
+ Host *.amazonaws.com
+ IdentityFile <path_of_private_key>
+ Note that <path_of_private_key> should be replaced by the path to the file that stores the private key for the key
+ pair that you uploaded to AWS and used in step 2. For example:
+ Host *.amazonaws.com
+ IdentityFile ~/.ssh/id_rsa
+
+5. Install ansible and boto3:
+ ansible: http://docs.ansible.com/ansible/intro_installation.html
+ boto3: pip install boto3
+
+6. Launch your cluster instance on AWS:
+ bin/start.sh
+ Now you can use the AWS-based instance.
+
+7. Terminate the cluster instance on AWS:
+ bin/stop.sh
+ Note that it will destroy the instance and terminate all ec2 instances that you launched in step 6.
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
new file mode 100644
index 0000000..3deab82
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_start.yml
@@ -0,0 +1,48 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Start AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Launch all instances
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: "{{ count }}"
+ count_tag:
+ Name: "{{ tag }}"
+ instance_tags:
+ Name: "{{ tag }}"
+ register: ec2
+ - name: Create local temporary directory
+ file:
+ path: /tmp/asterixdb
+ state: directory
+ mode: 0755
+ - name: Output the information of all nodes
+ local_action: copy content="{{ ec2|to_json }}" dest=/tmp/asterixdb/nodes
+
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
new file mode 100644
index 0000000..e1bdc86
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/aws_stop.yml
@@ -0,0 +1,35 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Stop AWS cluster
+ hosts: localhost
+ gather_facts: false
+ tasks:
+ - include_vars: ../conf/aws_settings.yml
+ - name: Stop all instance
+ ec2:
+ key_name: "{{ keypair }}"
+ group: "{{ group }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ wait: true
+ region: "{{ region }}"
+ aws_access_key: "{{ access_key_id }}"
+ aws_secret_key: "{{ secret_access_key }}"
+ exact_count: 0
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml b/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
new file mode 100644
index 0000000..bb69b68
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/customize_deployment.yml
@@ -0,0 +1,28 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- name: Ensure the working directory exists
+ file:
+ path: "{{ binarydir }}"
+ state: directory
+
+- name: Unzip binary
+ unarchive:
+ src: "{{ lookup('pipe', 'ls -1 ' + srcpattern) }}"
+ dest: "{{ binarydir }}"
diff --git a/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
new file mode 100644
index 0000000..743b70e
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/ansible/instance_start.yml
@@ -0,0 +1,63 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+- hosts: ncs
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Download JDK
+ shell: "wget -q --tries=5 --no-cookies --no-check-certificate --header \
+ \"Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie\" \
+ \"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm\""
+
+ - name: Install JDK
+ shell: sudo yum -y localinstall jdk-8u121-linux-x64.rpm
+
+ - include: customize_deployment.yml
+
+ - name: Ensure the log directory exists
+ file:
+ path: "{{ binarydir }}/logs"
+ state: directory
+
+ - name: Ensure the io device directory exit
+ file:
+ path: "{{ basedir }}/iodevice"
+ state: directory
+
+ - name: Start NC Service
+ shell: nohup "{{ ncservice }}" &> "{{ binarydir }}/logs/ncservice.log" &
+ async: 10
+ poll: 0
+
+- hosts: cc
+ tasks:
+ - include_vars: ../conf/instance_settings.yml
+ - name: Copy cluster config to CC
+ copy:
+ src: /tmp/asterixdb/cc.conf
+ dest: "{{ basedir }}/cc.conf"
+
+ - name: Update cluster config
+ shell: find -P "{{ basedir }}/cc.conf"|xargs perl -pi -e 's|command=asterixnc|command={{ ncbin }}|g'
+
+ - name: Start CC
+ shell: nohup "{{ cc }}" -config-file "{{ basedir }}/cc.conf" &> "{{ binarydir }}/logs/cc.log" &
+ async: 10
+ poll: 0
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/start.sh b/asterixdb/asterix-server/src/main/aws/bin/start.sh
new file mode 100755
index 0000000..3779952
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/start.sh
@@ -0,0 +1,38 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# Starts an AWS cluster.
+ansible-playbook -i "localhost," ansible/aws_start.yml
+
+# Generates an Ansible inventory file and an AsterixDB configuration file.
+temp=/tmp/asterixdb
+inventory=$temp/inventory
+conf=$temp/cc.conf
+java -cp "../repo/*" org.apache.asterixdb.aws.ConfigGenerator $temp/nodes $inventory $conf
+
+# Waits a while so that all instances are up and running.
+# TODO(yingyi) pull the "status check" field of each instance.
+sleep 90
+
+# Installs asterixdb on all AWS instances.
+export ANSIBLE_HOST_KEY_CHECKING=false
+ansible-playbook -i $inventory ansible/instance_start.yml
+
diff --git a/asterixdb/asterix-server/src/main/aws/bin/stop.sh b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
new file mode 100755
index 0000000..fd2c85b
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/bin/stop.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -x
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+
+# Terminates an AWS cluster
+ansible-playbook -i "localhost," ansible/aws_stop.yml
diff --git a/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
new file mode 100644
index 0000000..06a7d27
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/aws_settings.yml
@@ -0,0 +1,46 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The OS image id for ec2 instances.
+image: ami-76fa4116
+
+# The data center region for ec2 instances.
+region: us-west-2
+
+# The tag for each ec2 machine.
+tag: scale_test
+
+# The name of a security group that appears in your AWS console.
+group: default
+
+# The name of a key pair that appears in your AWS console.
+keypair: <to be filled>
+
+# The AWS access key id for your IAM user.
+access_key_id: <to be filled>
+
+# The AWS secrety key for your IAM user.
+secret_access_key: <to be filled>
+
+# The AWS instance type. A full list of available types are listed at:
+# https://aws.amazon.com/ec2/instance-types/
+instance_type: t2.micro
+
+# The number of ec2 instances that construct a cluster.
+count: 2
diff --git a/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
new file mode 100644
index 0000000..de83b7c
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/aws/conf/instance_settings.yml
@@ -0,0 +1,48 @@
+# ------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ------------------------------------------------------------
+
+# The name of the product (or extension) being used.
+product: asterixdb
+
+# The server binary zip.
+binary: asterix-server-*-binary-assembly.zip
+
+# The script that starts a nc service.
+ncsbin: "asterixncservice"
+
+# The script that starts a nc.
+ncbin: "asterixnc"
+
+# The script that starts a cc.
+ccbin: "asterixcc"
+
+# The parent directory for the working directory.
+basedir: /home/ec2-user
+
+# The working directory.
+binarydir: "{{ basedir }}/{{ product }}"
+
+# The pattern for retrieving the sever binary zip from the current build.
+srcpattern: "../../../{{ binary }}"
+
+# The nc service command (script).
+ncservice: "{{ binarydir}}/bin/{{ ncsbin }}"
+
+# The cc service command (script).
+cc: "{{ binarydir}}/bin/{{ ccbin }}"
diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
new file mode 100644
index 0000000..fb45643
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/AwsNode.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * The useful information for an AWS node.
+ */
+public class AwsNode {
+
+ private final String privateIp;
+ private final String dnsName;
+
+ public AwsNode(ObjectNode node) {
+ this.privateIp = node.get("private_ip").asText();
+ this.dnsName = node.get("dns_name").asText();
+ }
+
+ public String getPrivateIp() {
+ return privateIp;
+ }
+
+ public String getDnsName() {
+ return dnsName;
+ }
+}
diff --git a/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
new file mode 100644
index 0000000..25895e8
--- /dev/null
+++ b/asterixdb/asterix-server/src/main/java/org/apache/asterixdb/aws/ConfigGenerator.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterixdb.aws;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * This class is the binary that automatically generates :
+ * 1. an Ansible inventory file for the AWS cluster;
+ * 2. an AsterixDB configuration file for the AWS cluster
+ * from a JSON description file returned by the Ansible AWS startup script.
+ */
+public class ConfigGenerator {
+
+ private ConfigGenerator() {
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 3) {
+ System.err.println("The usage of ConfigGenerator: ");
+ System.err.println("<input node json file> <output inventory file> <output config file>");
+ System.exit(0);
+ }
+ String source = args[0];
+ String inventory = args[1];
+ String config = args[2];
+
+ // Read Json file data to String
+ byte[] jsonData = Files.readAllBytes(Paths.get(source));
+
+ // Get a list of cluster nodes
+ ObjectMapper objectMapper = new ObjectMapper();
+ ObjectNode root = (ObjectNode) objectMapper.readTree(jsonData);
+ ArrayNode nodes = (ArrayNode) root.get("tagged_instances");
+ Iterator<JsonNode> nodeIterator = nodes.iterator();
+ List<AwsNode> cluster = new ArrayList<>();
+ while (nodeIterator.hasNext()) {
+ ObjectNode node = (ObjectNode) nodeIterator.next();
+ cluster.add(new AwsNode(node));
+ }
+
+ if (cluster.isEmpty()) {
+ return;
+ }
+
+ // Generates inventory file.
+ generateInventoryFile(cluster, inventory);
+
+ // Generates asterixdb config
+ generateConfig(cluster, config);
+ }
+
+ private static void generateInventoryFile(List<AwsNode> cluster, String inventoryPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(inventoryPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getDnsName();
+ writer.println(masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[ncs]");
+ writer.println(masterIp); // There is a NC that co-locates with CC.
+ while (nodeIterator.hasNext()) {
+ writer.println(nodeIterator.next().getDnsName());
+ }
+ writer.println();
+
+ // Prints the user
+ writer.println("[all:vars]");
+ writer.println("ansible_ssh_user=ec2-user");
+ }
+ }
+
+ private static void generateConfig(List<AwsNode> cluster, String configPath) throws IOException {
+ Iterator<AwsNode> nodeIterator = cluster.iterator();
+ try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(configPath)))) {
+ // Prints the cc section
+ writer.println("[cc]");
+ String masterIp = nodeIterator.next().getPrivateIp();
+ writer.println("cluster.address=" + masterIp);
+ writer.println();
+
+ // Prints the nc section
+ writer.println("[nc/1]");
+ writer.println("address=" + masterIp); // There is a NC that co-locates with CC.
+ int ncCounter = 2;
+ while (nodeIterator.hasNext()) {
+ writer.println("[nc/" + ncCounter++ + "]");
+ writer.println("address=" + nodeIterator.next().getPrivateIp());
+ writer.println();
+ }
+
+ // Prints the nc parameter section.
+ writer.println("[nc]");
+ String rootDirectory = "/home/ec2-user/";
+ writer.println("txnlogdir=" + rootDirectory + "txnlog");
+
+ // TODO(yingyi): figure out how to get device mapping for SSD-based instances.
+ writer.println("iodevices=" + rootDirectory + "iodevice");
+ writer.println("command=asterixnc");
+ writer.println();
+ }
+ }
+}