[ASTERIXDB-2512][TEST] Adds SpiderSilk Integration
- user model changes: no
- storage format changes: no
- interface changes: no
details:
- Adds minimal SpiderSilk integration by adding the asterix-spidersilk module, reuqired maven
dependencies, sample Dockerfile for node's containers, simple config file for 3 node AsterixDB
cluster, a TestUtility to make working with SpiderSilk easier and a sample test case that uses
all of these together.
Change-Id: Ifd64aad26bd5cd3b6040fa53e87ae0d0d0c0a562
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3053
Reviewed-by: Ian Maxon <imaxon@apache.org>
Tested-by: Ian Maxon <imaxon@apache.org>
diff --git a/.gitignore b/.gitignore
index 569eb3d..1dcc3c2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,3 +28,4 @@
.m2*
ß
+.SpiderSilkWorkingDirectory
diff --git a/asterixdb/asterix-spidersilk/config/cc.conf b/asterixdb/asterix-spidersilk/config/cc.conf
new file mode 100644
index 0000000..3212003
--- /dev/null
+++ b/asterixdb/asterix-spidersilk/config/cc.conf
@@ -0,0 +1,35 @@
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing,
+; software distributed under the License is distributed on an
+; "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+; KIND, either express or implied. See the License for the
+; specific language governing permissions and limitations
+; under the License.
+
+[nc/nc1]
+txn.log.dir=/data/txnlog
+core.dump.dir=/data/coredump
+iodevices=/data
+address=nc1
+
+[nc/nc2]
+txn.log.dir=/data/txnlog
+core.dump.dir=/data/coredump
+iodevices=/data
+address=nc2
+
+[nc]
+app.class=org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint
+command=asterixnc
+
+[cc]
+address=cc
\ No newline at end of file
diff --git a/asterixdb/asterix-spidersilk/docker/Dockerfile b/asterixdb/asterix-spidersilk/docker/Dockerfile
new file mode 100644
index 0000000..3f7f6bb
--- /dev/null
+++ b/asterixdb/asterix-spidersilk/docker/Dockerfile
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM java:8-jre
+
+RUN apt update && apt install -y iptables
\ No newline at end of file
diff --git a/asterixdb/asterix-spidersilk/pom.xml b/asterixdb/asterix-spidersilk/pom.xml
new file mode 100644
index 0000000..a6bb2af
--- /dev/null
+++ b/asterixdb/asterix-spidersilk/pom.xml
@@ -0,0 +1,125 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements. See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership. The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License. You may obtain a copy of the License at
+ !
+ ! http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied. See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>asterix-spidersilk</artifactId>
+ <name>asterix-spidersilk</name>
+
+ <parent>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>apache-asterixdb</artifactId>
+ <version>0.9.5-SNAPSHOT</version>
+ </parent>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ <comments>A business-friendly OSS license</comments>
+ </license>
+ </licenses>
+
+ <properties>
+ <root.dir>${basedir}/..</root.dir>
+ </properties>
+
+ <repositories>
+ <repository>
+ <id>snapshots-repo</id>
+ <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+ <releases><enabled>false</enabled></releases>
+ <snapshots><enabled>true</enabled></snapshots>
+ </repository>
+ </repositories>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>me.arminb.spidersilk</groupId>
+ <artifactId>spidersilk</artifactId>
+ <version>0.3.2-SNAPSHOT</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.25</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.9.7</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.2.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-app</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-test-framework</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-app</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-server</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ </dependencies>
+
+ <!-- While these dependencies are declared and being used, the maven dependency plugin detects them as declared and
+ not used. To make this right, it is needed to force these dependencies as used -->
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <usedDependencies>
+ <usedDependency>ch.qos.logback:logback-classic</usedDependency>
+ <usedDependency>org.apache.asterix:asterix-app</usedDependency>
+ <usedDependency>org.apache.asterix:asterix-server</usedDependency>
+ </usedDependencies>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/SampleTestIT.java b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/SampleTestIT.java
new file mode 100644
index 0000000..49f35a0
--- /dev/null
+++ b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/SampleTestIT.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.spidersilk;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import me.arminb.spidersilk.SpiderSilkRunner;
+import me.arminb.spidersilk.dsl.entities.Deployment;
+import me.arminb.spidersilk.exceptions.RuntimeEngineException;
+import me.arminb.spidersilk.execution.single_node.SingleNodeRuntimeEngine;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+
+import java.util.List;
+import java.util.Map;
+
+public class SampleTestIT {
+ private static final Logger logger = LoggerFactory.getLogger(SampleTestIT.class);
+
+ protected static SpiderSilkRunner runner;
+
+ @BeforeClass
+ public static void before() throws RuntimeEngineException {
+ Deployment deployment = TestUtil.getSimpleClusterDeployment();
+ runner = SpiderSilkRunner.run(deployment, new SingleNodeRuntimeEngine(deployment));
+ TestUtil.waitForClusterToBeUp(runner);
+ logger.info("The cluster is UP!");
+ }
+
+ @AfterClass
+ public static void after() {
+ if (runner != null) {
+ runner.stop();
+ }
+ }
+
+ @Test
+ public void sampleTest() throws Exception {
+
+ TestExecutor testExecutor = TestUtil.getTestExecutor(runner);
+ String ddl = "drop dataverse company if exists;" + "create dataverse company;" + "use company;"
+ + "create type Emp as open {" + " id : int32," + " name : string" + "};"
+ + "create dataset Employee(Emp) primary key id;";
+
+ String insertStatements = "use company;" + "insert into Employee({ \"id\":123,\"name\":\"John Doe\"});";
+
+ String query = "use company;" + "select value emp from Employee emp;";
+
+ testExecutor.executeSqlppUpdateOrDdl(ddl, TestCaseContext.OutputFormat.CLEAN_JSON);
+ logger.info("Company dataverse and employee dataset are created!");
+ testExecutor.executeSqlppUpdateOrDdl(insertStatements, TestCaseContext.OutputFormat.CLEAN_JSON);
+ logger.info("A record is inserted into employee dataset");
+ InputStream resultStream = testExecutor.executeSqlppUpdateOrDdl(query, TestCaseContext.OutputFormat.CLEAN_JSON);
+
+ ObjectMapper objectMapper = new ObjectMapper();
+ List<Map<String, String>> result = objectMapper.readValue(resultStream, List.class);
+
+ Assert.assertEquals(1, result.size());
+ Assert.assertEquals(123, result.get(0).get("id"));
+ Assert.assertEquals("John Doe", result.get(0).get("name"));
+
+ logger.info("The fetched record matches the inserted record");
+ }
+}
diff --git a/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/TestUtil.java b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/TestUtil.java
new file mode 100644
index 0000000..5d2cc02
--- /dev/null
+++ b/asterixdb/asterix-spidersilk/src/test/java/org/apache/asterix/spidersilk/TestUtil.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.spidersilk;
+
+import me.arminb.spidersilk.SpiderSilkRunner;
+import me.arminb.spidersilk.dsl.entities.Deployment;
+import me.arminb.spidersilk.dsl.entities.PortType;
+import me.arminb.spidersilk.dsl.entities.ServiceType;
+import me.arminb.spidersilk.exceptions.RuntimeEngineException;
+import org.apache.asterix.test.common.TestExecutor;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+public class TestUtil {
+ private static String mavenVersion;
+ private static String asterixHome;
+
+ public static Deployment getSimpleClusterDeployment() {
+ mavenVersion = getMavenArtifactVersion();
+ asterixHome = "/asterix/apache-asterixdb-" + mavenVersion;
+
+ return new Deployment.DeploymentBuilder("simpleClusterDeployment")
+ // Service Definitions
+ .withService("asterix")
+ .applicationPath("../asterix-server/target/asterix-server-" + mavenVersion + "-binary-assembly.zip",
+ "/asterix", false, true, false)
+ .dockerFileAddress("docker/Dockerfile", false).dockerImage("spidersilk/test-asterix")
+ .instrumentablePath(asterixHome + "/repo/asterix-server-" + mavenVersion + ".jar")
+ .libraryPath(asterixHome + "/repo/*.jar").libraryPath(asterixHome + "/lib/*.jar")
+ .logDirectory(asterixHome + "/logs").serviceType(ServiceType.JAVA).and()
+ // Node Definitions
+ .withNode("cc", "asterix").applicationPath("config", "/asterix/config")
+ .startCommand(asterixHome + "/bin/asterixcc -config-file /asterix/config/cc.conf").tcpPort(19002).and()
+ .withNode("nc1", "asterix").startCommand(asterixHome + "/bin/asterixncservice").and()
+ .withNode("nc2", "asterix").startCommand(asterixHome + "/bin/asterixncservice").and().build();
+ }
+
+ public static String getMavenArtifactVersion() {
+ Properties mavenProperties = new Properties();
+ try {
+ mavenProperties.load(new FileInputStream("../asterix-server/target/maven-archiver/pom.properties"));
+ return mavenProperties.getProperty("version");
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot open pom.properties to get the maven version");
+ }
+ }
+
+ public static void waitForClusterToBeUp(SpiderSilkRunner runner) throws RuntimeEngineException {
+ runner.runtime().runCommandInNode("cc", asterixHome + "/bin/asterixhelper wait_for_cluster");
+ }
+
+ public static TestExecutor getTestExecutor(SpiderSilkRunner runner) {
+ return new TestExecutor(runner.runtime().ip("cc"), runner.runtime().portMapping("cc", 19002, PortType.TCP));
+ }
+}
diff --git a/asterixdb/asterix-spidersilk/src/test/resources/logback.xml b/asterixdb/asterix-spidersilk/src/test/resources/logback.xml
new file mode 100644
index 0000000..ca7aa79
--- /dev/null
+++ b/asterixdb/asterix-spidersilk/src/test/resources/logback.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<configuration>
+
+ <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
+ <layout class="ch.qos.logback.classic.PatternLayout">
+ <Pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</Pattern>
+ </layout>
+ </appender>
+
+ <logger name="me.arminb" level="DEBUG"/>
+ <logger name="org.apache.asterix" level="DEBUG"/>
+
+ <root level="ERROR">
+ <appender-ref ref="Console" />
+ </root>
+</configuration>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 9bf7533..bca6b62 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -189,6 +189,7 @@
<exclude>**/*.json</exclude>
<exclude>**/*.adm</exclude>
<exclude>**/*.template</exclude>
+ <exclude>**/.SpiderSilkWorkingDirectory/**</exclude>
<exclude>asterix-installer/**</exclude> <!-- in case -DskipInstaller -->
</excludes>
</configuration>
@@ -858,6 +859,7 @@
<module>asterix-client-helper</module>
<module>asterix-license</module>
<module>asterix-geo</module>
+ <module>asterix-spidersilk</module>
</modules>
<dependencyManagement>