[ASTERIXDB-2017][API] Access logs, log rotation
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Common Log Format (httpd) style logs for each servlet access.
- Log rotation by default in asterix-server package, rather than
console logging with no rotation
Change-Id: I9a8d76bec308b2fdb20c33370fc9e58a154ba968
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1915
Reviewed-by: Michael Blow <mblow@apache.org>
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index 0d9f54f..6971b2b 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -46,6 +46,7 @@
heartbeat.max.misses=25
[common]
+log.dir = logs/
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index ddf1438..65dbafc 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -46,6 +46,7 @@
heartbeat.max.misses=25
[common]
+log.dir = logs/
log.level = WARN
compiler.framesize=32KB
compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index b819f24..20aa70d 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -46,6 +46,7 @@
heartbeat.max.misses=25
[common]
+log.dir = logs/
log.level = WARN
compiler.framesize=32KB
compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc4.conf b/asterixdb/asterix-app/src/main/resources/cc4.conf
index 3b7a993..5bdf8ea 100644
--- a/asterixdb/asterix-app/src/main/resources/cc4.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc4.conf
@@ -44,6 +44,7 @@
heartbeat.max.misses=25
[common]
+log.dir = logs/
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/log4j2.xml b/asterixdb/asterix-app/src/main/resources/log4j2.xml
index 1debf82..2ea8d4a 100644
--- a/asterixdb/asterix-app/src/main/resources/log4j2.xml
+++ b/asterixdb/asterix-app/src/main/resources/log4j2.xml
@@ -17,15 +17,24 @@
! under the License.
!-->
<Configuration status="WARN">
+ <CustomLevels>
+ <CustomLevel name="ACCESS" intLevel="550" />
+ </CustomLevels>
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
+ <Console name="AccessLog" target="SYSTEM_OUT">
+ <PatternLayout pattern="%m%n"/>
+ </Console>
</Appenders>
<Loggers>
<Root level="WARN">
<AppenderRef ref="Console"/>
</Root>
<Logger name="org.apache.hyracks.control.nc.service" level="INFO"/>
+ <Logger name="org.apache.hyracks.http.server.CLFLogger" level="ACCESS" additivity="false">
+ <AppenderRef ref="AccessLog"/>
+ </Logger>
</Loggers>
</Configuration>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index fc95dd4..2694408 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -46,6 +46,7 @@
heartbeat.max.misses=25
[common]
+log.dir = logs/
log.level = INFO
compiler.framesize=32KB
compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index ccb8c45..79db9ae 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -15,6 +15,7 @@
"compiler\.sortmemory" : 327680,
"compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
+ "log\.dir" : "logs/",
"log\.level" : "INFO",
"max\.wait\.active\.cluster" : 60,
"max.web.request.size" : 52428800,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 56fdf91..6e5547d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -15,6 +15,7 @@
"compiler\.sortmemory" : 327680,
"compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
+ "log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
"max.web.request.size" : 52428800,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 658f4b1..3237551 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -15,6 +15,7 @@
"compiler\.sortmemory" : 327680,
"compiler\.textsearchmemory" : 163840,
"default\.dir" : "target/io/dir/asterixdb",
+ "log\.dir" : "logs/",
"log\.level" : "WARN",
"max\.wait\.active\.cluster" : 60,
"max.web.request.size" : 52428800,
diff --git a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
index 6fc0579..bd5f403 100644
--- a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
+++ b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
@@ -37,4 +37,5 @@
address = 127.0.0.1
[common]
-log.level=INFO
+log.dir = logs/
+log.level = INFO
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
index 6862582..86d6201 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
@@ -97,5 +97,9 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java
index a188594..c4ad139 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java
@@ -30,6 +30,10 @@
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
import org.kohsuke.args4j.CmdLineException;
public class CCDriver {
@@ -44,6 +48,12 @@
ICCApplication application = getApplication(args);
application.registerConfig(configManager);
CCConfig ccConfig = new CCConfig(configManager);
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration cfg = ctx.getConfiguration();
+ CCLogConfigurationFactory logCfgFactory = new CCLogConfigurationFactory(ccConfig);
+ ConfigurationFactory.setConfigurationFactory(logCfgFactory);
+ cfg.removeLogger("Console");
+ ctx.start(logCfgFactory.getConfiguration(ctx, ConfigurationSource.NULL_SOURCE));
ClusterControllerService ccService = new ClusterControllerService(ccConfig, application);
ccService.start();
while (true) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
new file mode 100644
index 0000000..59ef913
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.cc;
+
+import org.apache.hyracks.control.common.controllers.CCConfig;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
+import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
+import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
+
+import java.net.URI;
+
+public class CCLogConfigurationFactory extends ConfigurationFactory {
+ private CCConfig config;
+
+ public CCLogConfigurationFactory(CCConfig config) {
+ this.config = config;
+ }
+
+ public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
+ String logDir = config.getLogDir();
+ builder.setStatusLevel(Level.WARN);
+ builder.setConfigurationName("RollingBuilder");
+ // create a rolling file appender
+ LayoutComponentBuilder defaultLayout = builder.newLayout("PatternLayout").addAttribute("pattern",
+ "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n");
+ ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
+ .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
+ .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
+ AppenderComponentBuilder defaultRoll =
+ builder.newAppender("default", "RollingFile").addAttribute("fileName", logDir + "cc.log")
+ .addAttribute("filePattern", logDir + "cc-%d{MM-dd-yy}.log.gz").add(defaultLayout)
+ .addComponent(triggeringPolicy);
+ builder.add(defaultRoll);
+
+ // create the new logger
+ builder.add(builder.newRootLogger(Level.INFO).add(builder.newAppenderRef("default")));
+
+ LayoutComponentBuilder accessLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m%n");
+ AppenderComponentBuilder accessRoll =
+ builder.newAppender("access", "RollingFile").addAttribute("fileName", logDir + "access.log")
+ .addAttribute("filePattern", logDir + "access-%d{MM-dd-yy}.log.gz").add(accessLayout)
+ .addComponent(triggeringPolicy);
+ builder.add(accessRoll);
+ builder.add(builder.newLogger("org.apache.hyracks.http.server.CLFLogger", Level.forName("ACCESS", 550))
+ .add(builder.newAppenderRef("access")).addAttribute("additivity", false));
+
+ return builder.build();
+ }
+
+ @Override
+ public Configuration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) {
+ return getConfiguration(loggerContext, source.toString(), null);
+ }
+
+ @Override
+ public Configuration getConfiguration(final LoggerContext loggerContext, final String name,
+ final URI configLocation) {
+ ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder();
+ return createConfiguration(builder);
+ }
+
+ @Override
+ protected String[] getSupportedTypes() {
+ return new String[] { "*" };
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
index 8ecd312..65c7ca5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
@@ -20,6 +20,7 @@
import java.io.Serializable;
import java.net.URL;
+import java.util.function.Function;
import org.apache.hyracks.api.config.IApplicationConfig;
import org.apache.hyracks.api.config.IOption;
@@ -35,21 +36,32 @@
private static final long serialVersionUID = 1L;
public enum Option implements IOption {
- CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file", null),
- CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file", null),
+ CONFIG_FILE(OptionTypes.STRING, (String) null, "Specify path to master configuration file"),
+ CONFIG_FILE_URL(OptionTypes.URL, (URL) null, "Specify URL to master configuration file"),
DEFAULT_DIR(
OptionTypes.STRING,
"Directory where files are written to by default",
- FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")),;
+ FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")),
+ LOG_DIR(
+ OptionTypes.STRING,
+ (Function<IApplicationConfig, String>) appConfig -> FileUtil
+ .joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "logs/"),
+ "The directory where logs for this node are written");
private final IOptionType type;
private final String description;
- private String defaultValue;
+ private Object defaultValue;
- Option(IOptionType type, String description, String defaultValue) {
+ <T> Option(IOptionType<T> type, T defaultValue, String description) {
this.type = type;
- this.description = description;
this.defaultValue = defaultValue;
+ this.description = description;
+ }
+
+ <T> Option(IOptionType<T> type, Function<IApplicationConfig, T> defaultValue, String description) {
+ this.type = type;
+ this.defaultValue = defaultValue;
+ this.description = description;
}
@Override
@@ -106,4 +118,8 @@
public void setConfigFileUrl(URL configFileUrl) {
configManager.set(ControllerConfig.Option.CONFIG_FILE_URL, configFileUrl);
}
+
+ public String getLogDir() {
+ return configManager.getAppConfig().getString(ControllerConfig.Option.LOG_DIR);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
index c962029..d99b5ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
@@ -88,6 +88,10 @@
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-core</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java
index a03e0ce..fdd271c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java
@@ -28,6 +28,10 @@
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
import org.kohsuke.args4j.CmdLineException;
@SuppressWarnings("InfiniteLoopStatement")
@@ -44,6 +48,12 @@
INCApplication application = getApplication(args);
application.registerConfig(configManager);
NCConfig ncConfig = new NCConfig(nodeId, configManager);
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration cfg = ctx.getConfiguration();
+ NCLogConfigurationFactory logCfgFactory = new NCLogConfigurationFactory(ncConfig);
+ ConfigurationFactory.setConfigurationFactory(logCfgFactory);
+ cfg.removeLogger("Console");
+ ctx.start(logCfgFactory.getConfiguration(ctx, ConfigurationSource.NULL_SOURCE));
final NodeControllerService ncService = new NodeControllerService(ncConfig, application);
ncService.start();
while (true) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
new file mode 100644
index 0000000..990d6c9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc;
+
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
+import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
+import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
+
+import java.net.URI;
+
+public class NCLogConfigurationFactory extends ConfigurationFactory {
+ private NCConfig config;
+
+ public NCLogConfigurationFactory(NCConfig config) {
+ this.config = config;
+ }
+
+ public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
+ String nodeId = config.getNodeId();
+ String logDir = config.getLogDir();
+ builder.setStatusLevel(Level.WARN);
+ builder.setConfigurationName("RollingBuilder");
+ // create a rolling file appender
+ LayoutComponentBuilder defaultLayout = builder.newLayout("PatternLayout").addAttribute("pattern",
+ "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n");
+ ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
+ .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
+ .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
+ AppenderComponentBuilder defaultRoll =
+ builder.newAppender("default", "RollingFile").addAttribute("fileName", logDir + "nc-" + nodeId + ".log")
+ .addAttribute("filePattern", logDir + "nc-" + nodeId + "-%d{MM-dd-yy}.log.gz")
+ .add(defaultLayout).addComponent(triggeringPolicy);
+ builder.add(defaultRoll);
+
+ // create the new logger
+ builder.add(builder.newRootLogger(Level.INFO).add(builder.newAppenderRef("default")));
+
+ LayoutComponentBuilder accessLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m%n");
+ AppenderComponentBuilder accessRoll = builder.newAppender("access", "RollingFile")
+ .addAttribute("fileName", logDir + "access-" + nodeId + ".log")
+ .addAttribute("filePattern", logDir + "access-" + nodeId + "-%d{MM-dd-yy}.log.gz").add(accessLayout)
+ .addComponent(triggeringPolicy);
+ builder.add(accessRoll);
+ builder.add(builder.newLogger("org.apache.hyracks.http.server.CLFLogger", Level.forName("ACCESS", 550))
+ .add(builder.newAppenderRef("access")).addAttribute("additivity", false));
+
+ return builder.build();
+ }
+
+ @Override
+ public Configuration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) {
+ return getConfiguration(loggerContext, source.toString(), null);
+ }
+
+ @Override
+ public Configuration getConfiguration(final LoggerContext loggerContext, final String name,
+ final URI configLocation) {
+ ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder();
+ return createConfiguration(builder);
+ }
+
+ @Override
+ protected String[] getSupportedTypes() {
+ return new String[] { "*" };
+ }
+
+ @Override
+ public String toString() {
+ return "NCLogConfiguration";
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
new file mode 100644
index 0000000..cfbcad8
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.http.server;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+import io.netty.channel.socket.nio.NioSocketChannel;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.DefaultHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+
+//Based in part on LoggingHandler from Netty
+public class CLFLogger extends ChannelDuplexHandler {
+
+ private static final Logger accessLogger = LogManager.getLogger();
+ private static final String ACCESS_LOG_LEVEL = "ACCESS";
+ private static final DateTimeFormatter DATE_TIME_FORMATTER =
+ DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withZone(ZoneId.systemDefault());
+ private StringBuilder logLineBuilder;
+
+ private String clientIp;
+ private Instant requestTime;
+ private String reqLine;
+ private int statusCode;
+ private long respSize;
+ private String userAgentRef;
+ private boolean lastChunk = false;
+
+ public CLFLogger() {
+ this.logLineBuilder = new StringBuilder();
+ respSize = 0;
+ }
+
+ @Override
+ public void channelRead(ChannelHandlerContext ctx, Object msg) {
+ if (msg instanceof HttpRequest) {
+ HttpRequest req = (HttpRequest) msg;
+ clientIp = ((NioSocketChannel) ctx.channel()).remoteAddress().getAddress().toString().substring(1);
+ requestTime = Instant.now();
+ reqLine = req.method().toString() + " " + req.getUri() + " " + req.getProtocolVersion().toString();
+ userAgentRef = headerValueOrDash("Referer", req) + " " + headerValueOrDash("User-Agent", req);
+ lastChunk = false;
+ }
+ ctx.fireChannelRead(msg);
+ }
+
+ private String headerValueOrDash(String headerKey, HttpRequest req) {
+ String value = req.headers().get(headerKey);
+ if (value == null) {
+ value = "-";
+ } else {
+ value = "\"" + value + "\"";
+ }
+ return value;
+
+ }
+
+ @Override
+ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) {
+ if (msg instanceof DefaultHttpResponse) {
+ HttpResponse resp = (DefaultHttpResponse) msg;
+ statusCode = resp.status().code();
+ if (msg instanceof DefaultFullHttpResponse) {
+ lastChunk = true;
+ respSize = resp.headers().getInt(HttpHeaderNames.CONTENT_LENGTH);
+ }
+ } else if (msg instanceof DefaultHttpContent) {
+ HttpContent content = (DefaultHttpContent) msg;
+
+ respSize += content.content().readableBytes();
+ } else if (msg instanceof LastHttpContent) {
+ lastChunk = true;
+ }
+
+ ctx.write(msg, promise);
+ }
+
+ @Override
+ public void flush(ChannelHandlerContext ctx) throws Exception {
+ if (lastChunk) {
+ printAndPrepare();
+ lastChunk = false;
+ }
+ ctx.flush();
+ }
+
+ private void printAndPrepare() {
+ logLineBuilder.append(clientIp);
+ //identd value - not relevant here
+ logLineBuilder.append(" - ");
+ //no http auth or any auth either for that matter
+ logLineBuilder.append(" - [");
+ logLineBuilder.append(DATE_TIME_FORMATTER.format(requestTime));
+ logLineBuilder.append("] \"");
+ logLineBuilder.append(reqLine);
+ logLineBuilder.append("\"");
+ logLineBuilder.append(" ").append(statusCode);
+ logLineBuilder.append(" ").append(respSize);
+ logLineBuilder.append(" ").append(userAgentRef);
+ accessLogger.log(Level.forName(ACCESS_LOG_LEVEL, 550), logLineBuilder.toString());
+ respSize = 0;
+ logLineBuilder.setLength(0);
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 42b47fb..343faa4 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -28,6 +28,9 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+import io.netty.util.internal.logging.Log4J2LoggerFactory;
+import io.netty.util.internal.logging.Log4JLoggerFactory;
import org.apache.hyracks.http.api.IChannelClosedHandler;
import org.apache.hyracks.http.api.IServlet;
import org.apache.hyracks.util.MXHelper;
@@ -49,6 +52,8 @@
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+import io.netty.util.internal.logging.Log4J2LoggerFactory;
public class HttpServer {
// Constants
@@ -81,6 +86,10 @@
private Throwable cause;
private HttpServerConfig config;
+ static {
+ InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
+ }
+
public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) {
this(bossGroup, workerGroup, port, config, null);
}
@@ -91,6 +100,7 @@
this.workerGroup = workerGroup;
this.port = port;
this.closedHandler = closeHandler;
+ InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
this.config = config;
ctx = new ConcurrentHashMap<>();
servlets = new ArrayList<>();
@@ -100,11 +110,12 @@
runnable -> new Thread(runnable, "HttpExecutor(port:" + port + ")-" + threadId.getAndIncrement()));
long directMemoryBudget = numExecutorThreads * (long) HIGH_WRITE_BUFFER_WATER_MARK
+ numExecutorThreads * config.getMaxResponseChunkSize();
- LOGGER.log(Level.INFO, "The output direct memory budget for this server is " + directMemoryBudget + " bytes");
+ LOGGER.log(Level.DEBUG,
+ "The output direct memory budget for this server " + "is " + directMemoryBudget + " bytes");
long inputBudgetEstimate =
(long) config.getMaxRequestInitialLineLength() * (config.getRequestQueueSize() + numExecutorThreads);
inputBudgetEstimate = inputBudgetEstimate * 2;
- LOGGER.log(Level.INFO,
+ LOGGER.log(Level.DEBUG,
"The \"estimated\" input direct memory budget for this server is " + inputBudgetEstimate + " bytes");
// Having multiple arenas, memory fragments, and local thread cached buffers
// can cause the input memory usage to exceed estimate and custom buffer allocator must be used to avoid this
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
index e10da64..eafee2d 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
@@ -21,8 +21,11 @@
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.http.HttpObjectAggregator;
import io.netty.handler.codec.http.HttpRequestDecoder;
import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.logging.LogLevel;
+import org.apache.logging.log4j.Logger;
public class HttpServerInitializer extends ChannelInitializer<SocketChannel> {
@@ -49,6 +52,7 @@
p.addLast(new HttpRequestCapacityController(server));
p.addLast(new HttpRequestDecoder(maxRequestInitialLineLength, maxRequestHeaderSize, maxRequestChunkSize));
p.addLast(new HttpResponseEncoder());
+ p.addLast(new CLFLogger());
p.addLast(new HttpRequestAggregator(maxRequestSize));
p.addLast(server.createHttpHandler(maxResponseChunkSize));
}