diff --git a/asterixdb/asterix-app/src/main/resources/cc.conf b/asterixdb/asterix-app/src/main/resources/cc.conf
index 0d9f54f..6971b2b 100644
--- a/asterixdb/asterix-app/src/main/resources/cc.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc.conf
@@ -46,6 +46,7 @@
 heartbeat.max.misses=25
 
 [common]
+log.dir = logs/
 log.level = INFO
 compiler.framesize=32KB
 compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc2.conf b/asterixdb/asterix-app/src/main/resources/cc2.conf
index ddf1438..65dbafc 100644
--- a/asterixdb/asterix-app/src/main/resources/cc2.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc2.conf
@@ -46,6 +46,7 @@
 heartbeat.max.misses=25
 
 [common]
+log.dir = logs/
 log.level = WARN
 compiler.framesize=32KB
 compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc3.conf b/asterixdb/asterix-app/src/main/resources/cc3.conf
index b819f24..20aa70d 100644
--- a/asterixdb/asterix-app/src/main/resources/cc3.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc3.conf
@@ -46,6 +46,7 @@
 heartbeat.max.misses=25
 
 [common]
+log.dir = logs/
 log.level = WARN
 compiler.framesize=32KB
 compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/cc4.conf b/asterixdb/asterix-app/src/main/resources/cc4.conf
index 3b7a993..5bdf8ea 100644
--- a/asterixdb/asterix-app/src/main/resources/cc4.conf
+++ b/asterixdb/asterix-app/src/main/resources/cc4.conf
@@ -44,6 +44,7 @@
 heartbeat.max.misses=25
 
 [common]
+log.dir = logs/
 log.level = INFO
 compiler.framesize=32KB
 compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/main/resources/log4j2.xml b/asterixdb/asterix-app/src/main/resources/log4j2.xml
index 1debf82..2ea8d4a 100644
--- a/asterixdb/asterix-app/src/main/resources/log4j2.xml
+++ b/asterixdb/asterix-app/src/main/resources/log4j2.xml
@@ -17,15 +17,24 @@
  ! under the License.
  !-->
 <Configuration status="WARN">
+   <CustomLevels>
+    <CustomLevel name="ACCESS" intLevel="550" />
+  </CustomLevels>
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
       <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
     </Console>
+    <Console name="AccessLog" target="SYSTEM_OUT">
+      <PatternLayout pattern="%m%n"/>
+    </Console>
   </Appenders>
   <Loggers>
     <Root level="WARN">
       <AppenderRef ref="Console"/>
     </Root>
     <Logger name="org.apache.hyracks.control.nc.service" level="INFO"/>
+    <Logger name="org.apache.hyracks.http.server.CLFLogger" level="ACCESS" additivity="false">
+        <AppenderRef ref="AccessLog"/>
+    </Logger>
   </Loggers>
 </Configuration>
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/cc.conf b/asterixdb/asterix-app/src/test/resources/cc.conf
index fc95dd4..2694408 100644
--- a/asterixdb/asterix-app/src/test/resources/cc.conf
+++ b/asterixdb/asterix-app/src/test/resources/cc.conf
@@ -46,6 +46,7 @@
 heartbeat.max.misses=25
 
 [common]
+log.dir = logs/
 log.level = INFO
 compiler.framesize=32KB
 compiler.sortmemory=320KB
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index ccb8c45..79db9ae 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -15,6 +15,7 @@
     "compiler\.sortmemory" : 327680,
     "compiler\.textsearchmemory" : 163840,
     "default\.dir" : "target/io/dir/asterixdb",
+    "log\.dir" : "logs/",
     "log\.level" : "INFO",
     "max\.wait\.active\.cluster" : 60,
     "max.web.request.size" : 52428800,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 56fdf91..6e5547d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -15,6 +15,7 @@
     "compiler\.sortmemory" : 327680,
     "compiler\.textsearchmemory" : 163840,
     "default\.dir" : "target/io/dir/asterixdb",
+    "log\.dir" : "logs/",
     "log\.level" : "WARN",
     "max\.wait\.active\.cluster" : 60,
     "max.web.request.size" : 52428800,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 658f4b1..3237551 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -15,6 +15,7 @@
     "compiler\.sortmemory" : 327680,
     "compiler\.textsearchmemory" : 163840,
     "default\.dir" : "target/io/dir/asterixdb",
+    "log\.dir" : "logs/",
     "log\.level" : "WARN",
     "max\.wait\.active\.cluster" : 60,
     "max.web.request.size" : 52428800,
diff --git a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
index 6fc0579..bd5f403 100644
--- a/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
+++ b/asterixdb/asterix-server/src/main/opt/local/conf/cc.conf
@@ -37,4 +37,5 @@
 address = 127.0.0.1
 
 [common]
-log.level=INFO
+log.dir = logs/
+log.level = INFO
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
index 6862582..86d6201 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/pom.xml
@@ -97,5 +97,9 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-api</artifactId>
     </dependency>
+      <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+    </dependency>
   </dependencies>
 </project>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java
index a188594..c4ad139 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCDriver.java
@@ -30,6 +30,10 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
 import org.kohsuke.args4j.CmdLineException;
 
 public class CCDriver {
@@ -44,6 +48,12 @@
             ICCApplication application = getApplication(args);
             application.registerConfig(configManager);
             CCConfig ccConfig = new CCConfig(configManager);
+            LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+            Configuration cfg = ctx.getConfiguration();
+            CCLogConfigurationFactory logCfgFactory = new CCLogConfigurationFactory(ccConfig);
+            ConfigurationFactory.setConfigurationFactory(logCfgFactory);
+            cfg.removeLogger("Console");
+            ctx.start(logCfgFactory.getConfiguration(ctx, ConfigurationSource.NULL_SOURCE));
             ClusterControllerService ccService = new ClusterControllerService(ccConfig, application);
             ccService.start();
             while (true) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
new file mode 100644
index 0000000..59ef913
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/CCLogConfigurationFactory.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.cc;
+
+import org.apache.hyracks.control.common.controllers.CCConfig;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
+import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
+import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
+
+import java.net.URI;
+
+public class CCLogConfigurationFactory extends ConfigurationFactory {
+    private CCConfig config;
+
+    public CCLogConfigurationFactory(CCConfig config) {
+        this.config = config;
+    }
+
+    public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
+        String logDir = config.getLogDir();
+        builder.setStatusLevel(Level.WARN);
+        builder.setConfigurationName("RollingBuilder");
+        // create a rolling file appender
+        LayoutComponentBuilder defaultLayout = builder.newLayout("PatternLayout").addAttribute("pattern",
+                "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n");
+        ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
+                .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
+                .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
+        AppenderComponentBuilder defaultRoll =
+                builder.newAppender("default", "RollingFile").addAttribute("fileName", logDir + "cc.log")
+                        .addAttribute("filePattern", logDir + "cc-%d{MM-dd-yy}.log.gz").add(defaultLayout)
+                        .addComponent(triggeringPolicy);
+        builder.add(defaultRoll);
+
+        // create the new logger
+        builder.add(builder.newRootLogger(Level.INFO).add(builder.newAppenderRef("default")));
+
+        LayoutComponentBuilder accessLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m%n");
+        AppenderComponentBuilder accessRoll =
+                builder.newAppender("access", "RollingFile").addAttribute("fileName", logDir + "access.log")
+                        .addAttribute("filePattern", logDir + "access-%d{MM-dd-yy}.log.gz").add(accessLayout)
+                        .addComponent(triggeringPolicy);
+        builder.add(accessRoll);
+        builder.add(builder.newLogger("org.apache.hyracks.http.server.CLFLogger", Level.forName("ACCESS", 550))
+                .add(builder.newAppenderRef("access")).addAttribute("additivity", false));
+
+        return builder.build();
+    }
+
+    @Override
+    public Configuration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) {
+        return getConfiguration(loggerContext, source.toString(), null);
+    }
+
+    @Override
+    public Configuration getConfiguration(final LoggerContext loggerContext, final String name,
+            final URI configLocation) {
+        ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder();
+        return createConfiguration(builder);
+    }
+
+    @Override
+    protected String[] getSupportedTypes() {
+        return new String[] { "*" };
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
index 8ecd312..65c7ca5 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/ControllerConfig.java
@@ -20,6 +20,7 @@
 
 import java.io.Serializable;
 import java.net.URL;
+import java.util.function.Function;
 
 import org.apache.hyracks.api.config.IApplicationConfig;
 import org.apache.hyracks.api.config.IOption;
@@ -35,21 +36,32 @@
     private static final long serialVersionUID = 1L;
 
     public enum Option implements IOption {
-        CONFIG_FILE(OptionTypes.STRING, "Specify path to master configuration file", null),
-        CONFIG_FILE_URL(OptionTypes.URL, "Specify URL to master configuration file", null),
+        CONFIG_FILE(OptionTypes.STRING, (String) null, "Specify path to master configuration file"),
+        CONFIG_FILE_URL(OptionTypes.URL, (URL) null, "Specify URL to master configuration file"),
         DEFAULT_DIR(
                 OptionTypes.STRING,
                 "Directory where files are written to by default",
-                FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")),;
+                FileUtil.joinPath(System.getProperty(ConfigurationUtil.JAVA_IO_TMPDIR), "hyracks")),
+        LOG_DIR(
+                OptionTypes.STRING,
+                (Function<IApplicationConfig, String>) appConfig -> FileUtil
+                        .joinPath(appConfig.getString(ControllerConfig.Option.DEFAULT_DIR), "logs/"),
+                "The directory where logs for this node are written");
 
         private final IOptionType type;
         private final String description;
-        private String defaultValue;
+        private Object defaultValue;
 
-        Option(IOptionType type, String description, String defaultValue) {
+        <T> Option(IOptionType<T> type, T defaultValue, String description) {
             this.type = type;
-            this.description = description;
             this.defaultValue = defaultValue;
+            this.description = description;
+        }
+
+        <T> Option(IOptionType<T> type, Function<IApplicationConfig, T> defaultValue, String description) {
+            this.type = type;
+            this.defaultValue = defaultValue;
+            this.description = description;
         }
 
         @Override
@@ -106,4 +118,8 @@
     public void setConfigFileUrl(URL configFileUrl) {
         configManager.set(ControllerConfig.Option.CONFIG_FILE_URL, configFileUrl);
     }
+
+    public String getLogDir() {
+        return configManager.getAppConfig().getString(ControllerConfig.Option.LOG_DIR);
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
index c962029..d99b5ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/pom.xml
@@ -88,6 +88,10 @@
       <artifactId>log4j-api</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+    </dependency>
+    <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java
index a03e0ce..fdd271c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCDriver.java
@@ -28,6 +28,10 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
 import org.kohsuke.args4j.CmdLineException;
 
 @SuppressWarnings("InfiniteLoopStatement")
@@ -44,6 +48,12 @@
             INCApplication application = getApplication(args);
             application.registerConfig(configManager);
             NCConfig ncConfig = new NCConfig(nodeId, configManager);
+            LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+            Configuration cfg = ctx.getConfiguration();
+            NCLogConfigurationFactory logCfgFactory = new NCLogConfigurationFactory(ncConfig);
+            ConfigurationFactory.setConfigurationFactory(logCfgFactory);
+            cfg.removeLogger("Console");
+            ctx.start(logCfgFactory.getConfiguration(ctx, ConfigurationSource.NULL_SOURCE));
             final NodeControllerService ncService = new NodeControllerService(ncConfig, application);
             ncService.start();
             while (true) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
new file mode 100644
index 0000000..990d6c9
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NCLogConfigurationFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc;
+
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.ConfigurationFactory;
+import org.apache.logging.log4j.core.config.ConfigurationSource;
+import org.apache.logging.log4j.core.config.builder.api.AppenderComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder;
+import org.apache.logging.log4j.core.config.builder.api.LayoutComponentBuilder;
+import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration;
+
+import java.net.URI;
+
+public class NCLogConfigurationFactory extends ConfigurationFactory {
+    private NCConfig config;
+
+    public NCLogConfigurationFactory(NCConfig config) {
+        this.config = config;
+    }
+
+    public Configuration createConfiguration(ConfigurationBuilder<BuiltConfiguration> builder) {
+        String nodeId = config.getNodeId();
+        String logDir = config.getLogDir();
+        builder.setStatusLevel(Level.WARN);
+        builder.setConfigurationName("RollingBuilder");
+        // create a rolling file appender
+        LayoutComponentBuilder defaultLayout = builder.newLayout("PatternLayout").addAttribute("pattern",
+                "%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n");
+        ComponentBuilder triggeringPolicy = builder.newComponent("Policies")
+                .addComponent(builder.newComponent("CronTriggeringPolicy").addAttribute("schedule", "0 0 0 * * ?"))
+                .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "50M"));
+        AppenderComponentBuilder defaultRoll =
+                builder.newAppender("default", "RollingFile").addAttribute("fileName", logDir + "nc-" + nodeId + ".log")
+                        .addAttribute("filePattern", logDir + "nc-" + nodeId + "-%d{MM-dd-yy}.log.gz")
+                        .add(defaultLayout).addComponent(triggeringPolicy);
+        builder.add(defaultRoll);
+
+        // create the new logger
+        builder.add(builder.newRootLogger(Level.INFO).add(builder.newAppenderRef("default")));
+
+        LayoutComponentBuilder accessLayout = builder.newLayout("PatternLayout").addAttribute("pattern", "%m%n");
+        AppenderComponentBuilder accessRoll = builder.newAppender("access", "RollingFile")
+                .addAttribute("fileName", logDir + "access-" + nodeId + ".log")
+                .addAttribute("filePattern", logDir + "access-" + nodeId + "-%d{MM-dd-yy}.log.gz").add(accessLayout)
+                .addComponent(triggeringPolicy);
+        builder.add(accessRoll);
+        builder.add(builder.newLogger("org.apache.hyracks.http.server.CLFLogger", Level.forName("ACCESS", 550))
+                .add(builder.newAppenderRef("access")).addAttribute("additivity", false));
+
+        return builder.build();
+    }
+
+    @Override
+    public Configuration getConfiguration(final LoggerContext loggerContext, final ConfigurationSource source) {
+        return getConfiguration(loggerContext, source.toString(), null);
+    }
+
+    @Override
+    public Configuration getConfiguration(final LoggerContext loggerContext, final String name,
+            final URI configLocation) {
+        ConfigurationBuilder<BuiltConfiguration> builder = newConfigurationBuilder();
+        return createConfiguration(builder);
+    }
+
+    @Override
+    protected String[] getSupportedTypes() {
+        return new String[] { "*" };
+    }
+
+    @Override
+    public String toString() {
+        return "NCLogConfiguration";
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
new file mode 100644
index 0000000..cfbcad8
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/CLFLogger.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.http.server;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+import io.netty.channel.socket.nio.NioSocketChannel;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.netty.channel.ChannelDuplexHandler;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelPromise;
+import io.netty.handler.codec.http.DefaultFullHttpResponse;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.DefaultHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpRequest;
+import io.netty.handler.codec.http.HttpResponse;
+import io.netty.handler.codec.http.LastHttpContent;
+
+//Based in part on LoggingHandler from Netty
+public class CLFLogger extends ChannelDuplexHandler {
+
+    private static final Logger accessLogger = LogManager.getLogger();
+    private static final String ACCESS_LOG_LEVEL = "ACCESS";
+    private static final DateTimeFormatter DATE_TIME_FORMATTER =
+            DateTimeFormatter.ofPattern("dd/MMM/yyyy:HH:mm:ss Z").withZone(ZoneId.systemDefault());
+    private StringBuilder logLineBuilder;
+
+    private String clientIp;
+    private Instant requestTime;
+    private String reqLine;
+    private int statusCode;
+    private long respSize;
+    private String userAgentRef;
+    private boolean lastChunk = false;
+
+    public CLFLogger() {
+        this.logLineBuilder = new StringBuilder();
+        respSize = 0;
+    }
+
+    @Override
+    public void channelRead(ChannelHandlerContext ctx, Object msg) {
+        if (msg instanceof HttpRequest) {
+            HttpRequest req = (HttpRequest) msg;
+            clientIp = ((NioSocketChannel) ctx.channel()).remoteAddress().getAddress().toString().substring(1);
+            requestTime = Instant.now();
+            reqLine = req.method().toString() + " " + req.getUri() + " " + req.getProtocolVersion().toString();
+            userAgentRef = headerValueOrDash("Referer", req) + " " + headerValueOrDash("User-Agent", req);
+            lastChunk = false;
+        }
+        ctx.fireChannelRead(msg);
+    }
+
+    private String headerValueOrDash(String headerKey, HttpRequest req) {
+        String value = req.headers().get(headerKey);
+        if (value == null) {
+            value = "-";
+        } else {
+            value = "\"" + value + "\"";
+        }
+        return value;
+
+    }
+
+    @Override
+    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) {
+        if (msg instanceof DefaultHttpResponse) {
+            HttpResponse resp = (DefaultHttpResponse) msg;
+            statusCode = resp.status().code();
+            if (msg instanceof DefaultFullHttpResponse) {
+                lastChunk = true;
+                respSize = resp.headers().getInt(HttpHeaderNames.CONTENT_LENGTH);
+            }
+        } else if (msg instanceof DefaultHttpContent) {
+            HttpContent content = (DefaultHttpContent) msg;
+
+            respSize += content.content().readableBytes();
+        } else if (msg instanceof LastHttpContent) {
+            lastChunk = true;
+        }
+
+        ctx.write(msg, promise);
+    }
+
+    @Override
+    public void flush(ChannelHandlerContext ctx) throws Exception {
+        if (lastChunk) {
+            printAndPrepare();
+            lastChunk = false;
+        }
+        ctx.flush();
+    }
+
+    private void printAndPrepare() {
+        logLineBuilder.append(clientIp);
+        //identd value - not relevant here
+        logLineBuilder.append(" - ");
+        //no http auth or any auth either for that matter
+        logLineBuilder.append(" - [");
+        logLineBuilder.append(DATE_TIME_FORMATTER.format(requestTime));
+        logLineBuilder.append("] \"");
+        logLineBuilder.append(reqLine);
+        logLineBuilder.append("\"");
+        logLineBuilder.append(" ").append(statusCode);
+        logLineBuilder.append(" ").append(respSize);
+        logLineBuilder.append(" ").append(userAgentRef);
+        accessLogger.log(Level.forName(ACCESS_LOG_LEVEL, 550), logLineBuilder.toString());
+        respSize = 0;
+        logLineBuilder.setLength(0);
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 42b47fb..343faa4 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -28,6 +28,9 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import io.netty.util.internal.logging.InternalLoggerFactory;
+import io.netty.util.internal.logging.Log4J2LoggerFactory;
+import io.netty.util.internal.logging.Log4JLoggerFactory;
 import org.apache.hyracks.http.api.IChannelClosedHandler;
 import org.apache.hyracks.http.api.IServlet;
 import org.apache.hyracks.util.MXHelper;
@@ -49,6 +52,8 @@
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.logging.LogLevel;
 import io.netty.handler.logging.LoggingHandler;
+import io.netty.util.internal.logging.InternalLoggerFactory;
+import io.netty.util.internal.logging.Log4J2LoggerFactory;
 
 public class HttpServer {
     // Constants
@@ -81,6 +86,10 @@
     private Throwable cause;
     private HttpServerConfig config;
 
+    static {
+        InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
+    }
+
     public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) {
         this(bossGroup, workerGroup, port, config, null);
     }
@@ -91,6 +100,7 @@
         this.workerGroup = workerGroup;
         this.port = port;
         this.closedHandler = closeHandler;
+        InternalLoggerFactory.setDefaultFactory(Log4J2LoggerFactory.INSTANCE);
         this.config = config;
         ctx = new ConcurrentHashMap<>();
         servlets = new ArrayList<>();
@@ -100,11 +110,12 @@
                 runnable -> new Thread(runnable, "HttpExecutor(port:" + port + ")-" + threadId.getAndIncrement()));
         long directMemoryBudget = numExecutorThreads * (long) HIGH_WRITE_BUFFER_WATER_MARK
                 + numExecutorThreads * config.getMaxResponseChunkSize();
-        LOGGER.log(Level.INFO, "The output direct memory budget for this server is " + directMemoryBudget + " bytes");
+        LOGGER.log(Level.DEBUG,
+                "The output direct memory budget for this server " + "is " + directMemoryBudget + " bytes");
         long inputBudgetEstimate =
                 (long) config.getMaxRequestInitialLineLength() * (config.getRequestQueueSize() + numExecutorThreads);
         inputBudgetEstimate = inputBudgetEstimate * 2;
-        LOGGER.log(Level.INFO,
+        LOGGER.log(Level.DEBUG,
                 "The \"estimated\" input direct memory budget for this server is " + inputBudgetEstimate + " bytes");
         // Having multiple arenas, memory fragments, and local thread cached buffers
         // can cause the input memory usage to exceed estimate and custom buffer allocator must be used to avoid this
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
index e10da64..eafee2d 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerInitializer.java
@@ -21,8 +21,11 @@
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelPipeline;
 import io.netty.channel.socket.SocketChannel;
+import io.netty.handler.codec.http.HttpObjectAggregator;
 import io.netty.handler.codec.http.HttpRequestDecoder;
 import io.netty.handler.codec.http.HttpResponseEncoder;
+import io.netty.handler.logging.LogLevel;
+import org.apache.logging.log4j.Logger;
 
 public class HttpServerInitializer extends ChannelInitializer<SocketChannel> {
 
@@ -49,6 +52,7 @@
         p.addLast(new HttpRequestCapacityController(server));
         p.addLast(new HttpRequestDecoder(maxRequestInitialLineLength, maxRequestHeaderSize, maxRequestChunkSize));
         p.addLast(new HttpResponseEncoder());
+        p.addLast(new CLFLogger());
         p.addLast(new HttpRequestAggregator(maxRequestSize));
         p.addLast(server.createHttpHandler(maxResponseChunkSize));
     }