[NO ISSUE] IOCounter improvements
- cache results for short period
- split iostat, proc fs impls
Change-Id: I7789171db6b6d7eea3561c24467af63f065f5dc6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2515
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Murtadha Hubail <mhubail@apache.org>
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
index 3612d8f..a85ca2c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IIOCounter.java
@@ -22,12 +22,12 @@
public interface IIOCounter {
/**
- * @return the number of block reads from the very beginning
+ * @return the number of block reads from the very beginning, or -1 if not available on this platform
*/
- public long getReads();
+ long getReads();
/**
- * @return the number of block writes from the very beginning
+ * @return the number of block writes from the very beginning, or -1 if not available on this platform
*/
- public long getWrites();
+ long getWrites();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java
new file mode 100644
index 0000000..842d82b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterCache.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.control.nc.io.profiling;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hyracks.util.Span;
+
+abstract class IOCounterCache<T> implements IIOCounter {
+ private static final long TTL_NANOS = TimeUnit.MILLISECONDS.toNanos(500);
+ private Span span;
+ private T info;
+
+ protected synchronized T getInfo() throws IOException {
+ if (info == null || span.elapsed()) {
+ span = Span.start(TTL_NANOS, TimeUnit.NANOSECONDS);
+ info = calculateInfo();
+ }
+ return info;
+ }
+
+ protected abstract T calculateInfo() throws IOException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
index 1f8669d..d5ec9a0 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterDefault.java
@@ -21,14 +21,16 @@
public class IOCounterDefault implements IIOCounter {
+ public static final long IO_COUNTER_UNAVAILABLE = -1;
+
@Override
public long getReads() {
- return 0;
+ return IO_COUNTER_UNAVAILABLE;
}
@Override
public long getWrites() {
- return 0;
+ return IO_COUNTER_UNAVAILABLE;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
index 2301ae6..36b310d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterFactory.java
@@ -33,9 +33,12 @@
*/
public IIOCounter getIOCounter() {
String osName = System.getProperty("os.name").toLowerCase();
- if (osName.indexOf("nix") >= 0 || osName.indexOf("nux") >= 0 || osName.indexOf("aix") >= 0) {
- return new IOCounterLinux();
- } else if (osName.indexOf("mac") >= 0) {
+ if (osName.contains("nix") || osName.contains("nux") || osName.contains("aix")) {
+ if (IOCounterProc.STATFILE.exists()) {
+ return new IOCounterProc();
+ }
+ return new IOCounterIoStat();
+ } else if (osName.contains("mac")) {
return new IOCounterOSX();
} else {
return new IOCounterDefault();
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java
new file mode 100644
index 0000000..560035b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterIoStat.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.control.nc.io.profiling;
+
+import static org.apache.hyracks.control.nc.io.profiling.IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class IOCounterIoStat extends IOCounterCache<List<String>> {
+ private static final Logger LOGGER = LogManager.getLogger();
+ private static final String COMMAND = "iostat";
+ private static final int PAGE_SIZE = 512;
+ private long failureCount;
+
+ private long baseReads;
+ private long baseWrites;
+
+ IOCounterIoStat() {
+ baseReads = getReads();
+ baseWrites = getWrites();
+ }
+
+ @Override
+ public long getReads() {
+ try {
+ long reads = extractColumn(4) * PAGE_SIZE;
+ return reads == 0 ? IO_COUNTER_UNAVAILABLE : reads - baseReads;
+ } catch (Exception e) {
+ LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting reads", e);
+ return IO_COUNTER_UNAVAILABLE;
+ }
+ }
+
+ @Override
+ public long getWrites() {
+ try {
+ long writes = extractColumn(5) * PAGE_SIZE;
+ return writes == 0 ? IO_COUNTER_UNAVAILABLE : writes - baseWrites;
+ } catch (Exception e) {
+ LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting writes", e);
+ return IO_COUNTER_UNAVAILABLE;
+ }
+ }
+
+ private long extractColumn(int columnIndex) throws IOException {
+ boolean device = false;
+ long ios = 0;
+ for (String line : getInfo()) {
+ if (line.contains("Blk_read")) {
+ device = true;
+ continue;
+ }
+ if (device) {
+ StringTokenizer tokenizer = new StringTokenizer(line);
+ int i = 0;
+ while (tokenizer.hasMoreTokens()) {
+ String column = tokenizer.nextToken();
+ if (i == columnIndex) {
+ ios += Long.parseLong(column);
+ break;
+ }
+ i++;
+ }
+ }
+ }
+ return ios;
+ }
+
+ @Override
+ protected List<String> calculateInfo() throws IOException {
+ try (InputStream inputStream = Runtime.getRuntime().exec(COMMAND).getInputStream()) {
+ return IOUtils.readLines(inputStream, Charset.defaultCharset());
+ }
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java
deleted file mode 100644
index 3db4c8c..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterLinux.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.control.nc.io.profiling;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.StringTokenizer;
-
-public class IOCounterLinux implements IIOCounter {
- public static final String COMMAND = "iostat";
- public static final String STATFILE = "/proc/self/io";
- public static final int PAGE_SIZE = 4096;
-
- private long baseReads = 0;
- private long baseWrites = 0;
-
- public IOCounterLinux() {
- baseReads = getReads();
- baseWrites = getWrites();
- }
-
- @Override
- public long getReads() {
- try {
- long reads = extractRow(4);
- return reads;
- } catch (IOException e) {
- try {
- long reads = extractColumn(4) * PAGE_SIZE;
- return reads - baseReads;
- } catch (IOException e2) {
- return 0;
- }
- }
- }
-
- @Override
- public long getWrites() {
- try {
- long writes = extractRow(5);
- long cancelledWrites = extractRow(6);
- return (writes - cancelledWrites);
- } catch (IOException e) {
- try {
- long writes = extractColumn(5) * PAGE_SIZE;
- return writes - baseWrites;
- } catch (IOException e2) {
- return 0;
- }
- }
- }
-
- private long extractColumn(int columnIndex) throws IOException {
- BufferedReader reader = exec(COMMAND);
- String line = null;
- boolean device = false;
- long ios = 0;
- while ((line = reader.readLine()) != null) {
- if (line.contains("Blk_read")) {
- device = true;
- continue;
- }
- if (device == true) {
- StringTokenizer tokenizer = new StringTokenizer(line);
- int i = 0;
- while (tokenizer.hasMoreTokens()) {
- String column = tokenizer.nextToken();
- if (i == columnIndex) {
- ios += Long.parseLong(column);
- break;
- }
- i++;
- }
- }
- }
- reader.close();
- return ios;
- }
-
- private long extractRow(int rowIndex) throws IOException {
- BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(STATFILE)));
- String line = null;
- long ios = 0;
- int i = 0;
- while ((line = reader.readLine()) != null) {
- if (i == rowIndex) {
- StringTokenizer tokenizer = new StringTokenizer(line);
- int j = 0;
- while (tokenizer.hasMoreTokens()) {
- String column = tokenizer.nextToken();
- if (j == 1) {
- ios = Long.parseLong(column);
- break;
- }
- j++;
- }
- }
- i++;
- }
- reader.close();
- return ios;
- }
-
- private BufferedReader exec(String command) throws IOException {
- Process p = Runtime.getRuntime().exec(command);
- return new BufferedReader(new InputStreamReader(p.getInputStream()));
- }
-
-}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
index 729157b..a48d55d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterOSX.java
@@ -19,16 +19,18 @@
package org.apache.hyracks.control.nc.io.profiling;
+import static org.apache.hyracks.control.nc.io.profiling.IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+
public class IOCounterOSX implements IIOCounter {
@Override
public long getReads() {
- return 0;
+ return IO_COUNTER_UNAVAILABLE;
}
@Override
public long getWrites() {
- return 0;
+ return IO_COUNTER_UNAVAILABLE;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java
new file mode 100644
index 0000000..8882271
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/profiling/IOCounterProc.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.control.nc.io.profiling;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class IOCounterProc extends IOCounterCache<List<String>> {
+ private static final Logger LOGGER = LogManager.getLogger();
+ @SuppressWarnings("squid:S1075") // hardcoded URI
+ public static final File STATFILE = new File("/proc/self/io");
+ private long failureCount;
+
+ @Override
+ public long getReads() {
+ try {
+ return extractRow(getInfo(), 4);
+ } catch (Exception e) {
+ LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting reads", e);
+ return IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+ }
+ }
+
+ @Override
+ public long getWrites() {
+ try {
+ List<String> rows = getInfo();
+ long writes = extractRow(rows, 5);
+ long cancelledWrites = extractRow(rows, 6);
+ return writes - cancelledWrites;
+ } catch (Exception e) {
+ LOGGER.log(failureCount++ > 0 ? Level.DEBUG : Level.WARN, "Failure getting writes", e);
+ return IOCounterDefault.IO_COUNTER_UNAVAILABLE;
+ }
+ }
+
+ private long extractRow(List<String> rows, int rowIndex) {
+ return Long.parseLong(StringUtils.split(rows.get(rowIndex), ' ')[1]);
+ }
+
+ @Override
+ protected List<String> calculateInfo() throws IOException {
+ return FileUtils.readLines(STATFILE, Charset.defaultCharset());
+ }
+
+}