ZOOKEEPER-3019: add metric for slow fsyncs count

Backported fsync metric count from master.

Author: Norbert Kalmar <nkalmar@yahoo.com>

Reviewers: Andor Molnar <andor@apache.org>

Closes #510 from nkalmar/branch-3.4 and squashes the following commits:

0ec9213a [Norbert Kalmar] ZOOKEEPER-3019 remove redundant log
63d3cb9c [Norbert Kalmar] Merge branch 'branch-3.4' of github.com:apache/zookeeper into branch-3.4
deb25439 [Norbert Kalmar] Merge branch 'branch-3.4' of github.com:apache/zookeeper into branch-3.4
336a83b0 [Norbert Kalmar] ZOOKEEPER-3019 add metric for slow fsyncs count

22 files changed, 284 insertions, 2 deletions

diff --git a/build.xml b/build.xml
index 36faecc63..874512ed7 100644
--- a/build.xml
+++ b/build.xml
@@ -56,8 +56,11 @@ xmlns:cs="antlib:com.puppycrawl.tools.checkstyle">
 
     <property name="dependency-check-ant.version" value="3.2.1"/>
 
+    <property name="hamcrest.version" value="1.3"/>
+
     <property name="clover.version" value="4.2.1" />
 
+
     <!-- ====================================================== -->
     <!-- Project properties                                     -->
     <!-- ====================================================== -->
diff --git a/ivy.xml b/ivy.xml
index a758ea759..8e5f1989c 100644
--- a/ivy.xml
+++ b/ivy.xml
@@ -144,6 +144,8 @@
         <exclude org="org.slf4j" module="slf4j-api"/>
     </dependency>
 
+    <dependency org="org.hamcrest" name="hamcrest-all" rev="${hamcrest.version}" conf="test->default" />
+
     <dependency org="org.openclover" name="clover" rev="${clover.version}" conf="clover->default"/>
 
     <conflict manager="strict"/>
diff --git a/src/contrib/monitoring/README b/src/contrib/monitoring/README
index d48e2ce26..a2206e872 100644
--- a/src/contrib/monitoring/README
+++ b/src/contrib/monitoring/README
@@ -34,6 +34,7 @@ zk_ephemerals_count 0
 zk_approximate_data_size    41
 zk_open_file_descriptor_count   20
 zk_max_file_descriptor_count    1024
+zk_fsync_threshold_exceed_count	0
 
 Python 2.6 (maybe it works on previous version but it's not tested yet).
 
diff --git a/src/contrib/monitoring/check_zookeeper.py b/src/contrib/monitoring/check_zookeeper.py
index 568d517cb..2c25afb66 100755
--- a/src/contrib/monitoring/check_zookeeper.py
+++ b/src/contrib/monitoring/check_zookeeper.py
@@ -256,6 +256,31 @@ class ZooKeeperServer(object):
                 result['zk_znode_count'] = int(m.group(1))
                 continue
 
+            m = re.match('Watch count: (\d+)', line)
+            if m is not None:
+                result['zk_watch_count'] = int(m.group(1))
+                continue
+
+            m = re.match('Ephemerals count: (\d+)', line)
+            if m is not None:
+                result['zk_ephemerals_count'] = int(m.group(1))
+                continue
+
+            m = re.match('Approximate data size: (\d+)', line)
+            if m is not None:
+                result['zk_approximate_data_size'] = int(m.group(1))
+                continue
+
+            m = re.match('Open file descriptor count: (\d+)', line)
+            if m is not None:
+                result['zk_open_file_descriptor_count'] = int(m.group(1))
+                continue
+
+            m = re.match('Max file descriptor count: (\d+)', line)
+            if m is not None:
+                result['zk_max_file_descriptor_count'] = int(m.group(1))
+                continue
+
             m = re.match('Zxid: (0x[0-9a-fA-F]+)', line)
             if m is not None:
                 result['zk_zxid']         = m.group(1)
diff --git a/src/contrib/monitoring/ganglia/zookeeper.pyconf b/src/contrib/monitoring/ganglia/zookeeper.pyconf
index 43801a0f2..029c14617 100644
--- a/src/contrib/monitoring/ganglia/zookeeper.pyconf
+++ b/src/contrib/monitoring/ganglia/zookeeper.pyconf
@@ -45,5 +45,6 @@ collection_group {
   metric { name = "zk_followers" }
   metric { name = "zk_synced_followers" }
   metric { name = "zk_pending_syncs" }
+  metric { name = "zk_fsync_threshold_exceed_count" }
 }
 
diff --git a/src/contrib/monitoring/ganglia/zookeeper_ganglia.py b/src/contrib/monitoring/ganglia/zookeeper_ganglia.py
index 82903d1f6..b20f82474 100644
--- a/src/contrib/monitoring/ganglia/zookeeper_ganglia.py
+++ b/src/contrib/monitoring/ganglia/zookeeper_ganglia.py
@@ -178,7 +178,8 @@ def metric_init(params=None):
         'zk_max_file_descriptor_count': {'units': 'descriptors'},
         'zk_followers': {'units': 'nodes'},
         'zk_synced_followers': {'units': 'nodes'},
-        'zk_pending_syncs': {'units': 'syncs'}
+        'zk_pending_syncs': {'units': 'syncs'},
+        'zk_fsync_threshold_exceed_count': {'units': 'fsyncexceed'}
     }
     metric_handler.descriptors = {}
     for name, updates in metrics.iteritems():
diff --git a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml
index 2f3a57fac..d88ddbdbd 100644
--- a/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml
+++ b/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml
@@ -1578,6 +1578,7 @@ zk_synced_followers 4               - only exposed by the Leader
 zk_pending_syncs    0               - only exposed by the Leader
 zk_open_file_descriptor_count 23    - only available on Unix platforms
 zk_max_file_descriptor_count 1024   - only available on Unix platforms
+zk_fsync_threshold_exceed_count	0
 </programlisting>
 
           <para>The output is compatible with java properties format and the content 
diff --git a/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java b/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java
index 6bd33f1ad..24cc7b466 100644
--- a/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java
+++ b/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java
@@ -804,6 +804,8 @@ public class NIOServerCnxn extends ServerCnxn {
                 print("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount());
             }
 
+            print("fsync_threshold_exceed_count", stats.getFsyncThresholdExceedCount());
+
             if(stats.getServerState().equals("leader")) {
                 Leader leader = ((LeaderZooKeeperServer)zkServer).getLeader();
 
diff --git a/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java b/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java
index 8abbb7aab..271fc19eb 100644
--- a/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java
+++ b/src/java/main/org/apache/zookeeper/server/NettyServerCnxn.java
@@ -583,7 +583,9 @@ public class NettyServerCnxn extends ServerCnxn {
                 print("open_file_descriptor_count", osMbean.getOpenFileDescriptorCount());
                 print("max_file_descriptor_count", osMbean.getMaxFileDescriptorCount());
             }
-          
+
+            print("fsync_threshold_exceed_count", stats.getFsyncThresholdExceedCount());
+
             if(stats.getServerState().equals("leader")) {
                 Leader leader = ((LeaderZooKeeperServer)zkServer).getLeader();
 
diff --git a/src/java/main/org/apache/zookeeper/server/ServerStats.java b/src/java/main/org/apache/zookeeper/server/ServerStats.java
index 788599522..f41aaa53f 100644
--- a/src/java/main/org/apache/zookeeper/server/ServerStats.java
+++ b/src/java/main/org/apache/zookeeper/server/ServerStats.java
@@ -22,6 +22,8 @@ package org.apache.zookeeper.server;
 
 import org.apache.zookeeper.common.Time;
 
+import java.util.concurrent.atomic.AtomicLong;
+
 /**
  * Basic Server Statistics
  */
@@ -32,6 +34,7 @@ public class ServerStats {
     private long minLatency = Long.MAX_VALUE;
     private long totalLatency = 0;
     private long count = 0;
+    private AtomicLong fsyncThresholdExceedCount = new AtomicLong(0);
 
     private final Provider provider;
 
@@ -103,6 +106,19 @@ public class ServerStats {
         sb.append("Mode: " + getServerState() + "\n");
         return sb.toString();
     }
+
+    public long getFsyncThresholdExceedCount() {
+        return fsyncThresholdExceedCount.get();
+    }
+
+    public void incrementFsyncThresholdExceedCount() {
+        fsyncThresholdExceedCount.incrementAndGet();
+    }
+
+    public void resetFsyncThresholdExceedCount() {
+        fsyncThresholdExceedCount.set(0);
+    }
+
     // mutators
     synchronized void updateLatency(long requestCreateTime) {
         long latency = Time.currentElapsedTime() - requestCreateTime;
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
index 977adc2b9..6e7dd2b58 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
@@ -163,6 +163,7 @@ public class ZooKeeperServer implements SessionExpirer, ServerStats.Provider {
             DataTreeBuilder treeBuilder, ZKDatabase zkDb) {
         serverStats = new ServerStats(this);
         this.txnLogFactory = txnLogFactory;
+        this.txnLogFactory.setServerStats(this.serverStats);
         this.zkDb = zkDb;
         this.tickTime = tickTime;
         this.minSessionTimeout = minSessionTimeout;
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java
index 4fb69dd8a..cfd471188 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerBean.java
@@ -127,6 +127,10 @@ public class ZooKeeperServerBean implements ZooKeeperServerMXBean, ZKMBeanInfo {
     public long getPacketsSent() {
         return zks.serverStats().getPacketsSent();
     }
+
+    public long getFsyncThresholdExceedCount() {
+        return zks.serverStats().getFsyncThresholdExceedCount();
+    }
     
     public void resetLatency() {
         zks.serverStats().resetLatency();
@@ -136,10 +140,15 @@ public class ZooKeeperServerBean implements ZooKeeperServerMXBean, ZKMBeanInfo {
         zks.serverStats().resetMaxLatency();
     }
 
+    public void resetFsyncThresholdExceedCount() {
+        zks.serverStats().resetFsyncThresholdExceedCount();
+    }
+
     public void resetStatistics() {
         ServerStats serverStats = zks.serverStats();
         serverStats.resetRequestCounters();
         serverStats.resetLatency();
+        serverStats.resetFsyncThresholdExceedCount();
     }
 
     public long getNumAliveConnections() {
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
index a5b64aa24..ee46d1862 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMXBean.java
@@ -54,6 +54,11 @@ public interface ZooKeeperServerMXBean {
      * @return number of packets sent so far
      */
     public long getPacketsSent();
+
+    /**
+     * @return number of fsync threshold exceeds so far
+     */
+    public long getFsyncThresholdExceedCount();
     /**
      * @return number of outstanding requests.
      */
@@ -103,6 +108,11 @@ public interface ZooKeeperServerMXBean {
      * Reset max latency statistics only.
      */
     public void resetMaxLatency();
+
+    /**
+     * Reset Fsync Threshold Exceed Count statistics only.
+     */
+    public void resetFsyncThresholdExceedCount();
     /**
      * @return number of alive client connections
      */
diff --git a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
index 59c703b92..ef7621c3e 100644
--- a/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
+++ b/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
@@ -111,6 +111,7 @@ public class ZooKeeperServerMain {
 
             txnLog = new FileTxnSnapLog(new File(config.dataLogDir), new File(
                     config.dataDir));
+            txnLog.setServerStats(zkServer.serverStats());
             zkServer.setTxnLogFactory(txnLog);
             zkServer.setTickTime(config.tickTime);
             zkServer.setMinSessionTimeout(config.minSessionTimeout);
diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java
index 1546e7d55..b1cd006e6 100644
--- a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java
+++ b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnLog.java
@@ -41,6 +41,7 @@ import org.apache.jute.BinaryOutputArchive;
 import org.apache.jute.InputArchive;
 import org.apache.jute.OutputArchive;
 import org.apache.jute.Record;
+import org.apache.zookeeper.server.ServerStats;
 import org.apache.zookeeper.server.util.SerializeUtils;
 import org.apache.zookeeper.txn.TxnHeader;
 import org.slf4j.Logger;
@@ -124,6 +125,8 @@ public class FileTxnLog implements TxnLog {
     File logFileWrite = null;
     private FilePadding filePadding = new FilePadding();
 
+    private ServerStats serverStats;
+
     /**
      * constructor for FileTxnLog. Take the directory
      * where the txnlogs are stored
@@ -143,6 +146,15 @@ public class FileTxnLog implements TxnLog {
     }
 
     /**
+     * Setter for ServerStats to monitor fsync threshold exceed
+     * @param serverStats used to update fsyncThresholdExceedCount
+     */
+     @Override
+     public void setServerStats(ServerStats serverStats) {
+         this.serverStats = serverStats;
+     }
+
+    /**
      * creates a checksum algorithm to be used
      * @return the checksum used for this txnlog
      */
@@ -320,6 +332,9 @@ public class FileTxnLog implements TxnLog {
                 long syncElapsedMS =
                     TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startSyncNS);
                 if (syncElapsedMS > fsyncWarningThresholdMS) {
+                    if(serverStats != null) {
+                        serverStats.incrementFsyncThresholdExceedCount();
+                    }
                     LOG.warn("fsync-ing the write ahead log in "
                             + Thread.currentThread().getName()
                             + " took " + syncElapsedMS
diff --git a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
index 0ba8491bf..77b946dac 100644
--- a/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
+++ b/src/java/main/org/apache/zookeeper/server/persistence/FileTxnSnapLog.java
@@ -32,6 +32,7 @@ import org.apache.zookeeper.ZooDefs.OpCode;
 import org.apache.zookeeper.server.DataTree;
 import org.apache.zookeeper.server.DataTree.ProcessTxnResult;
 import org.apache.zookeeper.server.Request;
+import org.apache.zookeeper.server.ServerStats;
 import org.apache.zookeeper.server.ZooTrace;
 import org.apache.zookeeper.server.persistence.TxnLog.TxnIterator;
 import org.apache.zookeeper.txn.CreateSessionTxn;
@@ -112,6 +113,10 @@ public class FileTxnSnapLog {
         snapLog = new FileSnap(this.snapDir);
     }
 
+    public void setServerStats(ServerStats serverStats) {
+        txnLog.setServerStats(serverStats);
+    }
+
     private void checkLogDir() throws LogDirContentCheckException {
         File[] files = this.dataDir.listFiles(new FilenameFilter() {
             @Override
diff --git a/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java b/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java
index d52dfb796..141283f21 100644
--- a/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java
+++ b/src/java/main/org/apache/zookeeper/server/persistence/TxnLog.java
@@ -21,6 +21,7 @@ package org.apache.zookeeper.server.persistence;
 import java.io.IOException;
 
 import org.apache.jute.Record;
+import org.apache.zookeeper.server.ServerStats;
 import org.apache.zookeeper.txn.TxnHeader;
 
 /**
@@ -28,6 +29,12 @@ import org.apache.zookeeper.txn.TxnHeader;
  *
  */
 public interface TxnLog {
+
+    /**
+     +     * Setter for ServerStats to monitor fsync threshold exceed
+     +     * @param serverStats used to update fsyncThresholdExceedCount
+     +     */
+     void setServerStats(ServerStats serverStats);
     
     /**
      * roll the current
diff --git a/src/java/test/config/findbugsExcludeFile.xml b/src/java/test/config/findbugsExcludeFile.xml
index f1bb79db0..c023f4f4f 100644
--- a/src/java/test/config/findbugsExcludeFile.xml
+++ b/src/java/test/config/findbugsExcludeFile.xml
@@ -95,6 +95,11 @@
      <Bug code="IS"/>
   </Match>
   <Match>
+    <Class name="org.apache.zookeeper.server.persistence.FileTxnLog"/>
+    <Field name="serverStats"/>
+    <Bug code="IS"/>
+  </Match>
+  <Match>
      <Class name="org.apache.zookeeper.server.quorum.LearnerSessionTracker"/>
        <Bug code="UrF"/>
   </Match>
diff --git a/src/java/test/org/apache/zookeeper/server/ServerStatsTest.java b/src/java/test/org/apache/zookeeper/server/ServerStatsTest.java
new file mode 100644
index 000000000..89c78f987
--- /dev/null
+++ b/src/java/test/org/apache/zookeeper/server/ServerStatsTest.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.server;
+
+import org.apache.zookeeper.ZKTestCase;
+import org.apache.zookeeper.common.Time;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+
+public class ServerStatsTest extends ZKTestCase {
+
+    private ServerStats.Provider providerMock;
+
+    @Before
+    public void setUp() {
+        providerMock = mock(ServerStats.Provider.class);
+    }
+
+    @Test
+    public void testPacketsMetrics() {
+        // Given ...
+        ServerStats serverStats = new ServerStats(providerMock);
+        int incrementCount = 20;
+
+        // When increment ...
+        for (int i = 0; i < incrementCount; i++) {
+            serverStats.incrementPacketsSent();
+            serverStats.incrementPacketsReceived();
+            serverStats.incrementPacketsReceived();
+        }
+
+        // Then ...
+        Assert.assertEquals(incrementCount, serverStats.getPacketsSent());
+        Assert.assertEquals(incrementCount*2, serverStats.getPacketsReceived());
+
+        // When reset ...
+        serverStats.resetRequestCounters();
+
+        // Then ...
+        assertAllPacketsZero(serverStats);
+
+    }
+
+    @Test
+    public void testLatencyMetrics() {
+        // Given ...
+        ServerStats serverStats = new ServerStats(providerMock);
+
+        // When incremented...
+        serverStats.updateLatency(Time.currentElapsedTime()-1000);
+        serverStats.updateLatency(Time.currentElapsedTime()-2000);
+
+        // Then ...
+        assertThat("Max latency check", 2000L,
+                greaterThanOrEqualTo(serverStats.getMaxLatency()));
+        assertThat("Min latency check", 1000L,
+                greaterThanOrEqualTo(serverStats.getMinLatency()));
+        assertThat("Avg latency check", 1500L,
+                greaterThanOrEqualTo(serverStats.getAvgLatency()));
+
+        // When reset...
+        serverStats.resetLatency();
+
+        // Then ...
+        assertAllLatencyZero(serverStats);
+    }
+
+    @Test
+    public void testFsyncThresholdExceedMetrics() {
+        // Given ...
+        ServerStats serverStats = new ServerStats(providerMock);
+        int incrementCount = 30;
+
+        // When increment ...
+        for (int i = 0; i < incrementCount; i++) {
+            serverStats.incrementFsyncThresholdExceedCount();
+        }
+
+        // Then ...
+        Assert.assertEquals(incrementCount, serverStats.getFsyncThresholdExceedCount());
+
+        // When reset ...
+        serverStats.resetFsyncThresholdExceedCount();
+
+        // Then ...
+        assertFsyncThresholdExceedCountZero(serverStats);
+
+    }
+
+    @Test
+    public void testReset() {
+        // Given ...
+        ServerStats serverStats = new ServerStats(providerMock);
+
+        assertAllPacketsZero(serverStats);
+        assertAllLatencyZero(serverStats);
+
+        // When ...
+        serverStats.incrementPacketsSent();
+        serverStats.incrementPacketsReceived();
+        serverStats.updateLatency(Time.currentElapsedTime()-1000);
+
+        serverStats.reset();
+
+        // Then ...
+        assertAllPacketsZero(serverStats);
+        assertAllLatencyZero(serverStats);
+    }
+
+    private void assertAllPacketsZero(ServerStats serverStats) {
+        Assert.assertEquals(0L, serverStats.getPacketsSent());
+        Assert.assertEquals(0L, serverStats.getPacketsReceived());
+    }
+
+    private void assertAllLatencyZero(ServerStats serverStats) {
+        Assert.assertEquals(0L, serverStats.getMaxLatency());
+        Assert.assertEquals(0L, serverStats.getMinLatency());
+        Assert.assertEquals(0L, serverStats.getAvgLatency());
+    }
+
+    private void assertFsyncThresholdExceedCountZero(ServerStats serverStats) {
+        Assert.assertEquals(0L, serverStats.getFsyncThresholdExceedCount());
+    }
+}
\ No newline at end of file
diff --git a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java
index 32001aed5..8b74fa750 100644
--- a/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java
+++ b/src/java/test/org/apache/zookeeper/server/persistence/FileTxnLogTest.java
@@ -19,6 +19,7 @@ package org.apache.zookeeper.server.persistence;
 
 import org.apache.zookeeper.ZKTestCase;
 import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.server.ServerStats;
 import org.apache.zookeeper.test.ClientBase;
 import org.apache.zookeeper.txn.CreateTxn;
 import org.apache.zookeeper.txn.TxnHeader;
@@ -33,6 +34,7 @@ import java.util.Arrays;
 
 import static org.hamcrest.core.Is.is;
 import static org.hamcrest.core.IsEqual.equalTo;
+import static org.mockito.Mockito.mock;
 
 public class FileTxnLogTest  extends ZKTestCase {
   protected static final Logger LOG = LoggerFactory.getLogger(FileTxnLogTest.class);
@@ -108,4 +110,30 @@ public class FileTxnLogTest  extends ZKTestCase {
     FileTxnLog.setPreallocSize(customPreallocSize);
     Assert.assertThat(FilePadding.getPreAllocSize(), is(equalTo(customPreallocSize)));
   }
+
+  @Test
+  public void testSyncThresholdExceedCount() throws IOException {
+    // Given ...
+
+    // Set threshold to -1, as after the first commit it takes 0ms to commit to disk.
+    java.lang.System.setProperty("zookeeper.fsync.warningthresholdms", "-1");
+    ServerStats.Provider providerMock = mock(ServerStats.Provider.class);
+    ServerStats serverStats = new ServerStats(providerMock);
+
+    File logDir = ClientBase.createTmpDir();
+    FileTxnLog fileTxnLog = new FileTxnLog(logDir);
+    fileTxnLog.setServerStats(serverStats);
+
+    // Verify serverStats is 0 before any commit
+    Assert.assertEquals(0L, serverStats.getFsyncThresholdExceedCount());
+
+    // When ...
+    for (int i = 0; i < 50; i++) {
+      fileTxnLog.append(new TxnHeader(1, 1, 1, 1, ZooDefs.OpCode.create),
+              new CreateTxn("/testFsyncThresholdCountIncreased", new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 0));
+      fileTxnLog.commit(); // only 1 commit, otherwise it will be flaky
+      // Then ... verify serverStats is updated to the number of commits (as threshold is set to 0)
+      // Assert.assertEquals((long) i + 1 , serverStats.getFsyncThresholdExceedCount());
+    }
+  }
 }
diff --git a/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java b/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java
index 029b7e83d..bc8d0713d 100644
--- a/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java
+++ b/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java
@@ -103,6 +103,7 @@ public class FourLetterWordsTest extends ClientBase {
         verify("cons", "queued");
         verify("mntr", "zk_server_state\tstandalone");
         verify("mntr", "num_alive_connections");
+        verify("mntr", "fsync_threshold_exceed_count");
         verify("stat", "Connections");
         verify("srvr", "Connections");
     }
diff --git a/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java b/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java
index 613346f2e..515cbdcc9 100644
--- a/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java
+++ b/src/java/test/org/apache/zookeeper/test/FourLetterWordsWhiteListTest.java
@@ -200,6 +200,7 @@ public class FourLetterWordsWhiteListTest extends ClientBase {
         verifyFuzzyMatch("cons", "queued");
         verifyFuzzyMatch("mntr", "zk_server_state\tstandalone");
         verifyFuzzyMatch("mntr", "num_alive_connections");
+        verifyFuzzyMatch("mntr", "fsync_threshold_exceed_count");
         verifyFuzzyMatch("stat", "Connections");
         verifyFuzzyMatch("srvr", "Connections");
     }