From 05b215994f5e145c2758c4089828b57ba471b329 Mon Sep 17 00:00:00 2001 From: Szabolcs Bukros Date: Wed, 13 Jul 2022 13:08:53 +0200 Subject: ZOOKEEPER-4566: Create tool for recursive snapshot analysis add a tool to recursively collect and display child count and data stored in sub-trees Author: Szabolcs Bukros Reviewers: Andor Molnar , Mate Szalay-Beko Closes #1902 from BukrosSzabolcs/ZOOKEEPER-4566 --- bin/zkSnapshotRecursiveSummaryToolkit.cmd | 24 ++++ bin/zkSnapshotRecursiveSummaryToolkit.sh | 39 ++++++ .../src/main/resources/markdown/zookeeperTools.md | 38 ++++++ .../zookeeper/server/SnapshotRecursiveSummary.java | 137 +++++++++++++++++++++ 4 files changed, 238 insertions(+) create mode 100755 bin/zkSnapshotRecursiveSummaryToolkit.cmd create mode 100755 bin/zkSnapshotRecursiveSummaryToolkit.sh create mode 100644 zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java diff --git a/bin/zkSnapshotRecursiveSummaryToolkit.cmd b/bin/zkSnapshotRecursiveSummaryToolkit.cmd new file mode 100755 index 000000000..ee893108e --- /dev/null +++ b/bin/zkSnapshotRecursiveSummaryToolkit.cmd @@ -0,0 +1,24 @@ +@echo off +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal +call "%~dp0zkEnv.cmd" + +set ZOOMAIN=org.apache.zookeeper.server.SnapshotRecursiveSummary +call %JAVA% -cp "%CLASSPATH%" %ZOOMAIN% %* + +endlocal + diff --git a/bin/zkSnapshotRecursiveSummaryToolkit.sh b/bin/zkSnapshotRecursiveSummaryToolkit.sh new file mode 100755 index 000000000..f7659971b --- /dev/null +++ b/bin/zkSnapshotRecursiveSummaryToolkit.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# If this scripted is run out of /usr/bin or some other system bin directory +# it should be linked to and not copied. Things like java jar files are found +# relative to the canonical path of this script. +# + +# use POSIX interface, symlink is followed automatically +ZOOBIN="${BASH_SOURCE-$0}" +ZOOBIN="$(dirname "${ZOOBIN}")" +ZOOBINDIR="$(cd "${ZOOBIN}"; pwd)" + +if [ -e "$ZOOBIN/../libexec/zkEnv.sh" ]; then + # shellcheck source=/bin/zkEnv.sh + . "$ZOOBINDIR"/../libexec/zkEnv.sh +else + . "$ZOOBINDIR"/zkEnv.sh +fi + +"$JAVA" -cp "$CLASSPATH" "$JVMFLAGS" \ + org.apache.zookeeper.server.SnapshotRecursiveSummary "$@" + + diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md b/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md index 5141766d7..d4abe3854 100644 --- a/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md +++ b/zookeeper-docs/src/main/resources/markdown/zookeeperTools.md @@ -23,6 +23,7 @@ limitations under the License. * [zkCleanup.sh](#zkCleanup) * [zkTxnLogToolkit.sh](#zkTxnLogToolkit) * [zkSnapShotToolkit.sh](#zkSnapShotToolkit) + * [zkSnapshotRecursiveSummaryToolkit.sh](#zkSnapshotRecursiveSummaryToolkit) * [zkSnapshotComparer.sh](#zkSnapshotComparer) * [Benchmark](#Benchmark) @@ -218,6 +219,43 @@ USAGE: SnapshotFormatter [-d|-json] snapshot_file ./zkSnapShotToolkit.sh -json /data/zkdata/version-2/snapshot.fa01000186d [[1,0,{"progname":"SnapshotFormatter.java","progver":"0.01","timestamp":1559788148637},[{"name":"\/","asize":0,"dsize":0,"dev":0,"ino":1001},[{"name":"zookeeper","asize":0,"dsize":0,"dev":0,"ino":1002},{"name":"config","asize":0,"dsize":0,"dev":0,"ino":1003},[{"name":"quota","asize":0,"dsize":0,"dev":0,"ino":1004},[{"name":"test","asize":0,"dsize":0,"dev":0,"ino":1005},{"name":"zookeeper_limits","asize":52,"dsize":52,"dev":0,"ino":1006},{"name":"zookeeper_stats","asize":15,"dsize":15,"dev":0,"ino":1007}]]],{"name":"test","asize":0,"dsize":0,"dev":0,"ino":1008}]] ``` + + +### zkSnapshotRecursiveSummaryToolkit.sh +Recursively collect and display child count and data size for a selected node. + + $./zkSnapshotRecursiveSummaryToolkit.sh + USAGE: + + SnapshotRecursiveSummary + + snapshot_file: path to the zookeeper snapshot + starting_node: the path in the zookeeper tree where the traversal should begin + max_depth: defines the depth where the tool still writes to the output. 0 means there is no depth limit, every non-leaf node's stats will be displayed, 1 means it will only contain the starting node's and it's children's stats, 2 ads another level and so on. This ONLY affects the level of details displayed, NOT the calculation. + +```bash +# recursively collect and display child count and data for the root node and 2 levels below it +./zkSnapshotRecursiveSummaryToolkit.sh /data/zkdata/version-2/snapshot.fa01000186d / 2 + +/ + children: 1250511 + data: 1952186580 +-- /zookeeper +-- children: 1 +-- data: 0 +-- /solr +-- children: 1773 +-- data: 8419162 +---- /solr/configs +---- children: 1640 +---- data: 8407643 +---- /solr/overseer +---- children: 6 +---- data: 0 +---- /solr/live_nodes +---- children: 3 +---- data: 0 +``` diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java new file mode 100644 index 000000000..2d5f30b63 --- /dev/null +++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/SnapshotRecursiveSummary.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.server; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import org.apache.jute.BinaryInputArchive; +import org.apache.jute.InputArchive; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.zookeeper.server.persistence.FileSnap; +import org.apache.zookeeper.server.persistence.SnapStream; + +/** + * Recursively processes a snapshot file collecting child node count and summarizes the data size + * below each node. + * "starting_node" defines the node where the recursion starts + * "max_depth" defines the depth where the tool still writes to the output. + * 0 means there is no depth limit, every non-leaf node's stats will be displayed, 1 means it will + * only contain the starting node's and it's children's stats, 2 ads another level and so on. + * This ONLY affects the level of details displayed, NOT the calculation. + */ +@InterfaceAudience.Public public class SnapshotRecursiveSummary { + + /** + * USAGE: SnapsotRecursiveSummary snapshot_file starting_node max_depth + * + */ + public static void main(String[] args) throws Exception { + if (args.length != 3) { + System.err.println(getUsage()); + System.exit(2); + } + int maxDepth = 0; + try { + maxDepth = Integer.parseInt(args[2]); + } catch (NumberFormatException e) { + System.err.println(getUsage()); + System.exit(2); + } + + new SnapshotRecursiveSummary().run(args[0], args[1], maxDepth); + } + + public void run(String snapshotFileName, String startingNode, int maxDepth) throws IOException { + File snapshotFile = new File(snapshotFileName); + try (InputStream is = SnapStream.getInputStream(snapshotFile)) { + InputArchive ia = BinaryInputArchive.getArchive(is); + + FileSnap fileSnap = new FileSnap(null); + + DataTree dataTree = new DataTree(); + Map sessions = new HashMap(); + + fileSnap.deserialize(dataTree, sessions, ia); + + printZnodeDetails(dataTree, startingNode, maxDepth); + } + } + + private void printZnodeDetails(DataTree dataTree, String startingNode, int maxDepth) { + StringBuilder builder = new StringBuilder(); + printZnode(dataTree, startingNode, builder, 0, maxDepth); + System.out.println(builder); + } + + private long[] printZnode(DataTree dataTree, String name, StringBuilder builder, int level, + int maxDepth) { + DataNode n = dataTree.getNode(name); + Set children; + long dataSum = 0L; + synchronized (n) { // keep findbugs happy + if (n.data != null) { + dataSum += n.data.length; + } + children = n.getChildren(); + } + + long[] result = {1L, dataSum}; + if (children.size() == 0) { + return result; + } + StringBuilder childBuilder = new StringBuilder(); + for (String child : children) { + long[] childResult = + printZnode(dataTree, name + (name.equals("/") ? "" : "/") + child, childBuilder, + level + 1, maxDepth); + result[0] = result[0] + childResult[0]; + result[1] = result[1] + childResult[1]; + } + + if (maxDepth == 0 || level <= maxDepth) { + String tab = String.join("", Collections.nCopies(level, "--")); + builder.append(tab + " " + name + "\n"); + builder.append(tab + " children: " + (result[0] - 1) + "\n"); + builder.append(tab + " data: " + result[1] + "\n"); + builder.append(childBuilder); + } + return result; + } + + public static String getUsage() { + String newLine = System.getProperty("line.separator"); + return String.join(newLine, + "USAGE:", + newLine, + "SnapshotRecursiveSummary ", + newLine, + "snapshot_file: path to the zookeeper snapshot", + "starting_node: the path in the zookeeper tree where the traversal should begin", + "max_depth: defines the depth where the tool still writes to the output. " + + "0 means there is no depth limit, every non-leaf node's stats will be displayed, " + + "1 means it will only contain the starting node's and it's children's stats, " + + "2 ads another level and so on. This ONLY affects the level of details displayed, " + + "NOT the calculation."); + } +} -- cgit v1.2.1