1 files changed, 298 insertions, 0 deletions
diff --git a/tests/benchmarks/benchmark.sh b/tests/benchmarks/benchmark.sh
new file mode 100755
index 000000000..4a89807b7
--- /dev/null
+++ b/tests/benchmarks/benchmark.sh
@@ -0,0 +1,298 @@
+#!/bin/bash
+
+set -eo pipefail
+
+#
+# parse the command line
+#
+
+usage() { echo "usage: $(basename "$0") [--cli <path>] [--baseline-cli <path>] [--suite <suite>] [--json <path>] [--zip <path>] [--verbose] [--debug]"; }
+
+TEST_CLI="git"
+BASELINE_CLI=
+SUITE=
+JSON_RESULT=
+ZIP_RESULT=
+OUTPUT_DIR=
+VERBOSE=
+DEBUG=
+NEXT=
+
+for a in "$@"; do
+	if [ "${NEXT}" = "cli" ]; then
+		TEST_CLI="${a}"
+		NEXT=
+	elif [ "${NEXT}" = "baseline-cli" ]; then
+		BASELINE_CLI="${a}"
+		NEXT=
+	elif [ "${NEXT}" = "suite" ]; then
+		SUITE="${a}"
+		NEXT=
+	elif [ "${NEXT}" = "json" ]; then
+		JSON_RESULT="${a}"
+		NEXT=
+	elif [ "${NEXT}" = "zip" ]; then
+		ZIP_RESULT="${a}"
+		NEXT=
+	elif [ "${NEXT}" = "output-dir" ]; then
+		OUTPUT_DIR="${a}"
+		NEXT=
+	elif [ "${a}" = "c" ] || [ "${a}" = "--cli" ]; then
+		NEXT="cli"
+	elif [[ "${a}" == "-c"* ]]; then
+		TEST_CLI="${a/-c/}"
+	elif [ "${a}" = "b" ] || [ "${a}" = "--baseline-cli" ]; then
+		NEXT="baseline-cli"
+	elif [[ "${a}" == "-b"* ]]; then
+		BASELINE_CLI="${a/-b/}"
+	elif [ "${a}" = "-s" ] || [ "${a}" = "--suite" ]; then
+		NEXT="suite"
+	elif [[ "${a}" == "-s"* ]]; then
+		SUITE="${a/-s/}"
+	elif [ "${a}" = "-v" ] || [ "${a}" == "--verbose" ]; then
+		VERBOSE=1
+	elif [ "${a}" == "--debug" ]; then
+		VERBOSE=1
+		DEBUG=1
+	elif [ "${a}" = "-j" ] || [ "${a}" == "--json" ]; then
+		NEXT="json"
+	elif [[ "${a}" == "-j"* ]]; then
+		JSON_RESULT="${a/-j/}"
+	elif [ "${a}" = "-z" ] || [ "${a}" == "--zip" ]; then
+		NEXT="zip"
+	elif [[ "${a}" == "-z"* ]]; then
+		ZIP_RESULT="${a/-z/}"
+	elif [ "${a}" = "--output-dir" ]; then
+		NEXT="output-dir"
+	else
+		echo "$(basename "$0"): unknown option: ${a}" 1>&2
+		usage 1>&2
+		exit 1
+	fi
+done
+
+if [ "${NEXT}" != "" ]; then
+	usage 1>&2
+	exit 1
+fi
+
+if [ "${OUTPUT_DIR}" = "" ]; then
+	OUTPUT_DIR=${OUTPUT_DIR:="$(mktemp -d)"}
+	CLEANUP_DIR=1
+fi
+
+#
+# collect some information about the test environment
+#
+
+SYSTEM_OS=$(uname -s)
+if [ "${SYSTEM_OS}" = "Darwin" ]; then SYSTEM_OS="macOS"; fi
+
+SYSTEM_KERNEL=$(uname -v)
+
+fullpath() {
+	if [[ "$(uname -s)" == "MINGW"* && $(cygpath -u "${TEST_CLI}") == "/"* ]]; then
+		echo "${TEST_CLI}"
+	elif [[ "${TEST_CLI}" == "/"* ]]; then
+		echo "${TEST_CLI}"
+	else
+		which "${TEST_CLI}"
+	fi
+}
+
+cli_version() {
+	if [[ "$(uname -s)" == "MINGW"* ]]; then
+		$(cygpath -u "$1") --version
+	else
+		"$1" --version
+	fi
+}
+
+TEST_CLI_NAME=$(basename "${TEST_CLI}")
+TEST_CLI_PATH=$(fullpath "${TEST_CLI}")
+TEST_CLI_VERSION=$(cli_version "${TEST_CLI}")
+
+if [ "${BASELINE_CLI}" != "" ]; then
+	if [[ "${BASELINE_CLI}" == "/"* ]]; then
+		BASELINE_CLI_PATH="${BASELINE_CLI}"
+	else
+		BASELINE_CLI_PATH=$(which "${BASELINE_CLI}")
+	fi
+
+	BASELINE_CLI_NAME=$(basename "${BASELINE_CLI}")
+	BASELINE_CLI_PATH=$(fullpath "${BASELINE_CLI}")
+	BASELINE_CLI_VERSION=$(cli_version "${BASELINE_CLI}")
+fi
+
+#
+# run the benchmarks
+#
+
+echo "##############################################################################"
+if [ "${SUITE}" != "" ]; then
+	SUITE_PREFIX="${SUITE/::/__}"
+	echo "## Running ${SUITE} benchmarks"
+else
+	echo "## Running all benchmarks"
+fi
+echo "##############################################################################"
+echo ""
+
+if [ "${BASELINE_CLI}" != "" ]; then
+	echo "# Baseline CLI: ${BASELINE_CLI} (${BASELINE_CLI_VERSION})"
+fi
+echo "# Test CLI: ${TEST_CLI} (${TEST_CLI_VERSION})"
+echo ""
+
+BENCHMARK_DIR=${BENCHMARK_DIR:=$(dirname "$0")}
+ANY_FOUND=
+ANY_FAILED=
+
+indent() { sed "s/^/  /"; }
+time_in_ms() { if [ "$(uname -s)" = "Darwin" ]; then date "+%s000"; else date "+%s%N" ; fi; }
+humanize_secs() {
+	units=('s' 'ms' 'us' 'ns')
+	unit=0
+	time="${1}"
+
+	if [ "${time}" = "" ]; then
+		echo ""
+		return
+	fi
+
+	# bash doesn't do floating point arithmetic.  ick.
+	while [[ "${time}" == "0."* ]] && [ "$((unit+1))" != "${#units[*]}" ]; do
+		time="$(echo | awk "{ print ${time} * 1000 }")"
+		unit=$((unit+1))
+	done
+
+	echo "${time} ${units[$unit]}"
+}
+
+TIME_START=$(time_in_ms)
+
+for TEST_PATH in "${BENCHMARK_DIR}"/*; do
+	TEST_FILE=$(basename "${TEST_PATH}")
+
+	if [ ! -f "${TEST_PATH}" ] || [ ! -x "${TEST_PATH}" ]; then
+		continue
+	fi
+
+	if [[ "${TEST_FILE}" != *"__"* ]]; then
+		continue
+	fi
+
+	if [[ "${TEST_FILE}" != "${SUITE_PREFIX}"* ]]; then
+		continue
+	fi
+
+	ANY_FOUND=1
+	TEST_NAME="${TEST_FILE/__/::}"
+
+	echo -n "${TEST_NAME}:"
+	if [ "${VERBOSE}" = "1" ]; then
+		echo ""
+	else
+		echo -n "  "
+	fi
+
+	if [ "${DEBUG}" = "1" ]; then
+		SHOW_OUTPUT="--show-output"
+	fi
+
+	OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}.out"
+	JSON_FILE="${OUTPUT_DIR}/${TEST_FILE}.json"
+	ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}.err"
+
+	FAILED=
+	${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}" || FAILED=1
+
+	if [ "${FAILED}" = "1" ]; then
+		if [ "${VERBOSE}" != "1" ]; then
+			echo "failed!"
+		fi
+
+		indent < "${ERROR_FILE}"
+		ANY_FAILED=1
+		continue
+	fi
+
+	# in verbose mode, just print the hyperfine results; otherwise,
+	# pull the useful information out of its json and summarize it
+	if [ "${VERBOSE}" = "1" ]; then
+		indent < "${OUTPUT_FILE}"
+	else
+		jq -r '[ .results[0].mean, .results[0].stddev, .results[1].mean, .results[1].stddev ] | @tsv' < "${JSON_FILE}" | while IFS=$'\t' read -r one_mean one_stddev two_mean two_stddev; do
+			one_mean=$(humanize_secs "${one_mean}")
+			one_stddev=$(humanize_secs "${one_stddev}")
+
+			if [ "${two_mean}" != "" ]; then
+				two_mean=$(humanize_secs "${two_mean}")
+				two_stddev=$(humanize_secs "${two_stddev}")
+
+				echo "${one_mean} ± ${one_stddev}  vs  ${two_mean} ± ${two_stddev}"
+			else
+				echo "${one_mean} ± ${one_stddev}"
+			fi
+		done
+	fi
+
+	# add our metadata to the hyperfine json result
+	jq ". |= { \"name\": \"${TEST_NAME}\" } + ." < "${JSON_FILE}" > "${JSON_FILE}.new" && mv "${JSON_FILE}.new" "${JSON_FILE}"
+done
+
+TIME_END=$(time_in_ms)
+
+if [ "$ANY_FOUND" != "1" ]; then
+	echo ""
+	echo "error: no benchmark suite \"${SUITE}\"."
+	echo ""
+	exit 1
+fi
+
+escape() {
+	echo "${1//\\/\\\\}"
+}
+
+# combine all the individual benchmark results into a single json file
+if [ "${JSON_RESULT}" != "" ]; then
+	if [ "${VERBOSE}" = "1" ]; then
+		echo ""
+		echo "# Writing JSON results: ${JSON_RESULT}"
+	fi
+
+	SYSTEM_JSON="{ \"os\": \"${SYSTEM_OS}\",  \"kernel\": \"${SYSTEM_KERNEL}\" }"
+	TIME_JSON="{ \"start\": ${TIME_START}, \"end\": ${TIME_END} }"
+	TEST_CLI_JSON="{ \"name\": \"${TEST_CLI_NAME}\", \"path\": \"$(escape "${TEST_CLI_PATH}")\", \"version\": \"${TEST_CLI_VERSION}\" }"
+	BASELINE_CLI_JSON="{ \"name\": \"${BASELINE_CLI_NAME}\", \"path\": \"$(escape "${BASELINE_CLI_PATH}")\", \"version\": \"${BASELINE_CLI_VERSION}\" }"
+
+	if [ "${BASELINE_CLI}" != "" ]; then
+		EXECUTOR_JSON="{ \"baseline\": ${BASELINE_CLI_JSON}, \"cli\": ${TEST_CLI_JSON} }"
+	else
+		EXECUTOR_JSON="{ \"cli\": ${TEST_CLI_JSON} }"
+	fi
+
+	# add our metadata to all the test results
+	jq -n "{ \"system\": ${SYSTEM_JSON}, \"time\": ${TIME_JSON}, \"executor\": ${EXECUTOR_JSON}, \"tests\": [inputs] }" "${OUTPUT_DIR}"/*.json > "${JSON_RESULT}"
+fi
+
+# combine all the data into a zip if requested
+if [ "${ZIP_RESULT}" != "" ]; then
+	if [ "${VERBOSE}" = "1" ]; then
+		if [ "${JSON_RESULT}" = "" ]; then echo ""; fi
+		echo "# Writing ZIP results: ${ZIP_RESULT}"
+	fi
+
+	zip -jr "${ZIP_RESULT}" "${OUTPUT_DIR}" >/dev/null
+fi
+
+if [ "$CLEANUP_DIR" = "1" ]; then
+	rm -f "${OUTPUT_DIR}"/*.out
+	rm -f "${OUTPUT_DIR}"/*.err
+	rm -f "${OUTPUT_DIR}"/*.json
+	rmdir "${OUTPUT_DIR}"
+fi
+
+if [ "$ANY_FAILED" = "1" ]; then
+	exit 1
+fi