Merge pull request #2523 from xrmx/harakiri-graceful-master

core: add new flags for harakiri configuration:

6 files changed, 83 insertions, 15 deletions

diff --git a/core/init.c b/core/init.c
index 2c744e00..d8ef00da 100644
--- a/core/init.c
+++ b/core/init.c
@@ -205,6 +205,8 @@ void uwsgi_init_default() {
 	uwsgi.notify_socket_fd = -1;
 
 	uwsgi.mule_msg_recv_size = 65536;
+
+	uwsgi.harakiri_graceful_signal = SIGTERM;
 }
 
 void uwsgi_setup_reload() {
diff --git a/core/master_checks.c b/core/master_checks.c
index 3e6b76a0..9fdffd4b 100644
--- a/core/master_checks.c
+++ b/core/master_checks.c
@@ -180,28 +180,57 @@ done:
 
 }
 
+int uwsgi_master_check_harakiri(int w, int c, int harakiri) {
+	/**
+	 * Triggers a harakiri when the following conditions are met:
+	 * - harakiri timeout > current time
+	 * - listen queue pressure (ie backlog > harakiri_queue_threshold)
+	 *
+	 * The first harakiri attempt on a worker will be graceful if harakiri_graceful_timeout > 0,
+	 * then the worker has harakiri_graceful_timeout seconds to shutdown cleanly, otherwise
+	 * a second harakiri will trigger a SIGKILL
+	 *
+	 */
+#ifdef __linux__
+	int backlog = uwsgi.shared->backlog;
+#else
+	int backlog = 0;
+#endif
+	if (harakiri == 0 || harakiri > (time_t) uwsgi.current_time) {
+		return 0;
+	}
+	// no pending harakiri for the worker and no backlog pressure, safe to skip
+	if (uwsgi.workers[w].pending_harakiri == 0 &&  backlog < uwsgi.harakiri_queue_threshold) {
+		uwsgi_log_verbose("HARAKIRI: Skipping harakiri on worker %d. Listen queue is smaller than the threshold (%d < %d)\n",
+			w, backlog, uwsgi.harakiri_queue_threshold);
+		return 0;
+	}
+
+	trigger_harakiri(w);
+	if (uwsgi.harakiri_graceful_timeout > 0) {
+		uwsgi.workers[w].cores[c].harakiri = harakiri + uwsgi.harakiri_graceful_timeout;
+		uwsgi_log_verbose("HARAKIRI: graceful termination attempt on worker %d with signal %d. Next harakiri: %d\n",
+			w, uwsgi.harakiri_graceful_signal, uwsgi.workers[w].cores[c].harakiri);
+	}
+	return 1;
+}
+
 int uwsgi_master_check_workers_deadline() {
 	int i,j;
 	int ret = 0;
 	for (i = 1; i <= uwsgi.numproc; i++) {
 		for(j=0;j<uwsgi.cores;j++) {
 			/* first check for harakiri */
-			if (uwsgi.workers[i].cores[j].harakiri > 0) {
-				if (uwsgi.workers[i].cores[j].harakiri < (time_t) uwsgi.current_time) {
-					uwsgi_log_verbose("HARAKIRI triggered by worker %d core %d !!!\n", i, j);
-					trigger_harakiri(i);
-					ret = 1;
-					break;
-				}
+			if (uwsgi_master_check_harakiri(i, j, uwsgi.workers[i].cores[j].harakiri)) {
+				uwsgi_log_verbose("HARAKIRI triggered by worker %d core %d !!!\n", i, j);
+				ret = 1;
+				break;
 			}
 			/* then user-defined harakiri */
-			if (uwsgi.workers[i].cores[j].user_harakiri > 0) {
+			if (uwsgi_master_check_harakiri(i, j, uwsgi.workers[i].cores[j].user_harakiri)) {
 				uwsgi_log_verbose("HARAKIRI (user) triggered by worker %d core %d !!!\n", i, j);
-				if (uwsgi.workers[i].cores[j].user_harakiri < (time_t) uwsgi.current_time) {
-					trigger_harakiri(i);
-					ret = 1;
-					break;
-				}
+				ret = 1;
+				break;
 			}
 		}
 		// then for evil memory checkers
diff --git a/core/master_utils.c b/core/master_utils.c
index 9a54b35e..22977c07 100644
--- a/core/master_utils.c
+++ b/core/master_utils.c
@@ -1624,7 +1624,10 @@ void uwsgi_register_cheaper_algo(char *name, int (*func) (int)) {
 
 void trigger_harakiri(int i) {
 	int j;
-	uwsgi_log_verbose("*** HARAKIRI ON WORKER %d (pid: %d, try: %d) ***\n", i, uwsgi.workers[i].pid, uwsgi.workers[i].pending_harakiri + 1);
+	uwsgi_log_verbose("*** HARAKIRI ON WORKER %d (pid: %d, try: %d, graceful: %s) ***\n", i,
+				uwsgi.workers[i].pid,
+				uwsgi.workers[i].pending_harakiri + 1,
+				uwsgi.workers[i].pending_harakiri > 0 ? "no": "yes");
 	if (uwsgi.harakiri_verbose) {
 #ifdef __linux__
 		int proc_file;
@@ -1673,7 +1676,11 @@ void trigger_harakiri(int i) {
 		}
 
 		uwsgi_dump_worker(i, "HARAKIRI");
-		kill(uwsgi.workers[i].pid, SIGKILL);
+		if (uwsgi.workers[i].pending_harakiri == 0 && uwsgi.harakiri_graceful_timeout > 0) {
+			kill(uwsgi.workers[i].pid, uwsgi.harakiri_graceful_signal);
+		} else {
+			kill(uwsgi.workers[i].pid, SIGKILL);
+		}
 		if (!uwsgi.workers[i].pending_harakiri)
 			uwsgi.workers[i].harakiri_count++;
 		uwsgi.workers[i].pending_harakiri++;
diff --git a/core/uwsgi.c b/core/uwsgi.c
index d7bf318b..16137f6e 100755
--- a/core/uwsgi.c
+++ b/core/uwsgi.c
@@ -60,6 +60,9 @@ static struct uwsgi_option uwsgi_base_options[] = {
 	{"thunder-lock-watchdog", no_argument, 0, "watchdog for buggy pthread robust mutexes", uwsgi_opt_true, &uwsgi.use_thunder_lock_watchdog, 0},
 	{"harakiri", required_argument, 't', "set harakiri timeout", uwsgi_opt_set_int, &uwsgi.harakiri_options.workers, 0},
 	{"harakiri-verbose", no_argument, 0, "enable verbose mode for harakiri", uwsgi_opt_true, &uwsgi.harakiri_verbose, 0},
+	{"harakiri-graceful-timeout", required_argument, 0, "interval between graceful harakiri attempt and a sigkill", uwsgi_opt_set_int, &uwsgi.harakiri_graceful_timeout, 0},
+	{"harakiri-graceful-signal", required_argument, 0, "use this signal instead of sigterm for graceful harakiri attempts", uwsgi_opt_set_int, &uwsgi.harakiri_graceful_signal, 0},
+	{"harakiri-queue-threshold", required_argument, 0, "only trigger harakiri if queue is greater than this threshold", uwsgi_opt_set_int, &uwsgi.harakiri_queue_threshold, 0},
 	{"harakiri-no-arh", no_argument, 0, "do not enable harakiri during after-request-hook", uwsgi_opt_true, &uwsgi.harakiri_no_arh, 0},
 	{"no-harakiri-arh", no_argument, 0, "do not enable harakiri during after-request-hook", uwsgi_opt_true, &uwsgi.harakiri_no_arh, 0},
 	{"no-harakiri-after-req-hook", no_argument, 0, "do not enable harakiri during after-request-hook", uwsgi_opt_true, &uwsgi.harakiri_no_arh, 0},
diff --git a/tests/harakiri.py b/tests/harakiri.py
new file mode 100644
index 00000000..81085a4e
--- /dev/null
+++ b/tests/harakiri.py
@@ -0,0 +1,24 @@
+import time
+import uwsgi
+import signal
+import sys
+import atexit
+
+def sig_handler(n, fp):
+    print("[Python App] attempting graceful shutdown triggered by harakiri (signal %d)" % n)
+    exit(1)
+
+def application(e, s):
+    print("[Python App] sleeping")
+    time.sleep(3)
+    s('200 OK', [('Content-Type', 'text/html')])
+    return [b"OK"]
+
+
+def exit_handler():
+    time.sleep(3)
+    # Should not reach this line (graceful harakiri deadline expired)
+    print("[Python App] exiting now")
+
+atexit.register(exit_handler)
+signal.signal(signal.SIGSYS, sig_handler)
diff --git a/uwsgi.h b/uwsgi.h
index 5fd6528f..1980bc23 100755
--- a/uwsgi.h
+++ b/uwsgi.h
@@ -2486,6 +2486,9 @@ struct uwsgi_server {
 	int backtrace_depth;
 
 	int harakiri_verbose;
+	int harakiri_graceful_timeout;
+	int harakiri_graceful_signal;
+	int harakiri_queue_threshold;
 	int harakiri_no_arh;
 
 	int magic_table_first_round;