PR27531: retry within default retry_limit will be supported.

In debuginfod-client.c (debuginfod_query_server),insert a
goto statement for jumping back to the beginning of curl
handles set up if query fails and a non ENOENT error is returned.

Also introduced DEBUGINFOD_RETRY_LIMIT_ENV_VAR and default
DEBUGINFOD_RETRY_LIMIT(which is 2).

Correponding test has been added to tests/run-debuginfod-find.sh

Signed-off-by: Alice Zhang <alizhang@redhat.com>

8 files changed, 68 insertions, 3 deletions

diff --git a/config/ChangeLog b/config/ChangeLog
index 2cdcfa72..70a1e923 100644
--- a/config/ChangeLog
+++ b/config/ChangeLog
@@ -1,3 +1,7 @@
+2021-07-06  Alice Zhang  <alizhang@redhat.com>
+
+        * debuginfod.sysconfig: Introduce default retry limit.
+
 2021-05-10  Mark Wielaard  <mark@klomp.org>
 
 	* elfutils.spec.in: Update for 0.185.
diff --git a/config/debuginfod.sysconfig b/config/debuginfod.sysconfig
index 44603874..890a1a25 100644
--- a/config/debuginfod.sysconfig
+++ b/config/debuginfod.sysconfig
@@ -11,4 +11,5 @@ DEBUGINFOD_PATHS="-t43200 -F -R /usr/lib/debug /usr/bin /usr/libexec /usr/sbin /
 # upstream debuginfods
 #DEBUGINFOD_URLS="http://secondhost:8002 http://thirdhost:8002"
 #DEBUGINFOD_TIMEOUT="5"
+#DEBUGINFOD_RETRY_LIMIT="2"
 #DEBUGINFOD_CACHE_DIR=""
diff --git a/debuginfod/ChangeLog b/debuginfod/ChangeLog
index a4f20a78..e71436ca 100644
--- a/debuginfod/ChangeLog
+++ b/debuginfod/ChangeLog
@@ -1,3 +1,10 @@
+2021-07-06  Alice Zhang  <alizhang@redhat.com>
+
+        PR27531
+	* debuginfod-client.c (debuginfod_query_server): Retry failed queries
+	if error code is not ENOENT.
+	* debuginfod.h.in: Introduce DEBUGINFOD_RETRY_LIMIT_ENV_VAR.
+
 2021-07-01  Noah Sanci <nsanci@redhat.com>
 
 	PR27711
diff --git a/debuginfod/debuginfod-client.c b/debuginfod/debuginfod-client.c
index f417b40a..f12f609c 100644
--- a/debuginfod/debuginfod-client.c
+++ b/debuginfod/debuginfod-client.c
@@ -157,6 +157,8 @@ static const char url_delim_char = ' ';
 /* Timeout for debuginfods, in seconds (to get at least 100K). */
 static const long default_timeout = 90;
 
+/* Default retry count for download error. */
+static const long default_retry_limit = 2;
 
 /* Data associated with a particular CURL easy handle. Passed to
    the write callback.  */
@@ -770,9 +772,14 @@ debuginfod_query_server (debuginfod_client *c,
         && (i == 0 || server_urls[i - 1] == url_delim_char))
       num_urls++;
   
+  int retry_limit = default_retry_limit;
+  const char* retry_limit_envvar = getenv(DEBUGINFOD_RETRY_LIMIT_ENV_VAR);
+  if (retry_limit_envvar != NULL)
+    retry_limit = atoi (retry_limit_envvar);
+
   CURLM *curlm = c->server_mhandle;
   assert (curlm != NULL);
-  
+
   /* Tracks which handle should write to fd. Set to the first
      handle that is ready to write the target file to the cache.  */
   CURL *target_handle = NULL;
@@ -785,6 +792,10 @@ debuginfod_query_server (debuginfod_client *c,
 
   /* thereafter, goto out1 on error.  */
 
+ /*The beginning of goto block query_in_parallel.*/
+ query_in_parallel:
+  rc = -ENOENT; /* Reset rc to default.*/
+
   /* Initialize handle_data with default values. */
   for (int i = 0; i < num_urls; i++)
     {
@@ -1074,8 +1085,27 @@ debuginfod_query_server (debuginfod_client *c,
         close(efd);
     }
 
+  /* If the verified_handle is NULL and rc != -ENOENT, the query fails with
+   * an error code other than 404, then do several retry within the retry_limit.
+   * Clean up all old handles and jump back to the beginning of query_in_parallel,
+   * reinitialize handles and query again.*/
   if (verified_handle == NULL)
-    goto out1;
+    {
+      if (rc != -ENOENT && retry_limit-- > 0)
+        {
+	  if (vfd >= 0)
+            dprintf (vfd, "Retry failed query, %d attempt(s) remaining\n", retry_limit);
+	  /* remove all handles from multi */
+          for (int i = 0; i < num_urls; i++)
+            {
+              curl_multi_remove_handle(curlm, data[i].handle); /* ok to repeat */
+              curl_easy_cleanup (data[i].handle);
+            }
+	    goto query_in_parallel;
+	}
+      else
+	goto out1;
+    }
 
   if (vfd >= 0)
     {
diff --git a/debuginfod/debuginfod.h.in b/debuginfod/debuginfod.h.in
index 559ea947..282e523d 100644
--- a/debuginfod/debuginfod.h.in
+++ b/debuginfod/debuginfod.h.in
@@ -35,6 +35,7 @@
 #define DEBUGINFOD_TIMEOUT_ENV_VAR "DEBUGINFOD_TIMEOUT"
 #define DEBUGINFOD_PROGRESS_ENV_VAR "DEBUGINFOD_PROGRESS"
 #define DEBUGINFOD_VERBOSE_ENV_VAR "DEBUGINFOD_VERBOSE"
+#define DEBUGINFOD_RETRY_LIMIT_ENV_VAR "DEBUGINFOD_RETRY_LIMIT"
 
 /* The libdebuginfod soname.  */
 #define DEBUGINFOD_SONAME "@LIBDEBUGINFOD_SONAME@"
diff --git a/doc/debuginfod_find_debuginfo.3 b/doc/debuginfod_find_debuginfo.3
index 5ae44a98..7730dd32 100644
--- a/doc/debuginfod_find_debuginfo.3
+++ b/doc/debuginfod_find_debuginfo.3
@@ -277,6 +277,11 @@ program is reexecuted. If XDG_CACHE_HOME is set then
 $XDG_CACHE_HOME/debuginfod_client is the default location, otherwise
 $HOME/.cache/debuginfod_client is used.
 
+.TP 21
+.B DEBUGINFOD_RETRY_LITMIT
+This environment variable governs the default limit of retry attempts. If a
+query failed with errno other than ENOENT, will initiate several attempts
+within the limit.
 
 .SH "ERRORS"
 The following list is not comprehensive. Error codes may also
diff --git a/tests/ChangeLog b/tests/ChangeLog
index 3a30e06b..1460b422 100644
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,8 @@
+2021-07-06  Alice Zhang  <alizhang@redhat.com>
+
+        PR27531
+	* run-debuginfod-find.sh: Add test case for retry mechanism.
+
 2021-07-01  Noah Sanci <nsanci@redhat.com>
 
 	PR2711
diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh
index a4c1a13a..73bbe65b 100755
--- a/tests/run-debuginfod-find.sh
+++ b/tests/run-debuginfod-find.sh
@@ -620,6 +620,7 @@ curl -s http://127.0.0.1:$PORT1/metrics | grep 'http_responses_after_you.*'
 
 ########################################################################
 # Corrupt the sqlite database and get debuginfod to trip across its errors
+
 curl -s http://127.0.0.1:$PORT1/metrics | grep 'sqlite3.*reset'
 ls -al $DB
 dd if=/dev/zero of=$DB bs=1 count=1
@@ -737,4 +738,15 @@ wait_ready $PORT3 'groom{statistic="files scanned (#)"}' 0
 wait_ready $PORT3 'groom{statistic="files scanned (mb)"}' 0
 
 kill $PID4
-exit 0;
+
+########################################################################
+# set up tests for retrying failed queries.
+retry_attempts=`(testrun env DEBUGINFOD_URLS=http://255.255.255.255/JUNKJUNK DEBUGINFOD_RETRY_LIMIT=10 DEBUGINFOD_VERBOSE=1 \
+	${abs_top_builddir}/debuginfod/debuginfod-find debuginfo /bin/ls || true) 2>&1 >/dev/null \
+	| grep -c 'Retry failed query'`
+if [ $retry_attempts -ne 10 ]; then
+    echo "retry mechanism failed."
+    exit 1;
+  fi
+
+exit 0