Bugzilla bug #70295: do not hold _pr_flock_lock while calling the MD

lockfile function to avoid a deadlock.  The lockCount field of
PRFilePrivate may have the value of -1 to indicate that the MD lockfile
function is in progress.  Use _pr_flock_cv to wait for the completion
of the MD lockfile function.
Modified files: primpl.h, prfile.c, prio.c, ptio.c

4 files changed, 31 insertions, 4 deletions

diff --git a/pr/include/private/primpl.h b/pr/include/private/primpl.h
index d4c52cec..b3bf72d2 100644
--- a/pr/include/private/primpl.h
+++ b/pr/include/private/primpl.h
@@ -1696,7 +1696,9 @@ struct PRFilePrivate {
     PRBool nonblocking;
     _PRTriStateBool inheritable;
     PRFileDesc *next;
-    PRIntn lockCount;
+    PRIntn lockCount;   /*   0: not locked
+                         *  -1: a native lockfile call is in progress
+                         * > 0: # times the file is locked */
 #ifdef _PR_HAVE_PEEK_BUFFER
     char *peekBuffer;
     PRInt32 peekBufSize;
diff --git a/pr/src/io/prfile.c b/pr/src/io/prfile.c
index d7aa68fe..e1932307 100644
--- a/pr/src/io/prfile.c
+++ b/pr/src/io/prfile.c
@@ -49,6 +49,7 @@
 #endif /* XP_UNIX */
 
 extern PRLock *_pr_flock_lock;
+extern PRCondVar *_pr_flock_cv;
 
 static PRInt32 PR_CALLBACK FileRead(PRFileDesc *fd, void *buf, PRInt32 amount)
 {
@@ -633,10 +634,17 @@ PR_IMPLEMENT(PRStatus) PR_LockFile(PRFileDesc *fd)
 
     PR_Lock(_pr_flock_lock);
     if (fd->secret->lockCount == 0) {
+        fd->secret->lockCount == -1;
+        PR_Unlock(_pr_flock_lock);
         status = _PR_MD_LOCKFILE(fd->secret->md.osfd);
-        if (status == PR_SUCCESS)
+        PR_Lock(_pr_flock_lock);
+        if (status == PR_SUCCESS) {
             fd->secret->lockCount = 1;
+            PR_NotifyAllCondVar(_pr_flock_cv);
+        }
     } else {
+        while (fd->secret->lockCount == -1)
+            PR_WaitCondVar(_pr_flock_cv, PR_INTERVAL_NO_TIMEOUT);
         fd->secret->lockCount++;
     }
     PR_Unlock(_pr_flock_lock);
diff --git a/pr/src/io/prio.c b/pr/src/io/prio.c
index e03fb622..ce5c021b 100644
--- a/pr/src/io/prio.c
+++ b/pr/src/io/prio.c
@@ -40,6 +40,7 @@
 /************************************************************************/
 
 PRLock *_pr_flock_lock;
+PRCondVar *_pr_flock_cv;
 
 void _PR_InitIO(void)
 {
@@ -48,6 +49,7 @@ void _PR_InitIO(void)
     _PR_InitFdCache();
 
     _pr_flock_lock = PR_NewLock();
+    _pr_flock_cv = PR_NewCondVar(_pr_flock_lock);
 
 #ifdef WIN32
     _pr_stdin = PR_AllocFileDesc((PRInt32)GetStdHandle(STD_INPUT_HANDLE),
diff --git a/pr/src/pthreads/ptio.c b/pr/src/pthreads/ptio.c
index 093832ad..121be8d6 100644
--- a/pr/src/pthreads/ptio.c
+++ b/pr/src/pthreads/ptio.c
@@ -208,6 +208,7 @@ static PRFileDesc *pt_SetMethods(
     PRIntn osfd, PRDescType type, PRBool isAcceptedSocket, PRBool imported);
 
 static PRLock *_pr_flock_lock;  /* For PR_LockFile() etc. */
+static PRCondVar *_pr_flock_cv;  /* For PR_LockFile() etc. */
 static PRLock *_pr_rename_lock;  /* For PR_Rename() */
 
 /**************************************************************************/
@@ -1052,6 +1053,8 @@ void _PR_InitIO()
 
     _pr_flock_lock = PR_NewLock();
     PR_ASSERT(NULL != _pr_flock_lock);
+    _pr_flock_cv = PR_NewCondVar(_pr_flock_lock);
+    PR_ASSERT(NULL != _pr_flock_cv);
     _pr_rename_lock = PR_NewLock();
     PR_ASSERT(NULL != _pr_rename_lock); 
 
@@ -4162,10 +4165,22 @@ PR_IMPLEMENT(PRStatus) PR_LockFile(PRFileDesc *fd)
     PR_Lock(_pr_flock_lock);
     if (0 == fd->secret->lockCount)
     {
+        fd->secret->lockCount = -1;
+        PR_Unlock(_pr_flock_lock);
         status = _PR_MD_LOCKFILE(fd->secret->md.osfd);
-        if (PR_SUCCESS == status) fd->secret->lockCount = 1;
+        PR_Lock(_pr_flock_lock);
+        if (PR_SUCCESS == status)
+        {
+            fd->secret->lockCount = 1;
+            PR_NotifyAllCondVar(_pr_flock_cv);
+        }
+    }
+    else
+    {
+        while (-1 == fd->secret->lockCount)
+            PR_WaitCondVar(_pr_flock_cv, PR_INTERVAL_NO_TIMEOUT);
+        fd->secret->lockCount += 1;
     }
-    else fd->secret->lockCount += 1;
     PR_Unlock(_pr_flock_lock);
  
     return status;