From 10331a1a3dcef3ca3b054eab5871c2e71becb045 Mon Sep 17 00:00:00 2001
From: Fred Nicolson <fred.nicolson@outlook.com>
Date: Wed, 7 Nov 2018 15:57:33 +0000
Subject: [PATCH] Replaced log locking using mutex with fcntl lock

When reloading Nginx, there is a race condition which is visible under high load. As the logging mutex is shared between multiple workers, when a worker is sent a stop signal during a reload, and the log mutex is held, write() will never return, which means that the mutex will never unlock. As other workers share this mutex, they will deadlock.

fcntl does not suffer from this issue.
---
 src/utils/shared_files.cc | 16 +++++++++++-----
 src/utils/shared_files.h  |  1 -
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/utils/shared_files.cc b/src/utils/shared_files.cc
index 4830d1bd95..4c76ba3e52 100644
--- a/src/utils/shared_files.cc
+++ b/src/utils/shared_files.cc
@@ -104,7 +104,6 @@ std::pair<msc_file_handler *, FILE *> SharedFiles::add_new_handler(
 
     if (toBeCreated) {
         memset(new_debug_log, '\0', sizeof(msc_file_handler_t));
-        pthread_mutex_init(&new_debug_log->lock, NULL);
         new_debug_log->shm_id_structure = shm_id;
         memcpy(new_debug_log->file_name, fileName.c_str(), fileName.size());
         new_debug_log->file_name[fileName.size()] = '\0';
@@ -222,6 +221,7 @@ bool SharedFiles::write(const std::string& fileName,
     std::pair<msc_file_handler *, FILE *> a;
     std::string lmsg = msg;
     size_t wrote;
+    struct flock lock{};
     bool ret = true;
 
     a = find_handler(fileName);
@@ -230,15 +230,21 @@ bool SharedFiles::write(const std::string& fileName,
         return false;
     }
 
-    pthread_mutex_lock(&a.first->lock);
-    wrote = fwrite(reinterpret_cast<const char *>(lmsg.c_str()), 1,
-        lmsg.size(), a.second);
+    //Exclusively lock whole file
+    lock.l_start = lock.l_len = lock.l_whence = 0;
+    lock.l_type = F_WRLCK;
+    fcntl(fileno(a.second), F_SETLKW, &lock);
+
+    wrote = fwrite(lmsg.c_str(), 1, lmsg.size(), a.second);
     if (wrote < msg.size()) {
         error->assign("failed to write: " + fileName);
         ret = false;
     }
     fflush(a.second);
-    pthread_mutex_unlock(&a.first->lock);
+
+    //Remove exclusive lock
+    lock.l_type = F_UNLCK;
+    fcntl(fileno(a.second), F_SETLKW, &lock);
 
     return ret;
 }
diff --git a/src/utils/shared_files.h b/src/utils/shared_files.h
index 7631656786..47f06d2d69 100644
--- a/src/utils/shared_files.h
+++ b/src/utils/shared_files.h
@@ -50,7 +50,6 @@ namespace utils {
 
 typedef struct msc_file_handler {
     int shm_id_structure;
-    pthread_mutex_t lock;
     char file_name[];
 } msc_file_handler_t;