diff --git a/src/common/isc_s_proto.h b/src/common/isc_s_proto.h index e10e201dc2..88f0da8e31 100644 --- a/src/common/isc_s_proto.h +++ b/src/common/isc_s_proto.h @@ -178,7 +178,7 @@ public: #define USE_FCNTL #endif -class CountedFd; +class SharedFileInfo; class FileLock { @@ -186,8 +186,8 @@ public: enum LockMode {FLM_EXCLUSIVE, FLM_TRY_EXCLUSIVE, FLM_SHARED, FLM_TRY_SHARED}; typedef void InitFunction(int fd); - explicit FileLock(const char* fileName, InitFunction* init = NULL); // main ctor - FileLock(const FileLock* main, int s); // creates additional lock for existing file + + explicit FileLock(const char* fileName, InitFunction* init = NULL); ~FileLock(); // Main function to lock file @@ -196,24 +196,18 @@ public: // Alternative locker is using status vector to report errors bool setlock(Firebird::CheckStatusWrapper* status, const LockMode mode); - // unlocking can only put error into log file - we can't throw in dtors + // Unlocking can only put error into log file - we can't throw in dtors void unlock(); + // Obvious access to file descriptor int getFd(); -private: enum LockLevel {LCK_NONE, LCK_SHARED, LCK_EXCL}; +private: + Firebird::RefPtr file; + InitFunction* initFunction; LockLevel level; - CountedFd* oFile; -#ifdef USE_FCNTL - int lStart; -#endif - class CountedRWLock* rwcl; // Due to order of init in ctor rwcl must go after fd & start - - Firebird::string getLockId(); - class CountedRWLock* getRw(); - void rwUnlock(); }; #endif // UNIX diff --git a/src/common/isc_sync.cpp b/src/common/isc_sync.cpp index da3b277d94..5405c1c4a6 100644 --- a/src/common/isc_sync.cpp +++ b/src/common/isc_sync.cpp @@ -68,6 +68,7 @@ #include "../common/classes/GenericMap.h" #include "../common/classes/RefMutex.h" #include "../common/classes/array.h" +#include "../common/classes/condition.h" #include "../common/StatusHolder.h" static int process_id; @@ -142,87 +143,26 @@ static bool event_blocked(const event_t* event, const SLONG value); #ifdef UNIX -static GlobalPtr openFdInit; +#ifdef FILELOCK_DEBUG -class DevNode +#include + +void DEB_FLOCK(const char* format, ...) { -public: - DevNode() - : f_dev(0), f_ino(0) - { } + va_list params; + va_start(params, format); + ::vfprintf(stderr, format, params); + va_end(params); +} - DevNode(dev_t d, ino_t i) - : f_dev(d), f_ino(i) - { } +#else - DevNode(const DevNode& v) - : f_dev(v.f_dev), f_ino(v.f_ino) - { } +void DEB_FLOCK(const char* format, ...) { } - dev_t f_dev; - ino_t f_ino; - - bool operator==(const DevNode& v) const - { - return f_dev == v.f_dev && f_ino == v.f_ino; - } - - bool operator>(const DevNode& v) const - { - return f_dev > v.f_dev ? true : - f_dev < v.f_dev ? false : - f_ino > v.f_ino; - } - - const DevNode& operator=(const DevNode& v) - { - f_dev = v.f_dev; - f_ino = v.f_ino; - return *this; - } -}; - -namespace Firebird { - -class CountedRWLock -{ -public: - CountedRWLock() - : sharedAccessCounter(0) - { } - RWLock rwlock; - AtomicCounter cnt; - Mutex sharedAccessMutex; - int sharedAccessCounter; -}; - -class CountedFd -{ -public: - explicit CountedFd(int f) - : fd(f), useCount(0) - { } - - ~CountedFd() - { - fb_assert(useCount == 0); - } - - int fd; - int useCount; - -private: - CountedFd(const CountedFd&); - const CountedFd& operator=(const CountedFd&); -}; - -} // namespace Firebird +#endif //FILELOCK_DEBUG namespace { - typedef GenericMap > > RWLocks; - GlobalPtr rwlocks; - GlobalPtr rwlocksMutex; #ifdef USE_FCNTL const char* NAME = "fcntl"; #else @@ -253,6 +193,44 @@ namespace { FileLock* lock; }; + class DevNode + { + public: + DevNode() + : f_dev(0), f_ino(0) + { } + + DevNode(dev_t d, ino_t i) + : f_dev(d), f_ino(i) + { } + + DevNode(const DevNode& v) + : f_dev(v.f_dev), f_ino(v.f_ino) + { } + + dev_t f_dev; + ino_t f_ino; + + bool operator==(const DevNode& v) const + { + return f_dev == v.f_dev && f_ino == v.f_ino; + } + + bool operator>(const DevNode& v) const + { + return f_dev > v.f_dev ? true : + f_dev < v.f_dev ? false : + f_ino > v.f_ino; + } + + const DevNode& operator=(const DevNode& v) + { + f_dev = v.f_dev; + f_ino = v.f_ino; + return *this; + } + }; + DevNode getNode(const char* name) { struct STAT statistics; @@ -279,80 +257,231 @@ namespace { return DevNode(statistics.st_dev, statistics.st_ino); } + GlobalPtr openFdInit; + } // anonymous namespace -typedef GenericMap > > FdNodes; -static GlobalPtr fdNodesMutex; -static GlobalPtr fdNodes; +namespace Firebird { + +class SharedFileInfo : public RefCounted +{ + SharedFileInfo(int f, const DevNode& id) + : counter(0), threadId(0), fd(f), devNode(id) + { } + + ~SharedFileInfo() + { + fb_assert(sharedFilesMutex->locked()); + fb_assert(counter == 0); + + DEB_FLOCK("~ %p\n", this); + sharedFiles->remove(devNode); + close(fd); + } + +public: + static SharedFileInfo* get(const char* fileName) + { + DevNode id(getNode(fileName)); + + MutexLockGuard g(sharedFilesMutex, FB_FUNCTION); + + SharedFileInfo* file = nullptr; + if (id.f_ino) + { + SharedFileInfo** got = sharedFiles->get(id); + if (got) + { + file = *got; + DEB_FLOCK("'%s': in map %p\n", fileName, file); + file->assertNonZero(); + } + } + + if (!file) + { + int fd = os_utils::openCreateSharedFile(fileName, 0); + id = getNode(fd); + file = FB_NEW_POOL(*getDefaultMemoryPool()) SharedFileInfo(fd, id); + SharedFileInfo** put = sharedFiles->put(id); + fb_assert(put); + *put = file; + DEB_FLOCK("'%s': new %p\n", fileName, file); + } + + file->addRef(); + return file; + } + + int release() const override + { + // Release should be executed under mutex protection + // in order to modify reference counter & map atomically + MutexLockGuard guard(sharedFilesMutex, FB_FUNCTION); + + return RefCounted::release(); + } + + int lock(bool shared, bool wait, FileLock::InitFunction* init) + { + MutexEnsureUnlock guard(mutex, FB_FUNCTION); + if (wait) + guard.enter(); + else if (!guard.tryEnter()) + return -1; + + DEB_FLOCK("%d lock %p %c%c\n", Thread::getId(), this, shared ? 's' : 'X', wait ? 'W' : 't'); + + while (counter != 0) // file lock belongs to our process + { + // check for compatible locks + if (shared && counter > 0) + { + // one more shared lock + ++counter; + DEB_FLOCK("%d fast %p c=%d\n", Thread::getId(), this, counter); + return 0; + } + if ((!shared) && counter < 0 && threadId == Thread::getId()) + { + // recursive excl lock + --counter; + DEB_FLOCK("%d fast %p c=%d\n", Thread::getId(), this, counter); + return 0; + } + + // non compatible lock needed + // wait for another thread to release a lock + if (!wait) + { + DEB_FLOCK("%d failed internally %p c=%d rc -1\n", Thread::getId(), this, counter); + return -1; + } + + DEB_FLOCK("%d wait %p c=%d\n", Thread::getId(), this, counter); + waitOn.wait(mutex); + } + + // Take lock on a file + fb_assert(counter == 0); +#ifdef USE_FCNTL + struct FLOCK lock; + lock.l_type = shared ? F_RDLCK : F_WRLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 1; + if (fcntl(fd, wait ? F_SETLKW : F_SETLK, &lock) == -1) + { + int rc = errno; + if (!wait && (rc == EACCES || rc == EAGAIN)) + rc = -1; +#else + if (flock(fd, (shared ? LOCK_SH : LOCK_EX) | (wait ? 0 : LOCK_NB))) + { + int rc = errno; + if (!wait && (rc == EWOULDBLOCK)) + rc = -1; +#endif + DEB_FLOCK("%d failed on file %p c=%d rc %d\n", Thread::getId(), this, counter, rc); + return rc; + } + + if (!shared) + { + threadId = Thread::getId(); + + // call init() when needed + if (init && !shared) + init(fd); + } + + // mark lock as taken + counter = shared ? 1 : -1; + DEB_FLOCK("%d filelock %p c=%d\n", Thread::getId(), this, counter); + return 0; + } + + void unlock() + { + fb_assert(counter != 0); + + MutexEnsureUnlock guard(mutex, FB_FUNCTION); + guard.enter(); + + DEB_FLOCK("%d UNlock %p c=%d\n", Thread::getId(), this, counter); + + if (counter < 0) + ++counter; + else + --counter; + + if (counter != 0) + { + DEB_FLOCK("%d done %p c=%d\n", Thread::getId(), this, counter); + return; + } + + // release file lock +#ifdef USE_FCNTL + struct FLOCK lock; + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = 0; + lock.l_len = 1; + + if (fcntl(fd, F_SETLK, &lock) != 0) + { +#else + if (flock(fd, LOCK_UN) != 0) + { +#endif + LocalStatus ls; + CheckStatusWrapper local(&ls); + error(&local, NAME, errno); + iscLogStatus("Unlock error", &local); + } + + DEB_FLOCK("%d file-done %p\n", Thread::getId(), this); + waitOn.notifyAll(); + } + + int getFd() + { + return fd; + } + +private: + typedef GenericMap > > SharedFiles; + static GlobalPtr sharedFiles; + static GlobalPtr sharedFilesMutex; + + Condition waitOn; + Mutex mutex; + int counter; + ThreadId threadId; + int fd; + DevNode devNode; +}; + +GlobalPtr SharedFileInfo::sharedFiles; +GlobalPtr SharedFileInfo::sharedFilesMutex; + +} // namespace Firebird + FileLock::FileLock(const char* fileName, InitFunction* init) - : level(LCK_NONE), oFile(NULL), -#ifdef USE_FCNTL - lStart(0), -#endif - rwcl(NULL) -{ - MutexLockGuard g(fdNodesMutex, FB_FUNCTION); - - DevNode id(getNode(fileName)); - - if (id.f_ino) - { - CountedFd** got = fdNodes->get(id); - if (got) - { - oFile = *got; - } - } - - if (!oFile) - { - int fd = os_utils::openCreateSharedFile(fileName, 0); - oFile = FB_NEW_POOL(*getDefaultMemoryPool()) CountedFd(fd); - CountedFd** put = fdNodes->put(getNode(fd)); - fb_assert(put); - *put = oFile; - - if (init) - { - init(fd); - } - } - - rwcl = getRw(); - ++(oFile->useCount); -} - + : file(REF_NO_INCR, SharedFileInfo::get(fileName)), initFunction(init), level(LCK_NONE) +{ } FileLock::~FileLock() { unlock(); - - { // guard scope - MutexLockGuard g(rwlocksMutex, FB_FUNCTION); - - if (--(rwcl->cnt) == 0) - { - rwlocks->remove(getLockId()); - delete rwcl; - } - } - { // guard scope - MutexLockGuard g(fdNodesMutex, FB_FUNCTION); - - if (--(oFile->useCount) == 0) - { - fdNodes->remove(getNode(oFile->fd)); - close(oFile->fd); - delete oFile; - } - } } int FileLock::getFd() { - return oFile->fd; + return file->getFd(); } int FileLock::setlock(const LockMode mode) @@ -383,99 +512,11 @@ int FileLock::setlock(const LockMode mode) return wait ? EBUSY : -1; } - // first take appropriate rwlock to avoid conflicts with other threads in our process - bool rc = true; - try - { - switch (mode) - { - case FLM_TRY_EXCLUSIVE: - rc = rwcl->rwlock.tryBeginWrite(FB_FUNCTION); - break; - case FLM_EXCLUSIVE: - rwcl->rwlock.beginWrite(FB_FUNCTION); - break; - case FLM_TRY_SHARED: - rc = rwcl->rwlock.tryBeginRead(FB_FUNCTION); - break; - case FLM_SHARED: - rwcl->rwlock.beginRead(FB_FUNCTION); - break; - } - } - catch (const system_call_failed& fail) - { - return fail.getErrorCode(); - } - if (!rc) - { - return -1; - } + int rc = file->lock(shared, wait, initFunction); + if (rc == 0) // lock taken + level = newLevel; - // For shared lock we must take into an account reenterability - MutexEnsureUnlock guard(rwcl->sharedAccessMutex, FB_FUNCTION); - if (shared) - { - if (wait) - { - guard.enter(); - } - else if (!guard.tryEnter()) - { - return -1; - } - - fb_assert(rwcl->sharedAccessCounter >= 0); - if (rwcl->sharedAccessCounter++ > 0) - { - // counter is non-zero - we already have file lock - level = LCK_SHARED; - return 0; - } - } - -#ifdef USE_FCNTL - // Take lock on a file - struct FLOCK lock; - lock.l_type = shared ? F_RDLCK : F_WRLCK; - lock.l_whence = SEEK_SET; - lock.l_start = lStart; - lock.l_len = 1; - if (fcntl(oFile->fd, wait ? F_SETLKW : F_SETLK, &lock) == -1) - { - int rc = errno; - if (!wait && (rc == EACCES || rc == EAGAIN)) - { - rc = -1; - } -#else - if (flock(oFile->fd, (shared ? LOCK_SH : LOCK_EX) | (wait ? 0 : LOCK_NB))) - { - int rc = errno; - if (!wait && (rc == EWOULDBLOCK)) - { - rc = -1; - } -#endif - - try - { - if (shared) - { - rwcl->sharedAccessCounter--; - rwcl->rwlock.endRead(); - } - else - rwcl->rwlock.endWrite(); - } - catch (const Exception&) - { } - - return rc; - } - - level = newLevel; - return 0; + return rc; } bool FileLock::setlock(CheckStatusWrapper* status, const LockMode mode) @@ -492,25 +533,6 @@ bool FileLock::setlock(CheckStatusWrapper* status, const LockMode mode) return true; } -void FileLock::rwUnlock() -{ - fb_assert(level != LCK_NONE); - - try - { - if (level == LCK_SHARED) - rwcl->rwlock.endRead(); - else - rwcl->rwlock.endWrite(); - } - catch (const Exception& ex) - { - iscLogException("rwlock end-operation error", ex); - } - - level = LCK_NONE; -} - void FileLock::unlock() { if (level == LCK_NONE) @@ -518,97 +540,8 @@ void FileLock::unlock() return; } - // For shared lock we must take into an account reenterability - MutexEnsureUnlock guard(rwcl->sharedAccessMutex, FB_FUNCTION); - if (level == LCK_SHARED) - { - guard.enter(); - - fb_assert(rwcl->sharedAccessCounter > 0); - if (--(rwcl->sharedAccessCounter) > 0) - { - // counter is non-zero - we must keep file lock - rwUnlock(); - return; - } - } - -#ifdef USE_FCNTL - struct FLOCK lock; - lock.l_type = F_UNLCK; - lock.l_whence = SEEK_SET; - lock.l_start = lStart; - lock.l_len = 1; - - if (fcntl(oFile->fd, F_SETLK, &lock) != 0) - { -#else - if (flock(oFile->fd, LOCK_UN) != 0) - { -#endif - LocalStatus ls; - CheckStatusWrapper local(&ls); - error(&local, NAME, errno); - iscLogStatus("Unlock error", &local); - } - - rwUnlock(); -} - -string FileLock::getLockId() -{ - fb_assert(oFile); - - DevNode id(getNode(oFile->fd)); - - const size_t len1 = sizeof(id.f_dev); - const size_t len2 = sizeof(id.f_ino); -#ifdef USE_FCNTL - const size_t len3 = sizeof(int); -#endif - - string rc(len1 + len2 -#ifdef USE_FCNTL - + len3 -#endif - , ' '); - char* p = rc.begin(); - - memcpy(p, &id.f_dev, len1); - p += len1; - memcpy(p, &id.f_ino, len2); -#ifdef USE_FCNTL - p += len2; - memcpy(p, &lStart, len3); -#endif - - return rc; -} - -CountedRWLock* FileLock::getRw() -{ - string id = getLockId(); - CountedRWLock* rc = NULL; - - MutexLockGuard g(rwlocksMutex, FB_FUNCTION); - - CountedRWLock** got = rwlocks->get(id); - if (got) - { - rc = *got; - } - - if (!rc) - { - rc = FB_NEW_POOL(*getDefaultMemoryPool()) CountedRWLock; - CountedRWLock** put = rwlocks->put(id); - fb_assert(put); - *put = rc; - } - - ++(rc->cnt); - - return rc; + file->unlock(); + level = LCK_NONE; } #endif // UNIX