Fixed CORE-5041: Database may get corrupted after ALTER DATABASE ENCRYPT/DECRYPT issued multiple times per database open time

src/jrd/CryptoManager.cpp

@@ -43,6 +43,7 @@
 #include "../jrd/cch_proto.h"
 #include "../jrd/lck_proto.h"
 #include "../jrd/pag_proto.h"
+#include "../jrd/os/pio_proto.h"
 #include "../common/isc_proto.h"
 #include "../common/classes/GetPlugins.h"
 #include "../common/classes/RefMutex.h"
@@ -60,32 +61,17 @@ namespace {
 
 	class Header
 	{
+	protected:
+		Header()
+			: header(NULL)
+		{ }
+
+		void setHeader(void* buf)
+		{
+			header = reinterpret_cast<Ods::header_page*>(buf);
+		}
+
 	public:
-		Header(Jrd::thread_db* p_tdbb, USHORT lockType)
-			: tdbb(p_tdbb),
-			  window(Jrd::HEADER_PAGE_NUMBER),
-			  header((Ods::header_page*) CCH_FETCH(tdbb, &window, lockType, pag_header))
-		{
-			if (!header)
-			{
-				ERR_punt();
-			}
-		}
-
-		Ods::header_page* write()
-		{
-			CCH_MARK_MUST_WRITE(tdbb, &window);
-			return header;
-		}
-
-		void depends(Stack<ULONG>& pages)
-		{
-			while (pages.hasData())
-			{
-				CCH_precedence(tdbb, &window, pages.pop());
-			}
-		}
-
 		const Ods::header_page* operator->() const
 		{
 			return header;
@@ -96,27 +82,105 @@ namespace {
 			return header;
 		}
 
-		~Header()
+		// This routine is looking for a clumplet on header page but is not ready to handle continuation
+		// Fortunately, modern pages of size 4k and bigger can contain everything on one page.
+		bool searchEntry(UCHAR type, UCHAR& out_len, const UCHAR* &entry)
+		{
+			const UCHAR* end = ((const UCHAR*)header) + header->hdr_page_size;
+			for (const UCHAR* p = header->hdr_data; (p < end - 2) && (*p != Ods::HDR_end); p += 2u + p[1])
+			{
+				if (*p == type)
+				{
+					out_len = p[1];
+					entry = p + 2;
+					if (entry + out_len > end)
+						out_len = end - entry;
+					return true;
+				}
+			}
+			return false;
+		}
+
+	private:
+		Ods::header_page* header;
+	};
+
+
+	class CchHdr : public Header
+	{
+	public:
+		CchHdr(Jrd::thread_db* p_tdbb, USHORT lockType)
+			: window(Jrd::HEADER_PAGE_NUMBER),
+			  tdbb(p_tdbb)
+		{
+			void* h = CCH_FETCH(tdbb, &window, lockType, pag_header);
+			if (!h)
+			{
+				ERR_punt();
+			}
+			setHeader(h);
+		}
+
+		Ods::header_page* write()
+		{
+			CCH_MARK_MUST_WRITE(tdbb, &window);
+			return const_cast<Ods::header_page*>(operator->());
+		}
+
+		void depends(Stack<ULONG>& pages)
+		{
+			while (pages.hasData())
+			{
+				CCH_precedence(tdbb, &window, pages.pop());
+			}
+		}
+
+		~CchHdr()
 		{
 			CCH_RELEASE(tdbb, &window);
 		}
 
 	private:
-		Jrd::thread_db* tdbb;
 		Jrd::WIN window;
-		Ods::header_page* header;
+		Jrd::thread_db* tdbb;
 	};
+
+	class PhysHdr : public Header
+	{
+	public:
+		PhysHdr(Jrd::thread_db* tdbb)
+		{
+			Jrd::Database* dbb = tdbb->getDatabase();
+
+			UCHAR* h = FB_NEW_POOL(*Firebird::MemoryPool::getContextPool()) UCHAR[dbb->dbb_page_size + PAGE_ALIGNMENT];
+			buffer.reset(h);
+
+			h = FB_ALIGN(h, PAGE_ALIGNMENT);
+			PIO_header(tdbb, h, dbb->dbb_page_size);
+			setHeader(h);
+		}
+
+	private:
+		AutoPtr<UCHAR, ArrayDelete<UCHAR> > buffer;
+	};
+
+	const UCHAR CRYPT_RELEASE = LCK_SR;
+	const UCHAR CRYPT_NORMAL = LCK_PR;
+	const UCHAR CRYPT_CHANGE = LCK_PW;
+	const UCHAR CRYPT_INIT = LCK_EX;
 }
 
+
 namespace Jrd {
 
 	CryptoManager::CryptoManager(thread_db* tdbb)
 		: PermanentStorage(*tdbb->getDatabase()->dbb_permanent),
+		  sync(this),
 		  keyHolderPlugins(getPool()),
 		  cryptThreadId(0),
 		  cryptPlugin(NULL),
 		  dbb(*tdbb->getDatabase()),
-		  needLock(true),
+		  slowIO(0),
 		  crypt(false),
 		  process(false),
 		  down(false)
@@ -124,8 +188,6 @@ namespace Jrd {
 		stateLock = FB_NEW_RPT(getPool(), 0)
 			Lock(tdbb, 0, LCK_crypt_status, this, blockingAstChangeCryptState);
 		threadLock = FB_NEW_RPT(getPool(), 0) Lock(tdbb, 0, LCK_crypt);
-
-		takeStateLock(tdbb);
 	}
 
 	CryptoManager::~CryptoManager()
@@ -157,28 +219,64 @@ namespace Jrd {
 		LCK_release(tdbb, stateLock);
 	}
 
-	void CryptoManager::takeStateLock(thread_db* tdbb)
+	void CryptoManager::doOnTakenWriteSync(thread_db* tdbb)
 	{
 		fb_assert(stateLock);
-		fb_assert(tdbb->getAttachment());
+		if (stateLock->lck_physical > CRYPT_RELEASE)
+			return;
 
-		if (needLock)
+		fb_assert(tdbb);
+		lockAndReadHeader(tdbb, CRYPT_HDR_NOWAIT);
+	}
+
+	void CryptoManager::lockAndReadHeader(thread_db* tdbb, unsigned flags)
 	{
-			if (!LCK_lock(tdbb, stateLock, LCK_SR, LCK_WAIT))
+		if (flags & CRYPT_HDR_INIT)
+		{
+			if (LCK_lock(tdbb, stateLock, CRYPT_INIT, LCK_NO_WAIT))
+			{
+				LCK_write_data(tdbb, stateLock, 1);
+				if (!LCK_convert(tdbb, stateLock, CRYPT_NORMAL, LCK_NO_WAIT))
 				{
 					fb_assert(tdbb->tdbb_status_vector->getState() & IStatus::STATE_ERRORS);
 					ERR_punt();
 				}
-
+			}
+			else if (!LCK_lock(tdbb, stateLock, CRYPT_NORMAL, LCK_WAIT))
+			{
+				fb_assert(false);
+			}
+		}
+		else
+		{
+			if (!LCK_convert(tdbb, stateLock, CRYPT_NORMAL,
+				(flags & CRYPT_HDR_NOWAIT) ? LCK_NO_WAIT : LCK_WAIT))
+			{
+				slowIO = LCK_read_data(tdbb, stateLock);
+				fb_assert(slowIO);
+			}
+			else
+				slowIO = 0;
+		}
 		tdbb->tdbb_status_vector->init();
-			needLock = false;
+
+		PhysHdr hdr(tdbb);
+		crypt = hdr->hdr_flags & Ods::hdr_encrypted;
+		process = hdr->hdr_flags & Ods::hdr_crypt_process;
+		if (crypt || process)
+		{
+			loadPlugin(hdr->hdr_crypt_plugin);
 		}
 	}
 
 	void CryptoManager::loadPlugin(const char* pluginName)
 	{
-		MutexLockGuard guard(pluginLoadMtx, FB_FUNCTION);
+		if (cryptPlugin)
+		{
+			return;
+		}
 
+		MutexLockGuard guard(pluginLoadMtx, FB_FUNCTION);
 		if (cryptPlugin)
 		{
 			return;
@@ -206,7 +304,7 @@ namespace Jrd {
 
 		const bool newCryptState = plugName.hasData();
 		{	// window scope
-			Header hdr(tdbb, LCK_read);
+			CchHdr hdr(tdbb, LCK_read);
 
 			// Check header page for flags
 			if (hdr->hdr_flags & Ods::hdr_crypt_process)
@@ -236,10 +334,20 @@ namespace Jrd {
 
 	void CryptoManager::changeCryptState(thread_db* tdbb, const Firebird::string& plugName)
 	{
+		if (plugName.length() > 31)
+		{
+			(Arg::Gds(isc_cp_name_too_long) << Arg::Num(31)).raise();
+		}
+
 		const bool newCryptState = plugName.hasData();
 
-		{	// window scope
+		try
-			Header hdr(tdbb, LCK_write);
+		{
+			BarSync::LockGuard writeGuard(tdbb, sync);
+
+			// header scope
+			CchHdr hdr(tdbb, LCK_write);
+			writeGuard.lock();
 
 			// Check header page for flags
 			if (hdr->hdr_flags & Ods::hdr_crypt_process)
@@ -254,17 +362,19 @@ namespace Jrd {
 			}
 
 			fb_assert(stateLock);
-			// Take exclusive stateLock
+			// Trigger lock on ChangeCryptState
-			bool ret = needLock ? LCK_lock(tdbb, stateLock, LCK_PW, LCK_WAIT) :
+			if (!LCK_convert(tdbb, stateLock, CRYPT_CHANGE, LCK_WAIT))
-								  LCK_convert(tdbb, stateLock, LCK_PW, LCK_WAIT);
-			if (!ret)
 			{
 				fb_assert(tdbb->tdbb_status_vector->getState() & IStatus::STATE_ERRORS);
 				ERR_punt();
 			}
 			fb_utils::init_status(tdbb->tdbb_status_vector);
-			needLock = false;
 
+			// Load plugin
+			if (newCryptState)
+			{
+				loadPlugin(plugName.c_str());
+			}
 			crypt = newCryptState;
 
 			// Write modified header page
@@ -272,27 +382,40 @@ namespace Jrd {
 			if (crypt)
 			{
 				header->hdr_flags |= Ods::hdr_encrypted;
-				plugName.copyTo(header->hdr_crypt_plugin, sizeof header->hdr_crypt_plugin);
+				plugName.copyTo(header->hdr_crypt_plugin, sizeof(header->hdr_crypt_plugin));
 			}
 			else
 			{
 				header->hdr_flags &= ~Ods::hdr_encrypted;
 			}
-			header->hdr_flags |= Ods::hdr_crypt_process;
+
+			// Set hdr_crypt_page for crypt thread
 			header->hdr_crypt_page = 1;
+			header->hdr_flags |= Ods::hdr_crypt_process;
 			process = true;
 		}
-
+		catch (const Exception&)
-		// Trigger lock on ChangeCryptState
-		if (!LCK_convert(tdbb, stateLock, LCK_EX, LCK_WAIT))
 		{
-			ERR_punt();
+			if (stateLock->lck_physical != CRYPT_NORMAL)
+			{
+				try
+				{
+					if (!LCK_convert(tdbb, stateLock, CRYPT_RELEASE, LCK_NO_WAIT))
+						fb_assert(false);
+					lockAndReadHeader(tdbb);
+				}
+				catch(const Exception&)
+				{ }
+			}
+			throw;
 		}
 
-		if (!LCK_convert(tdbb, stateLock, LCK_SR, LCK_WAIT))
+		SINT64 next = LCK_read_data(tdbb, stateLock) + 1;
-		{
+		LCK_write_data(tdbb, stateLock, next);
-			ERR_punt();
+
-		}
+		if (!LCK_convert(tdbb, stateLock, CRYPT_RELEASE, LCK_NO_WAIT))
+			fb_assert(false);
+		lockAndReadHeader(tdbb);
 		fb_utils::init_status(tdbb->tdbb_status_vector);
 
 		startCryptThread(tdbb);
@@ -302,9 +425,16 @@ namespace Jrd {
 	{
 		AsyncContextHolder tdbb(&dbb, FB_FUNCTION);
 
+		if (stateLock->lck_physical != CRYPT_CHANGE && stateLock->lck_physical != CRYPT_INIT)
+		{
+			sync.ast(tdbb);
+		}
+	}
+
+	void CryptoManager::doOnAst(thread_db* tdbb)
+	{
 		fb_assert(stateLock);
-		LCK_release(tdbb, stateLock);
+		LCK_convert(tdbb, stateLock, CRYPT_RELEASE, LCK_NO_WAIT);
-		needLock = true;
 	}
 
 	void CryptoManager::startCryptThread(thread_db* tdbb)
@@ -335,7 +465,7 @@ namespace Jrd {
 			down = false;
 
 			// Determine current page from the header
-			Header hdr(tdbb, LCK_read);
+			CchHdr hdr(tdbb, LCK_read);
 			process = hdr->hdr_flags & Ods::hdr_crypt_process ? true : false;
 			if (!process)
 			{
@@ -375,23 +505,7 @@ namespace Jrd {
 	{
 		keyHolderPlugins.attach(att, dbb.dbb_config);
 
-		if (!cryptPlugin)
+		lockAndReadHeader(tdbb, CRYPT_HDR_INIT);
-		{
-			// Lock crypt state
-			takeStateLock(tdbb);
-
-			Header hdr(tdbb, LCK_read);
-			crypt = hdr->hdr_flags & Ods::hdr_encrypted;
-			process = hdr->hdr_flags & Ods::hdr_crypt_process;
-
-			if (crypt || process)
-			{
-				loadPlugin(hdr->hdr_crypt_plugin);
-
-				if (!cryptPlugin)
-					(Arg::Gds(isc_decrypt_error)).raise();
-			}
-		}
 	}
 
 	void CryptoManager::cryptThread()
@@ -543,7 +657,7 @@ namespace Jrd {
 
 	void CryptoManager::writeDbHeader(thread_db* tdbb, ULONG runpage, Stack<ULONG>& pages)
 	{
-		Header hdr(tdbb, LCK_write);
+		CchHdr hdr(tdbb, LCK_write);
 		hdr.depends(pages);
 
 		Ods::header_page* header = hdr.write();
@@ -555,51 +669,43 @@ namespace Jrd {
 		}
 	}
 
-	bool CryptoManager::decrypt(FbStatusVector* sv, Ods::pag* page)
+	bool CryptoManager::read(thread_db* tdbb, FbStatusVector* sv, jrd_file* file,
+		BufferDesc* bdb, Ods::pag* page, bool noShadows, PageSpace* pageSpace)
 	{
 		// Code calling us is not ready to process exceptions correctly
 		// Therefore use old (status vector based) method
 		try
 		{
-			if (page->pag_flags & Ods::crypted_page)
+			if (slowIO)
 			{
-				if (!cryptPlugin)
+				BarSync::LockGuard lockGuard(tdbb, sync);
+				for (SINT64 previous = slowIO; ; previous = slowIO)
 				{
-					// This may happen only in case of classic server when other connection encrypted alive database
+					switch (internalRead(tdbb, sv, file, bdb, page, noShadows, pageSpace))
-					// We are invoked from shared cache manager, i.e. no valid attachment in tdbb
-					// Therefore create system temporary attachment like in crypt thread to be able to work with locks
-					UserId user;
-					user.usr_user_name = "(Crypt plugin loader)";
-
-					Jrd::Attachment* const attachment = Jrd::Attachment::create(&dbb);
-					RefPtr<SysStableAttachment> sAtt(FB_NEW SysStableAttachment(attachment));
-					attachment->setStable(sAtt);
-					attachment->att_filename = dbb.dbb_filename;
-					attachment->att_user = &user;
-
-					BackgroundContextHolder tdbb(&dbb, attachment, sv, FB_FUNCTION);
-
-					// Lock crypt state
-					takeStateLock(tdbb);
-
-					Header hdr(tdbb, LCK_read);
-					crypt = hdr->hdr_flags & Ods::hdr_encrypted;
-					process = hdr->hdr_flags & Ods::hdr_crypt_process;
-
-					if (crypt || process)
 					{
-						loadPlugin(hdr->hdr_crypt_plugin);
+					case SUCCESS_ALL:
+						return true;
+
+					case FAILED_IO:
+						return false;
+
+					case FAILED_CRYPT:
+						if (slowIO)
+						{
+							lockAndReadHeader(tdbb, CRYPT_HDR_NOWAIT);
+							if (slowIO == previous)
+								return false;
+							break;
 						}
-
+						else
-					if (!cryptPlugin)
-					{
-						(Arg::Gds(isc_decrypt_error)).raise();
 							return false;
 					}
 				}
-
+			}
-				cryptPlugin->decrypt(sv, dbb.dbb_page_size - sizeof(Ods::pag), &page[1], &page[1]);
+			else
-				if (sv->getState() & IStatus::STATE_ERRORS)
+			{
+				BarSync::IoGuard ioGuard(tdbb, sync);
+				if (internalRead(tdbb, sv, file, bdb, page, noShadows, pageSpace) != SUCCESS_ALL)
 					return false;
 			}
 
@@ -612,37 +718,145 @@ namespace Jrd {
 		return false;
 	}
 
-	Ods::pag* CryptoManager::encrypt(FbStatusVector* sv, Ods::pag* from, Ods::pag* to)
+	CryptoManager::IoResult CryptoManager::internalRead(thread_db* tdbb, FbStatusVector* sv,
+		jrd_file* file, BufferDesc* bdb, Ods::pag* page, bool noShadows, PageSpace* pageSpace)
+	{
+		fb_assert(pageSpace || noShadows);
+		int retryCount = 0;
+
+		while (!PIO_read(tdbb, file, bdb, page, sv))
+		{
+			if (noShadows)
+				return FAILED_IO;
+
+			if (!CCH_rollover_to_shadow(tdbb, &dbb, file, false))
+				return FAILED_IO;
+
+			if (file != pageSpace->file)
+				file = pageSpace->file;
+			else
+			{
+				if (retryCount++ == 3)
+				{
+					gds__log("IO error loop Unwind to avoid a hang");
+					return FAILED_IO;
+				}
+			}
+		}
+
+		if (page->pag_flags & Ods::crypted_page)
+		{
+			fb_assert(cryptPlugin);
+			if (!cryptPlugin)
+			{
+				Arg::Gds(isc_decrypt_error).copyTo(sv);
+				return FAILED_CRYPT;
+			}
+
+			cryptPlugin->decrypt(sv, dbb.dbb_page_size - sizeof(Ods::pag),
+				&page[1], &page[1]);
+			if (sv->getState() & IStatus::STATE_ERRORS)
+				return FAILED_CRYPT;
+		}
+
+		return SUCCESS_ALL;
+	}
+
+	bool CryptoManager::write(thread_db* tdbb, FbStatusVector* sv, jrd_file* file,
+		BufferDesc* bdb, Ods::pag* page)
 	{
 		// Code calling us is not ready to process exceptions correctly
 		// Therefore use old (status vector based) method
 		try
 		{
-			if (crypt && cryptPlugin && Ods::pag_crypt_page[from->pag_type % (pag_max + 1)])
+			if (slowIO)
 			{
-				to[0] = from[0];
+				BarSync::LockGuard lockGuard(tdbb, sync);
+				for (SINT64 previous = slowIO; ; previous = slowIO)
+				{
+					switch (internalWrite(tdbb, sv, file, bdb, page))
+					{
+					case SUCCESS_ALL:
+						if (slowIO)
+						{
+							lockAndReadHeader(tdbb, CRYPT_HDR_NOWAIT);
+							if (slowIO == previous)
+								return true;
+							break;
+						}
+						else
+							return true;
 
-				cryptPlugin->encrypt(sv, dbb.dbb_page_size - sizeof(Ods::pag), &from[1], &to[1]);
+					case FAILED_IO:
-				if (sv->getState() & IStatus::STATE_ERRORS)
+						return false;
-					return NULL;
 
-				to->pag_flags |= Ods::crypted_page; // Mark page that is going to be written as encrypted
+					case FAILED_CRYPT:
-				from->pag_flags |= Ods::crypted_page; // Set the mark for page in cache as well
+						if (slowIO)
-													// If page write fail, nothing bad can happen
+						{
-													// it will be encrypted next time is modified
+							lockAndReadHeader(tdbb, CRYPT_HDR_NOWAIT);
-				return to;
+							if (slowIO == previous)
+								return false;
+							break;
+						}
+						else
+							return false;
+					}
+				}
 			}
 			else
 			{
-				from->pag_flags &= ~Ods::crypted_page;
+				BarSync::IoGuard ioGuard(tdbb, sync);
-				return from;
+				if (internalWrite(tdbb, sv, file, bdb, page) != SUCCESS_ALL)
+                    return false;
 			}
+
+			return true;
 		}
 		catch (const Exception& ex)
 		{
 			ex.stuffException(sv);
 		}
-		return NULL;
+		return false;
+	}
+
+	CryptoManager::IoResult CryptoManager::internalWrite(thread_db* tdbb, FbStatusVector* sv,
+		jrd_file* file, BufferDesc* bdb, Ods::pag* page)
+	{
+		Buffer to;
+		Ods::pag* dest = page;
+		UCHAR savedFlags = page->pag_flags;
+
+		fb_assert ((!crypt) || cryptPlugin);
+		if (crypt && Ods::pag_crypt_page[page->pag_type % (pag_max + 1)])
+		{
+			if (!cryptPlugin)
+			{
+				Arg::Gds(isc_decrypt_error).copyTo(sv);	//!!!!
+				return FAILED_CRYPT;
+			}
+
+			to[0] = page[0];
+			cryptPlugin->encrypt(sv, dbb.dbb_page_size - sizeof(Ods::pag),
+				&page[1], &to[1]);
+			if (sv->getState() & IStatus::STATE_ERRORS)
+				return FAILED_CRYPT;
+
+			to->pag_flags |= Ods::crypted_page;		// Mark page that is going to be written as encrypted
+			page->pag_flags |= Ods::crypted_page;	// Set the mark for page in cache as well
+			dest = to;								// Choose correct destination
+		}
+		else
+		{
+			page->pag_flags &= ~Ods::crypted_page;
+		}
+
+		if (!PIO_write(tdbb, file, bdb, dest, sv))
+		{
+			page->pag_flags = savedFlags;
+			return FAILED_IO;
+		}
+
+		return SUCCESS_ALL;
 	}
 
 	int CryptoManager::blockingAstChangeCryptState(void* object)

src/jrd/CryptoManager.h

@@ -35,6 +35,7 @@
 #include "../common/classes/fb_string.h"
 #include "../common/classes/objects_array.h"
 #include "../common/classes/stack.h"
+#include "../common/classes/condition.h"
 #include "../common/ThreadStart.h"
 #include "../jrd/ods.h"
 #include "../jrd/status.h"
@@ -56,8 +57,188 @@ class jrd_file;
 class BufferDesc;
 class thread_db;
 class Lock;
+class PageSpace;
 
-class CryptoManager : public Firebird::PermanentStorage
+class BarSync
+{
+public:
+	class IBar
+	{
+	public:
+		virtual void doOnTakenWriteSync(Jrd::thread_db* tdbb) = 0;
+		virtual void doOnAst(Jrd::thread_db* tdbb) = 0;
+	};
+
+	BarSync(IBar* i)
+		: callback(i), counter(0), lockMode(0), flagWriteLock(false)
+	{ }
+
+	class IoGuard
+	{
+	public:
+		IoGuard(Jrd::thread_db* p_tdbb, BarSync& p_bs)
+			: tdbb(p_tdbb), bs(p_bs)
+		{
+			bs.ioBegin(tdbb);
+		}
+
+		~IoGuard()
+		{
+			bs.ioEnd(tdbb);
+		}
+
+	private:
+		Jrd::thread_db* tdbb;
+		BarSync& bs;
+	};
+
+	class LockGuard
+	{
+	public:
+		LockGuard(Jrd::thread_db* p_tdbb, BarSync& p_bs)
+			: tdbb(p_tdbb), bs(p_bs), flagLocked(false)
+		{ }
+
+		void lock()
+		{
+			fb_assert(!flagLocked);
+			if (!flagLocked)
+			{
+				bs.lockBegin(tdbb);
+				flagLocked = true;
+			}
+		}
+
+		~LockGuard()
+		{
+			if (flagLocked)
+			{
+				bs.lockEnd(tdbb);
+			}
+		}
+
+	private:
+		Jrd::thread_db* tdbb;
+		BarSync& bs;
+		bool flagLocked;
+	};
+
+	void ioBegin(Jrd::thread_db* tdbb)
+	{
+		Firebird::MutexLockGuard g(mutex, FB_FUNCTION);
+
+		if (counter < 0)
+		{
+			if ((counter % BIG_VALUE == 0) && (!flagWriteLock))
+			{
+				if (lockMode)
+				{
+					// Someone is waiting for write lock
+					lockCond.notifyOne();
+					barCond.wait(mutex);
+				}
+				else
+				{
+					// Ast done
+					callWriteLockHandler(tdbb);
+					counter = 0;
+				}
+			}
+			else if (!(flagWriteLock && (thread == getThreadId())))
+				barCond.wait(mutex);
+		}
+		++counter;
+	}
+
+	void ioEnd(Jrd::thread_db* tdbb)
+	{
+		Firebird::MutexLockGuard g(mutex, FB_FUNCTION);
+
+		if (--counter < 0 && counter % BIG_VALUE == 0)
+		{
+			if (!(flagWriteLock && (thread == getThreadId())))
+			{
+				if (lockMode)
+					lockCond.notifyOne();
+				else
+				{
+					callWriteLockHandler(tdbb);
+					finishWriteLock();
+				}
+			}
+		}
+	}
+
+	void ast(Jrd::thread_db* tdbb)
+	{
+		Firebird::MutexLockGuard g(mutex, FB_FUNCTION);
+		if (counter >= 0)
+		{
+			counter -= BIG_VALUE;
+		}
+		callback->doOnAst(tdbb);
+	}
+
+	void lockBegin(Jrd::thread_db* tdbb)
+	{
+		Firebird::MutexLockGuard g(mutex, FB_FUNCTION);
+
+		if ((counter -= BIG_VALUE) != -BIG_VALUE)
+		{
+			++lockMode;
+			try
+			{
+				lockCond.wait(mutex);
+			}
+			catch(const Firebird::Exception&)
+			{
+				--lockMode;
+				throw;
+			}
+			--lockMode;
+		}
+
+		thread = getThreadId();
+		flagWriteLock = true;
+	}
+
+	void lockEnd(Jrd::thread_db* tdbb)
+	{
+		Firebird::MutexLockGuard g(mutex, FB_FUNCTION);
+
+		flagWriteLock = false;
+		finishWriteLock();
+	}
+
+private:
+	void callWriteLockHandler(Jrd::thread_db* tdbb)
+	{
+		thread = getThreadId();
+		flagWriteLock = true;
+		callback->doOnTakenWriteSync(tdbb);
+		flagWriteLock = false;
+	}
+
+	void finishWriteLock()
+	{
+		if ((counter += BIG_VALUE) == 0)
+			barCond.notifyAll();
+		else
+			lockCond.notifyOne();
+	}
+
+	Firebird::Condition barCond, lockCond;
+	Firebird::Mutex mutex;
+	IBar* callback;
+	ThreadId thread;
+	int counter;
+	int lockMode;
+	bool flagWriteLock;
+
+	static const int BIG_VALUE = 1000000;
+};
+
+class CryptoManager FB_FINAL : public Firebird::PermanentStorage, public BarSync::IBar
 {
 public:
 	explicit CryptoManager(thread_db* tdbb);
@@ -73,13 +254,16 @@ public:
 	void startCryptThread(thread_db* tdbb);
 	void terminateCryptThread(thread_db* tdbb);
 
-	bool decrypt(FbStatusVector* sv, Ods::pag* page);
+	bool read(thread_db* tdbb, FbStatusVector* sv, jrd_file* file, BufferDesc* bdb,
-	Ods::pag* encrypt(FbStatusVector* sv, Ods::pag* from, Ods::pag* to);
+		Ods::pag* page, bool noShadows = true, PageSpace* pageSpace = NULL);
+	bool write(thread_db* tdbb, FbStatusVector* sv, jrd_file* file, BufferDesc* bdb,
+		Ods::pag* page);
 
 	void cryptThread();
 
 	ULONG getCurrentPage();
 
+private:
 	class Buffer
 	{
 	public:
@@ -88,11 +272,15 @@ public:
 			return reinterpret_cast<Ods::pag*>(FB_ALIGN(buf, PAGE_ALIGNMENT));
 		}
 
+		Ods::pag* operator->()
+		{
+			return reinterpret_cast<Ods::pag*>(FB_ALIGN(buf, PAGE_ALIGNMENT));
+		}
+
 	private:
 		char buf[MAX_PAGE_SIZE + PAGE_ALIGNMENT - 1];
 	};
 
-private:
 	class HolderAttachments
 	{
 	public:
@@ -134,11 +322,25 @@ private:
 	static int blockingAstChangeCryptState(void*);
 	void blockingAstChangeCryptState();
 
-	void takeStateLock(thread_db* tdbb);
+	// IBar's pure virtual functions are implemented here
+	void doOnTakenWriteSync(thread_db* tdbb);
+	void doOnAst(thread_db* tdbb);
+
 	void loadPlugin(const char* pluginName);
 	ULONG getLastPage(thread_db* tdbb);
 	void writeDbHeader(thread_db* tdbb, ULONG runpage, Firebird::Stack<ULONG>& pages);
 
+	void lockAndReadHeader(thread_db* tdbb, unsigned flags = 0);
+	static const unsigned CRYPT_HDR_INIT =		0x01;
+	static const unsigned CRYPT_HDR_NOWAIT =	0x02;
+
+	enum IoResult {SUCCESS_ALL, FAILED_CRYPT, FAILED_IO};
+	IoResult internalRead(thread_db* tdbb, FbStatusVector* sv, jrd_file* file,
+		BufferDesc* bdb, Ods::pag* page, bool noShadows, PageSpace* pageSpace);
+	IoResult internalWrite(thread_db* tdbb, FbStatusVector* sv, jrd_file* file,
+		BufferDesc* bdb, Ods::pag* page);
+
+	BarSync sync;
 	Firebird::AtomicCounter currentPage;
 	Firebird::Mutex pluginLoadMtx, cryptThreadMtx;
 	KeyHolderPlugins keyHolderPlugins;
@@ -147,7 +349,8 @@ private:
 	Database& dbb;
 	Lock* stateLock;
 	Lock* threadLock;
-	bool needLock, crypt, process, down;
+	SINT64 slowIO;
+	bool crypt, process, down;
 };
 
 } // namespace Jrd

src/jrd/cch.cpp

@@ -765,7 +765,6 @@ void CCH_fetch_page(thread_db* tdbb, WIN* window, const bool read_shadow)
 	fb_assert(pageSpace);
 
 	jrd_file* file = pageSpace->file;
-	SSHORT retryCount = 0;
 	const bool isTempPage = pageSpace->isTemporary();
 
 	/*
@@ -808,36 +807,9 @@ void CCH_fetch_page(thread_db* tdbb, WIN* window, const bool read_shadow)
 			bdb->bdb_page.getPageSpaceID(), bdb->bdb_page.getPageNum(), bak_state, diff_page));
 
 		// Read page from disk as normal
-		bool error = false;
+		if (!dbb->dbb_crypto_manager->read(tdbb, status, file, bdb, page, isTempPage || !read_shadow, pageSpace))
-		while (!PIO_read(tdbb, file, bdb, page, status))
 		{
-			if (isTempPage || !read_shadow) {
+			if (read_shadow && !isTempPage)
-				error = true;
-				break;
-			}
-
-			if (!CCH_rollover_to_shadow(tdbb, dbb, file, false))
-			{
-				PAGE_LOCK_RELEASE(tdbb, bcb, bdb->bdb_lock);
-				CCH_unwind(tdbb, true);
-			}
-
-			if (file != pageSpace->file)
-				file = pageSpace->file;
-			else
-			{
-				if (retryCount++ == 3)
-				{
-					fprintf(stderr, "IO error loop Unwind to avoid a hang\n");
-					PAGE_LOCK_RELEASE(tdbb, bcb, bdb->bdb_lock);
-					CCH_unwind(tdbb, true);
-				}
-			}
-		}
-
-		if (!error)
-		{
-			if (!dbb->dbb_crypto_manager->decrypt(status, page))
 			{
 				PAGE_LOCK_RELEASE(tdbb, bcb, bdb->bdb_lock);
 				CCH_unwind(tdbb, true);
@@ -865,36 +837,9 @@ void CCH_fetch_page(thread_db* tdbb, WIN* window, const bool read_shadow)
 			NBAK_TRACE(("Re-reading page %d, state=%d, diff page=%d from DISK",
 				bdb->bdb_page, bak_state, diff_page));
 
-			bool error = false;
+			if (!dbb->dbb_crypto_manager->read(tdbb, status, file, bdb, page, !read_shadow, pageSpace))
-			while (!PIO_read(tdbb, file, bdb, page, status))
 			{
-				if (!read_shadow) {
+				if (read_shadow)
-					error = true;
-					break;
-				}
-
-				if (!CCH_rollover_to_shadow(tdbb, dbb, file, false))
-				{
-					PAGE_LOCK_RELEASE(tdbb, bcb, bdb->bdb_lock);
-					CCH_unwind(tdbb, true);
-				}
-
-				if (file != pageSpace->file)
-					file = pageSpace->file;
-				else
-				{
-					if (retryCount++ == 3)
-					{
-						fprintf(stderr, "IO error loop Unwind to avoid a hang\n");
-						PAGE_LOCK_RELEASE(tdbb, bcb, bdb->bdb_lock);
-						CCH_unwind(tdbb, true);
-					}
-				}
-			}
-
-			if (!error)
-			{
-				if (!dbb->dbb_crypto_manager->decrypt(status, page))
 				{
 					PAGE_LOCK_RELEASE(tdbb, bcb, bdb->bdb_lock);
 					CCH_unwind(tdbb, true);
@@ -2289,10 +2234,7 @@ bool CCH_write_all_shadows(thread_db* tdbb, Shadow* shadow, BufferDesc* bdb,
 		// shadow to be deleted at the next available opportunity when we
 		// know we don't have a page fetched
 
-		CryptoManager::Buffer buffer;
+		if (!dbb->dbb_crypto_manager->write(tdbb, status, sdw->sdw_file, bdb, page))
-		pag* writePage = dbb->dbb_crypto_manager->encrypt(status, page, buffer);
-
-		if (!(writePage && PIO_write(tdbb, sdw->sdw_file, bdb, writePage, status)))
 		{
 			if (sdw->sdw_flags & SDW_manual)
 				result = false;
@@ -4902,18 +4844,8 @@ static bool write_page(thread_db* tdbb, BufferDesc* bdb, FbStatusVector* const s
 			else
 			{
 				// We need to write our pages to main database files
-				CryptoManager::Buffer buffer;
-				pag* writePage = dbb->dbb_crypto_manager->encrypt(status, page, buffer);
-
-				if (!writePage)
-				{
-					bdb->bdb_flags |= BDB_io_error;
-					dbb->dbb_flags |= DBB_suspend_bgio;
-					return false;
-				}
-
 				jrd_file* file = pageSpace->file;
-				while (!PIO_write(tdbb, file, bdb, writePage, status))
+				while (!dbb->dbb_crypto_manager->write(tdbb, status, file, bdb, page))
 				{
 					if (isTempPage || !CCH_rollover_to_shadow(tdbb, dbb, file, inAst))
 					{

src/jrd/jrd.cpp

@@ -1563,13 +1563,11 @@ JAttachment* JProvider::internalAttach(CheckStatusWrapper* user_status, const ch
 				dbb->dbb_backup_manager = FB_NEW_POOL(*dbb->dbb_permanent) BackupManager(tdbb,
 					dbb, Ods::hdr_nbak_unknown);
 				dbb->dbb_backup_manager->initializeAlloc(tdbb);
+				dbb->dbb_crypto_manager = FB_NEW_POOL(*dbb->dbb_permanent) CryptoManager(tdbb);
 
 				PAG_init2(tdbb, 0);
 				PAG_header(tdbb, false);
-
 				dbb->dbb_page_manager.initTempPageSpace(tdbb);
-
-				dbb->dbb_crypto_manager = FB_NEW_POOL(*dbb->dbb_permanent) CryptoManager(tdbb);
 				dbb->dbb_crypto_manager->attach(tdbb, attachment);
 
 				// initialize shadowing as soon as the database is ready for it
@@ -2669,13 +2667,13 @@ JAttachment* JProvider::createDatabase(CheckStatusWrapper* user_status, const ch
 			dbb->dbb_backup_manager = FB_NEW_POOL(*dbb->dbb_permanent) BackupManager(tdbb,
 				dbb, Ods::hdr_nbak_normal);
 			dbb->dbb_backup_manager->dbCreating = true;
+			dbb->dbb_crypto_manager = FB_NEW_POOL(*dbb->dbb_permanent) CryptoManager(tdbb);
 
 			PAG_format_header(tdbb);
 			INI_init2(tdbb);
 			PAG_format_pip(tdbb, *pageSpace);
 
 			dbb->dbb_page_manager.initTempPageSpace(tdbb);
-			dbb->dbb_crypto_manager = FB_NEW_POOL(*dbb->dbb_permanent) CryptoManager(tdbb);
 
 			if (options.dpb_set_page_buffers)
 				PAG_set_page_buffers(tdbb, options.dpb_page_buffers);

src/jrd/nbak.cpp

@@ -811,11 +811,7 @@ bool BackupManager::writeDifference(thread_db* tdbb, FbStatusVector* status, ULO
 	// Check that diff page is not allocation page
 	fb_assert(diff_page % (database->dbb_page_size / sizeof(ULONG)));
 
-	CryptoManager::Buffer buffer;
+	if (!database->dbb_crypto_manager->write(tdbb, status, diff_file, &temp_bdb, page))
-	Ods::pag* writePage = database->dbb_crypto_manager->encrypt(status, page, buffer);
-	if (!writePage)
-		return false;
-	if (!PIO_write(tdbb, diff_file, &temp_bdb, writePage, status))
 		return false;
 	return true;
 }
@@ -825,9 +821,7 @@ bool BackupManager::readDifference(thread_db* tdbb, ULONG diff_page, Ods::pag* p
 	BufferDesc temp_bdb(database->dbb_bcb);
 	temp_bdb.bdb_page = diff_page;
 	temp_bdb.bdb_buffer = page;
-	if (!PIO_read(tdbb, diff_file, &temp_bdb, page, tdbb->tdbb_status_vector))
+	if (!database->dbb_crypto_manager->read(tdbb, tdbb->tdbb_status_vector, diff_file, &temp_bdb, page))
-		return false;
-	if (!database->dbb_crypto_manager->decrypt(tdbb->tdbb_status_vector, page))
 		return false;
 	NBAK_TRACE(("read_diff page=%d, diff=%d", page->pag_pageno, diff_page));
 	return true;

src/jrd/sdw.cpp

@@ -1013,11 +1013,6 @@ void SDW_start(thread_db* tdbb, const TEXT* file_name,
 			ERR_punt();
 		}
 
-		if (!dbb->dbb_crypto_manager->decrypt(tdbb->tdbb_status_vector, (PAG) spare_page))
-		{
-			ERR_punt();
-		}
-
 		const header_page* shadow_header = (header_page*) spare_page;
 
 		//          NOTE ! NOTE! NOTE!