firebird-qa/tests/bugs/core_6048_test.py

#coding:utf-8

"""
ID:          issue-6298
ISSUE:       6298
TITLE:       Provide ability to see current state of DB encryption
DESCRIPTION:
    Test adds lot of data to database, changes FW to ON and runs 'alter database encrypt ...'.
    Then it starts loop with query:
        'select mon$crypt_page, mon$crypt_state from mon$database', with repeating it every 0.5 second.

    Loop continues until we find <ENCRYPTING_PAGES_MIN_CNT> different number of encrypted pages,
    or if timeout <MAX_WAITING_ENCR_FINISH> ms expired.

    If (after loop) number of pages detected in encrypted state less then <ENCRYPTING_PAGES_MIN_CNT>
    then test is considered as failed.
    NB: we do NOT wait for the encryption process to complete for the whole database because this
    time strongly depends on hardware of testing host and concurrent workload. We just want to see
    *several* different values of mon$crypt_page where mon$crypt_state = <RUNNING_ENCRYPTING_STATE>.

JIRA:        CORE-6048
FBTEST:      bugs.core_6048
NOTES:
    [13.06.2022] pzotov
    Checked on 5.0.0.509 - both on Linux and Windows.

    [11.03.2023] pzotov
    Checked on 5.0.0.972, 4.0.3.2907 (Windows).
    
    ::: NB-1 :::
    Before ~06-sep-2021 encryption *blocked* obtaining data from monitoring tables.
    See also: https://github.com/FirebirdSQL/firebird/issues/6947

    ::: NB-2 :::
    Careful tuning required on each tesing box for this test.

    [18.01.2025] pzotov
    Resultset of cursor that executes using instance of selectable PreparedStatement must be stored
    in some variable in order to have ability close it EXPLICITLY (before PS will be freed).
    Otherwise access violation raises during Python GC and pytest hangs at final point (does not return control to OS).
    This occurs at least for: Python 3.11.2 / pytest: 7.4.4 / firebird.driver: 1.10.6 / Firebird.Qa: 0.19.3
    The reason of that was explained by Vlad, 26.10.24 17:42 ("oddities when use instances of selective statements").
"""

import os
import time
import datetime as py_dt
from datetime import timedelta

import pytest
from firebird.qa import *
from firebird.driver import DatabaseError, DbWriteMode

###########################
###   S E T T I N G S   ###
###########################

# QA_GLOBALS -- dict, is defined in qa/plugin.py, obtain settings
# from act.files_dir/'test_config.ini':
enc_settings = QA_GLOBALS['encryption']


# ACHTUNG: this must be carefully tuned on every new host:
#
MAX_WAITING_ENCR_FINISH = int(enc_settings['MAX_WAIT_FOR_ENCR_FINISH_WIN' if os.name == 'nt' else 'MAX_WAIT_FOR_ENCR_FINISH_NIX'])
assert MAX_WAITING_ENCR_FINISH > 0

ENCRYPTION_PLUGIN = enc_settings['encryption_plugin'] # fbSampleDbCrypt
ENCRYPTION_KEY = enc_settings['encryption_key'] # Red

# How many *different* page numbers we want to see as being encrypted before break and finish test:
#
ENCRYPTING_PAGES_MIN_CNT = 3

# Value in mon$database.mon$crypt_state for completed encryption:
#
COMPLETED_ENCRYPTION_STATE = 1

# Value in mon$database.mon$crypt_state for CURRENTLY running encryption:
#
RUNNING_ENCRYPTING_STATE = 3

# How many rows will be inserted in order to make encryption thread do its work for some valuable time:
N_ROWS = 15000 if os.name == 'nt' else 10000

F_LEN = 16383
init_script = f"""
    set bail on;
    create table tlog(id bigint generated by default as identity constraint pk_tlog primary key, crypt_page int, crypt_state smallint);
    create table test(s varchar({F_LEN}));
    commit;
    set term ^;
    execute block as
        declare n bigint = {N_ROWS};
    begin
        while (n>0) do
        begin
            insert into test(s) values( lpad('',{F_LEN}, uuid_to_char(gen_uuid())) );
            n = n - 1;
        end
    end
    ^
    commit
    ^
"""

db = db_factory(init = init_script, charset='none', page_size = 4096)
act = python_act('db', substitutions=[('[ \t]+', ' ')])

@pytest.mark.encryption
@pytest.mark.version('>=4.0.2')
def test_1(act: Action, capsys):
   
    with act.connect_server() as srv:
        srv.database.set_write_mode(database=act.db.db_path, mode=DbWriteMode.SYNC)

    encryption_finished = False
    encryption_started = False
    with act.db.connect() as con, act.db.connect() as con2:

        t1=py_dt.datetime.now()
        d1 = t1-t1
        sttm = f'alter database encrypt with "{ENCRYPTION_PLUGIN}" key "{ENCRYPTION_KEY}"'
        try:
            con.execute_immediate(sttm)
            con.commit()
            encryption_started = True
        except DatabaseError as e:
            # -ALTER DATABASE failed
            # -Crypt plugin fbSampleDbCrypt failed to load
            #  ==> no sense to do anything else, encryption_started remains False.
            print( e.__str__() )

        cur2 = con2.cursor()
        ps, rs = None, None

        try:
            ps = cur2.prepare('select mon$crypt_page, mon$crypt_state from mon$database')

            # This will store different number of pages which are currently encrypted.
            # When length of this set will exceed ENCRYPTING_PAGES_MIN_CNT then we break from loop:
            #
            encrypting_pages_set = set()
            waiting_in_loop = -1
            while encryption_started:
                t2=py_dt.datetime.now()
                d1=t2-t1
                waiting_in_loop = d1.seconds*1000 + d1.microseconds//1000
                if waiting_in_loop > MAX_WAITING_ENCR_FINISH:
                    print(f'TIMEOUT EXPIRATION: encryption took {d1.seconds*1000 + d1.microseconds//1000} ms which exceeds limit = {MAX_WAITING_ENCR_FINISH} ms.')
                    break

                # ::: NB ::: 'ps' returns data, i.e. this is SELECTABLE expression.
                # We have to store result of cur.execute(<psInstance>) in order to
                # close it explicitly.
                # Otherwise AV can occur during Python garbage collection and this
                # causes pytest to hang on its final point.
                # Explained by hvlad, email 26.10.24 17:42
                rs = cur2.execute(ps)
                for r in rs:
                    crypt_page, crypt_state = r[:2]
    
                con2.commit()

                # 0 = non crypted;
                # 1 = has been encrypted;
                # 2 = is DEcrypting;
                # 3 = is Encrypting;
                if crypt_state == RUNNING_ENCRYPTING_STATE:
                    encrypting_pages_set.add(crypt_page,)

                if crypt_state == COMPLETED_ENCRYPTION_STATE:
                    encryption_finished = True
                    break
                elif len(encrypting_pages_set) > ENCRYPTING_PAGES_MIN_CNT:
                    break
                else:
                    time.sleep(0.1)

        except DatabaseError as e:
            print( e.__str__() )
            print(e.gds_codes)
        finally:
            if rs:
                rs.close() # <<< EXPLICITLY CLOSING CURSOR RESULTS
            if ps:
                ps.free()

        # ---------------------------------------------------------
        
        expected_msg = f'EXPECTED: at least {ENCRYPTING_PAGES_MIN_CNT} different mon$crypt_page values found during encryption process.'
        if encryption_started:
            if len(encrypting_pages_set) > ENCRYPTING_PAGES_MIN_CNT:
                print(expected_msg)
            else:
                print(f'UNEXPECTED: only {len(encrypting_pages_set)} different mon$crypt_page values found for {waiting_in_loop} ms: %s'  % ','.join( (str(x) for x in sorted(encrypting_pages_set)) ) )
                print(f'At least {ENCRYPTING_PAGES_MIN_CNT} different values expected to be found; encryption_finished = {encryption_finished}')
        else:
            print('UNEXPECTED: encryption did not start.')
        # ---------------------------------------------------------

        expected_stdout = f"""
            {expected_msg}
        """

        act.expected_stdout = expected_stdout
        act.stdout = capsys.readouterr().out
        assert act.clean_stdout == act.clean_expected_stdout
        act.reset()