From df5a046cdb9ae2b569ae00dad11e26478ffacd0b Mon Sep 17 00:00:00 2001 From: zotov Date: Fri, 23 Sep 2022 19:39:38 +0300 Subject: [PATCH] Added/Updated bugs\core_6248_test.py. Totally reimplemented. See notes. Checked on Windows and Linux, 4.0.1.2692 (SS/CS), 5.0.0.736 (SS/CS). --- tests/bugs/core_6248_test.py | 477 +++++++++++++++++++++++++---------- 1 file changed, 346 insertions(+), 131 deletions(-) diff --git a/tests/bugs/core_6248_test.py b/tests/bugs/core_6248_test.py index b1ad02b1..809b8dcc 100644 --- a/tests/bugs/core_6248_test.py +++ b/tests/bugs/core_6248_test.py @@ -5,155 +5,370 @@ ID: issue-6492 ISSUE: 6492 TITLE: A number of errors when database name is longer than 255 symbols DESCRIPTION: - Test verifies that one may to create DB with total path plus name length L = 255 and 259 characters. - Each DB is then subject for 'gbak -b', 'gbak -c', 'gstat -h', 'gfix -sweep' and 'gfix -v -full'. - All these commands must NOT issue something to their STDERR. + From ticket title one may to conclude that we have ability to really create databases with full length (path + file name) + more than 255 characters, but it is not so. + Actually, on Windows and Linux one can not create files with length more than 255 characters. + Moreover, FB has its own, more strict limitation: command CONNECT '' can not operate with files with length + more than (253 -

) symbols, including enclosing quotes, where

is length of used protocol prefix: 'inet://', 'localhost:'. - STDOUT-log of initial SQL must contain full DB name. - Changed part of firebird.log for SWEEP and VALIDATION also must have full DB name (this is verified using regexp): - +[tab]Database: C:\\FBTESTING\\QA\\FBT-REPO\\TMP\\ABC.FDB // for validation - +[tab]Database "C:\\FBTESTING\\QA\\FBT-REPO\\TMP\\ABC.FDB // for sweep + Test verifies ability to create DB with max possible length (i.e. which allows to run 'CONNECT ...' command) and does that + for all protocols that are supported by current OS (Linux + local, inet; Windows: local, inet, xnet). + For each used protocol we try to perform several operations using gfix utility and after this - fbsvcmgr. - STDOUT-logs of backup, restore and gstat currently (09-mar-2020) have only truncated name (~235...241 chars). - This may change in the future if FB developers will decide to fix this small inconveniences. + After each operation we check mon$database and 'construct' string based on values from this table that must be returned + as tail of 'Attributes' line in the 'gstat -h' command. These lines must be equal. + One need to pay attantion that when database is in 'backup-lock' state then changes in its attributes can not be seen + in the output of 'gstat -h'. Because of this, we do not change any attributes between 'alter database begin/end backup'. + Also, one need to take in account that we can not check full ability of 'nbackup -L' command for databases which names + are maximal possible, because adding of suffix '.delta' will cause exceeding max possible length = 253 characters. + Because of this, test uses 'alter database begin add difference file with name = '' (i.e. its has + the same length and main DB file). - For L=259 we must see in backup log following phrase: - gbak:text for attribute 7 is too large in put_asciz(), truncating to 255 bytes - - but currently this is not checked here. -[09.02.2022] pcisar - Fails on Windows10 / 4.0.1 with: - "CreateFile (create)" operation for file "..." - -Error while trying to create file - -System can't find specified path - Variant with 255 chars fails in init script, while 259 chars variant fails in database fixture while - db creation. - On national windows with OS i/o error messages in locale.getpreferredencoding(), it may fail while - reading stderr from isql. But using io_enc=locale.getpreferredencoding() will show the message. +NOTES: + [09.03.2020] pzotov // old comments, to be deleted later. + STDOUT-logs of backup, restore and gstat currently (09-mar-2020) have only truncated name (~235...241 chars). + This may change in the future if FB developers will decide to resolve this problem. + + For L=259 we must see in backup log following phrase: + gbak:text for attribute 7 is too large in put_asciz(), truncating to 255 bytes + - but currently this is not checked here. + [09.02.2022] pcisar + Fails on Windows10 / 4.0.1 with: + "CreateFile (create)" operation for file "..." + -Error while trying to create file + -System can't find specified path + Variant with 255 chars fails in init script, while 259 chars variant fails in database fixture while + db creation. + On national windows with OS i/o error messages in locale.getpreferredencoding(), it may fail while + reading stderr from isql. But using io_enc=locale.getpreferredencoding() will show the message. + [23.09.2022] pzotov + 1. Database header (that is produced by 'gstat -h' command) contains in its 1st line full path + name of DB file, + enclosed into double quotes. When length of path+filename is 243 or greater, this string begins look as 'cuted', + i.e. ellipsis will be shown at the end of this name (instead of extension and closed double quote). + This means that we have to be aware about applying regexp during parsing gstat output: final quote may miss! + 2. Currently one can not use 'act.svccmgr()' calls because of need to specify different protocols when check fbsvcmgr: + fbsvcmgr service_mgr ... + fbsvcmgr inet://localhost:service_mgr ... + fbsvcmgr xnet://service_mgr ... + Because of this, subprocess.run() is used to invoke fbsvcmgr + + Checked on Windows and Linux, 4.0.1.2692 (SS/CS), 5.0.0.736 (SS/CS). JIRA: CORE-6248 FBTEST: bugs.core_6248 """ - -import pytest +import os +import subprocess +import locale import re import time import platform -from difflib import unified_diff +from pathlib import Path +#from difflib import unified_diff +#from firebird.driver import SrvRepairFlag + +import pytest from firebird.qa import * -from firebird.driver import SrvRepairFlag - -init_script = """ - set list on; - - create exception exc_dbname_diff q'{Value in mon$database.mon$database_name differs from rdb$get_context('SYSTEM', 'DB_NAME'):@1@2@3=== vs ===@4@5}'; - set term ^; - execute block returns( - mon_database_column varchar(260) - ,sys_context_db_name varchar(260) - ) as - declare lf char(1) = x'0A'; - begin - select - mon$database_name as mon_database_column - from mon$database - into mon_database_column; - - sys_context_db_name = rdb$get_context('SYSTEM', 'DB_NAME'); - - if ( substring( sys_context_db_name from 1 for 255 ) is distinct from mon_database_column ) then - begin - exception exc_dbname_diff using( - lf - ,mon_database_column - ,lf - ,lf - ,sys_context_db_name - ); - end - - suspend; - end - ^ - set term ;^ - commit; -""" db = db_factory() +act = python_act('db', substitutions = [('[\t ]+', ' ')]) -act = python_act('db') +# We have to limit length of temp_file with 255 characters. +# CentOS-7: OSError: [Errno 36] File name too long: +tmp_file = temp_file( ('0123456789' * 26)[:255] ) -expected_stdout = """ - ddl : found at least 255 characters - backup : found truncated DB name. - restore : found truncated DB name. - gstat : found truncated DB name. - fblog_diff_sweep : found at least 255 characters - fblog_diff_validate : found at least 255 characters -""" +#----------------------------------------------- +def check_db_hdr_info(act: Action, db_file_chk:Path, interested_patterns, capsys): -@pytest.fixture -def test_db(request: pytest.FixtureRequest, db_path) -> Database: - required_name_len = request.param[0] - chars2fil = request.param[1] - filename = (chars2fil * 1000)[:required_name_len - len(str(db_path)) - 4] + '.fdb' - db = Database(db_path, filename) - db.create() - yield db - db.drop() + # 1. Obtain attributes from mon$database: get page buffers, 'build' attributes row and get sweep interval. + # These values will be displayed in the form of three separate LINES, without column names. + # Content of this output must be equal to gstat filtered values, with exception of leading spaces: + sql_txt = f""" + set list on; -MINIMAL_LEN_TO_SHOW = 255 + -- Make connect using local protocol. + -- NOTE: 'command error' raises here if length of '{db_file_chk}' (including qutes!) greater than 255. + connect '{db_file_chk}' user {act.db.user}; -PATTERN = re.compile('\\+\\s+Database[:]{0,1}\\s+"{0,1}', re.IGNORECASE) + select + 'Page buffers ' || mon$page_buffers as " " + ,'Attributes ' || iif(trim(attr_list) = '', '', substring(attr_list from 2)) + as " " + ,'Sweep interval: ' || mon$sweep_interval as " " + from ( + select + mon$page_buffers + ,mon$forced_writes + ,mon$backup_state + ,mon$reserve_space + ,mon$shutdown_mode + ,mon$read_only + ,mon$replica_mode + ,mon$sweep_interval + ,iif(mon$forced_writes = 1, ', force write', '') + || iif(mon$reserve_space = 0, ', no reserve', '') + || decode(mon$backup_state, 2, ', merge', 1, ', backup lock', '') + || decode(mon$shutdown_mode, 3, 'full shutdown', 2, ', single-user maintenance', 1, ', multi-user maintenance', '') + -- !! NEED TRIM() !! otherwise 10 spaces will be inserted if mon$read_only=0. + -- Discussed with Vladet al, letters since 23.09.2022 10:57. + || trim(iif(mon$read_only<>0, ', read only', '')) + || decode(mon$replica_mode, 2, ', read-write replica', 1, ', read-only replica', '') + as attr_list + from mon$database + ); + commit; + """ + # Example of output: + # Page buffers 3791 + # Attributes force write, no reserve, single-user maintenance, read only, read-write replica + # Sweep interval: 5678 -def check_filename_presence(lines, *, log_name: str, db: Database): - filename = str(db.db_path) # To convert Path to string - for line in lines: - if log_name not in ('fblog_diff_sweep', 'fblog_diff_validate') or line.startswith('+') and PATTERN.search(line): - if filename[:MINIMAL_LEN_TO_SHOW].upper() in line.upper(): - print(f'{log_name} : found at least {str(MINIMAL_LEN_TO_SHOW)} characters') - return - elif filename[:128].upper() in line.upper(): - print(f'{log_name} : found truncated DB name.') - return - print(f'{log_name} : DB NAME NOT FOUND') + act.isql(switches = ['-q'], input = sql_txt, connect_db=False, credentials = False, combine_output = True, io_enc = locale.getpreferredencoding()) + expected_attr_from_mon_db = act.stdout + + #------------------------------------------------------ + # 2. Run 'gstat -h', filtering its output and compare with data that was obtained from mon$database. + # NOTE: database name with backslashes (on Windows) must be checked without regexp work, only via 'IN'. + # On Winows databases are created in UPPER form, so we have to remove case sensitivity. + act.gstat(switches=['-h', db_file_chk], connect_db = False, io_enc = locale.getpreferredencoding()) + db_guid = '' + db_found = '' + db_cuted = ('Database "' + str(db_file_chk) + '"').lower()[:250] + for line in act.stdout.split('\n'): + if act.match_any(line, interested_patterns): + if 'Database GUID' in line: + db_guid = line + elif db_cuted in line.lower(): + print(db_cuted) + db_found = 1 + else: + print(line) -@pytest.mark.skipif(platform.system() == 'Windows', reason='FIXME: see notes') + if db_found: + act.expected_stdout = f""" + {db_cuted} + {expected_attr_from_mon_db} + """ + act.stdout = capsys.readouterr().out + assert act.clean_stdout == act.clean_expected_stdout + act.reset() + else: + print('Cuted DB name:',db_cuted) + for line in act.stdout.split('\n'): + print('gstat output: ',line) + assert db_found,'COULD NOT FIND NAME OF DATABASE IN THE GSTAT HEADER' + # 3. Return GUID of database (can be compared after b/r with GUID of restored database: they always must differ): + return db_guid + +#----------------------------------------------- @pytest.mark.version('>=4.0') -@pytest.mark.parametrize('test_db', [pytest.param((255, 'abc255def'), id='255'), - pytest.param((259, 'qwe259rty'), id='259')], indirect=True) -def test_1(act: Action, test_db: Database, capsys): - # INIT test - act.isql(switches=['-q', test_db.dsn], input=init_script, connect_db=False) - check_filename_presence(act.stdout.splitlines(), log_name='ddl', db=test_db) - # GBAK BACKUP test - backup_name = test_db.db_path.with_name(f"tmp_6248_backup_{len(test_db.db_path.with_suffix('').name)}.fbk") - act.reset() - act.gbak(switches=['-b', '-se', 'localhost:servce_mgr', '-v', '-st', 'tdwr', str(test_db.db_path), str(backup_name)]) - check_filename_presence(act.stdout.splitlines(), log_name='backup', db=test_db) - # GBAK RESTORE test - act.reset() - act.gbak(switches=['-rep', '-se', 'localhost:servce_mgr', '-v', '-st', 'tdwr', str(backup_name), str(test_db.db_path)]) - check_filename_presence(act.stdout.splitlines(), log_name='restore', db=test_db) - # GSTAT test - act.reset() - act.gstat(switches=['-h', test_db.dsn], connect_db=False) - check_filename_presence(act.stdout.splitlines(), log_name='gstat', db=test_db) - # SWEEP test - log_before = act.get_firebird_log() - with act.connect_server() as srv: - srv.database.sweep(database=test_db.db_path) - time.sleep(1) # Let firebird.log to be fulfilled with text about just finished SWEEP - log_after = act.get_firebird_log() - check_filename_presence(list(unified_diff(log_before, log_after)), log_name='fblog_diff_sweep', db=test_db) - # VALIDATE test - log_before = act.get_firebird_log() - with act.connect_server() as srv: - srv.database.repair(database=test_db.db_path, flags=SrvRepairFlag.FULL | SrvRepairFlag.VALIDATE_DB) - time.sleep(1) # Let firebird.log to be fulfilled with text about just finished VALIDATION - log_after = act.get_firebird_log() - check_filename_presence(list(unified_diff(log_before, log_after)), log_name='fblog_diff_validate', db=test_db) - # Check - act.reset() - act.expected_stdout = expected_stdout - act.stdout = capsys.readouterr().out - assert act.clean_stdout == act.clean_expected_stdout +def test_1(act: Action, tmp_file:Path, capsys): + + ################# + # ### ACHTUNG ### + ################# + # DO NOT include ending double quote into the database name pattern! + # String with database name is CUTED OFF when length of full path + filename is 243 and above: + # 243 --> Database "C:\TEMP\...\01234567890<...>012345.F... + # 244 --> Database "C:\TEMP\...\01234567890<...>0123456.... + # 245 --> Database "C:\TEMP\...\01234567890<...>01234567... + # 246 ... 255 -- same as for 245. NB: for N=244 line differs from all others. + # All these (cuted) strings have length = 254 bytes and do NOT contain ending double quote. + # Because of this, we must include this character into the pattern only as OPTIONAL, i.e.: |'Database\s+"\S+(")?'| + # + interested_patterns = ( 'Database\s+"\S+(")?', '[\t ]*Attributes([\t ]+\w+)?', '[\t ]*Page buffers([\t ]+\d+)', '[\t ]*Sweep interval(:)?([\t ]+\d+)', 'Database GUID') + interested_patterns = [re.compile(p, re.IGNORECASE) for p in interested_patterns] + protocol_list = ('', 'inet://', 'xnet://') if os.name == 'nt' else ('', 'inet://',) + + full_str = str(tmp_file.absolute()) + + for utility in ('gfix', 'fbsvcmgr'): + for protocol_prefix in protocol_list: + + # NB: most strict limit for DB filename length origins from isql 'CONNECT' command: + # 'command error' raises there if length of '{db_file_chk}' (including qutes!) greater than 255. + # Because of this, we can not operate with files with length of full name greater than 253 bytes. + # + db_file_len = 253 - len(protocol_prefix) + + db_file_chk = Path((full_str[:db_file_len-4] + '.fdb').lower()) + db_file_dif = Path(os.path.splitext(db_file_chk)[0] + '.dif') + db_file_fbk = Path(os.path.splitext(db_file_chk)[0] + '.fbk') + + db_file_dsn = '' + svc_call_starting_part = [] + if utility == 'gfix': + db_file_dsn = protocol_prefix + str(db_file_chk) + else: + db_file_dsn = db_file_chk + fb_svc_name = protocol_prefix + 'service_mgr' + svc_call_starting_part = [ act.vars['fbsvcmgr'], fb_svc_name, '-user', act.db.user, '-password', act.db.password ] + + sql_txt = f""" + set list on; + create database '{db_file_dsn}' user {act.db.user} password '{act.db.password}'; + select lower(mon$database_name) as mon_db_name from mon$database; + select lower(rdb$get_context('SYSTEM', 'DB_NAME')) as ctx_db_name from mon$database; + commit; + """ + + act.expected_stdout = f""" + MON_DB_NAME {db_file_chk} + CTX_DB_NAME {db_file_chk} + """ + + act.isql(switches = ['-q'], input = sql_txt, connect_db=False, credentials = False, combine_output = True, io_enc = locale.getpreferredencoding()) + assert act.clean_stdout == act.clean_expected_stdout + act.reset() + + svc_retcode = 0 + if utility == 'gfix': + act.gfix(switches=['-buffers', '3791', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_page_buffers', '3791', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + if utility == 'gfix': + act.gfix(switches=['-write','sync', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_write_mode', 'prp_wm_sync', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + if utility == 'gfix': + act.gfix(switches=['-housekeeping','5678', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_sweep_interval', '5678', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + if utility == 'gfix': + act.gfix(switches=['-use','full', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_reserve_space', 'prp_res_use_full', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + if utility == 'gfix': + act.gfix(switches=['-sweep', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_repair', 'rpr_sweep_db', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + if act.is_version('>=4'): + if utility == 'gfix': + act.gfix(switches=['-replica','read_write', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_replica_mode', 'prp_rm_readwrite', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + sql_txt = f""" + -- Make connect using local protocol. + -- NOTE: 'command error' raises here if length of '{db_file_chk}' (including qutes!) greater than 255. + connect '{db_file_chk}' user {act.db.user}; + alter database add difference file '{db_file_dif}'; + alter database begin backup; + alter database set linger to 100; + """ + act.isql(switches = ['-q'], input = sql_txt, connect_db=False, credentials = False, combine_output = True, io_enc = locale.getpreferredencoding()) + assert act.clean_stdout == act.clean_expected_stdout + act.reset() + _ = check_db_hdr_info(act, db_file_chk, interested_patterns, capsys) + + + sql_txt = f""" + -- Make connect using local protocol. + -- NOTE: 'command error' raises here if length of '{db_file_chk}' (including qutes!) greater than 255. + connect '{db_file_chk}' user {act.db.user}; + alter database set linger to 0; + alter database end backup; + """ + act.isql(switches = ['-q'], input = sql_txt, connect_db=False, credentials = False, combine_output = True, io_enc = locale.getpreferredencoding()) + assert act.clean_stdout == act.clean_expected_stdout + act.reset() + + + if utility == 'gfix': + act.gfix(switches=['-mode','read_only', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_access_mode', 'prp_am_readonly', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + _ = check_db_hdr_info(act, db_file_chk, interested_patterns, capsys) + + + if utility == 'gfix': + act.gfix(switches=['-shut','single', '-at', '20', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_shutdown_mode', 'prp_sm_single', 'prp_deny_new_attachments', '20', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + src_guid = check_db_hdr_info(act, db_file_chk, interested_patterns, capsys) + + + if utility == 'gfix': + act.gfix(switches=['-online', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_properties', 'prp_online_mode', 'prp_sm_normal', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + if utility == 'gfix': + act.gfix(switches=['-v', '-full', db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_repair', 'rpr_validate_db', 'rpr_full', 'dbname', db_file_chk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + if utility == 'gfix': + act.gbak(switches=['-b', db_file_dsn, db_file_fbk], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_backup', 'dbname', db_file_chk, 'bkp_file', db_file_fbk], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + + if utility == 'gfix': + act.gbak(switches=['-rep', db_file_fbk, db_file_dsn], combine_output = True, io_enc = locale.getpreferredencoding()) + else: + svc_retcode = (subprocess.run( svc_call_starting_part + ['action_restore', 'dbname', db_file_chk, 'bkp_file', db_file_fbk, 'res_replace' ], stderr = subprocess.STDOUT)).returncode + + assert '' == act.stdout and svc_retcode == 0 + act.reset() + + new_guid = check_db_hdr_info(act, db_file_chk, interested_patterns, capsys) + + + print('GUID changed ? ==> ', src_guid != new_guid) + act.expected_stdout = """ + GUID changed ? ==> True + """ + act.stdout = capsys.readouterr().out + assert act.clean_stdout == act.clean_expected_stdout + act.reset() + + for f in (db_file_chk,db_file_dif,db_file_fbk): + f.unlink(missing_ok=True) +