diff --git a/doc/README.services_extension b/doc/README.services_extension index a103fe035a..148b462197 100644 --- a/doc/README.services_extension +++ b/doc/README.services_extension @@ -265,3 +265,20 @@ Samples of use of new parameter in fbsvcmgr utility (supposing login and password are set using some other method): fbsvcmgr - action_nfix dbname /tmp/ecopy.fdb + +9) Services API extension - support of new nbackup feature to clean history table. +(Khorsun Vlad, hvlad@users.sourceforge.net, 2022) + +Action isc_action_svc_nbak get new parameter tags + isc_spb_nbk_clean_history : tell nbackup to clean RDB$HISTORY table + isc_spb_nbk_keep_days : specify how many recent rows should be kept in the history + isc_spb_nbk_keep_rows : specify how days back from today should be kept in the history + +Examples: +- make backup of level 1, clean RDB$HISTORY table and keep 1 recent row in it: + + fbsvcmgr action_nbak dbfile db.fdb nbk_file db.nbk nbk_level 1 nbk_clean_history nbk_keep_rows 1 + +- make backup of level 2, clean RDB$HISTORY table and keep rows for the last 7 days in it: + + fbsvcmgr action_nbak dbfile db.fdb nbk_file db.nbk nbk_level 2 nbk_clean_history nbk_keep_days 7 diff --git a/src/common/classes/ClumpletReader.cpp b/src/common/classes/ClumpletReader.cpp index b2ce2a3cbb..69cbb431c1 100644 --- a/src/common/classes/ClumpletReader.cpp +++ b/src/common/classes/ClumpletReader.cpp @@ -458,7 +458,11 @@ ClumpletReader::ClumpletType ClumpletReader::getClumpletType(UCHAR tag) const return StringSpb; case isc_spb_nbk_level: case isc_spb_options: + case isc_spb_nbk_keep_days: + case isc_spb_nbk_keep_rows: return IntSpb; + case isc_spb_nbk_clean_history: + return SingleTpb; } invalid_structure("unknown parameter for nbackup", tag); break; diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index 9595116fcf..57bd35f5e0 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -627,6 +627,9 @@ #define isc_spb_nbk_file 6 #define isc_spb_nbk_direct 7 #define isc_spb_nbk_guid 8 +#define isc_spb_nbk_clean_history 9 +#define isc_spb_nbk_keep_days 10 +#define isc_spb_nbk_keep_rows 11 #define isc_spb_nbk_no_triggers 0x01 #define isc_spb_nbk_inplace 0x02 #define isc_spb_nbk_sequence 0x04 diff --git a/src/include/firebird/impl/msg/nbackup.h b/src/include/firebird/impl/msg/nbackup.h index b98d41c45c..42ef7b1414 100644 --- a/src/include/firebird/impl/msg/nbackup.h +++ b/src/include/firebird/impl/msg/nbackup.h @@ -67,7 +67,7 @@ FB_IMPL_MSG(NBACKUP, 66, nbackup_err_eofhdr_restdb, -901, "00", "000", "Unexpect FB_IMPL_MSG(NBACKUP, 67, nbackup_lostguid_l0bk, -901, "00", "000", "Cannot get backup guid clumplet from L0 backup") FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 68, "Physical Backup Manager version @1") FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 69, "Enter name of the backup file of level @1 (\".\" - do not restore further):") -FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 70, " -D(IRECT) [ON | OFF] Use or not direct I/O when backing up database") +FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 70, " -D(IRECT) Use or not direct I/O when backing up database") FB_IMPL_MSG(NBACKUP, 71, nbackup_switchd_parameter, -901, "00", "000", "Wrong parameter @1 for switch -D, need ON or OFF") FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 72, "special options are:") FB_IMPL_MSG(NBACKUP, 73, nbackup_user_stop, -901, "08", "006", "Terminated due to user request") @@ -79,3 +79,10 @@ FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 78, " -I(NPLACE) Res FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 79, " -INPLACE option could corrupt the database that has changed since previous restore.") FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 80, " -SEQ(UENCE) Preserve original replication sequence") FB_IMPL_MSG(NBACKUP, 81, nbackup_seq_misuse, -901, "00", "000", "Switch -SEQ(UENCE) can be used only with -FIXUP or -RESTORE") +FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 82, " -CLEAN_HIST(ORY) Clean old records from backup history") +FB_IMPL_MSG_NO_SYMBOL(NBACKUP, 83, " -K(EEP) <(R)OWS | (D)AYS> How many recent rows (or days back from today) should be kept in the history") +FB_IMPL_MSG(NBACKUP, 84, nbackup_wrong_param, -901, "00", "000", "Wrong parameter value for switch @1") +FB_IMPL_MSG(NBACKUP, 85, nbackup_clean_hist_misuse, -901, "00", "000", "Switch -CLEAN_HISTORY can be used only with -BACKUP") +FB_IMPL_MSG(NBACKUP, 86, nbackup_clean_hist_missed, -901, "00", "000", "-KEEP can be used only with -CLEAN_HISTORY") +FB_IMPL_MSG(NBACKUP, 87, nbackup_keep_hist_missed, -901, "00", "000", "-KEEP is required with -CLEAN_HISTORY") +FB_IMPL_MSG(NBACKUP, 88, nbackup_second_keep_switch, -901, "00", "000", "-KEEP can be used one time only") diff --git a/src/include/gen/Firebird.pas b/src/include/gen/Firebird.pas index 6ec1ac4ce1..20edfdc1b3 100644 --- a/src/include/gen/Firebird.pas +++ b/src/include/gen/Firebird.pas @@ -4256,6 +4256,9 @@ const isc_spb_nbk_file = byte(6); isc_spb_nbk_direct = byte(7); isc_spb_nbk_guid = byte(8); + isc_spb_nbk_clean_history = byte(9); + isc_spb_nbk_keep_days = byte(10); + isc_spb_nbk_keep_rows = byte(11); isc_spb_nbk_no_triggers = $01; isc_spb_nbk_inplace = $02; isc_spb_nbk_sequence = $04; @@ -5780,6 +5783,11 @@ const isc_nbackup_deco_parse = 337117259; isc_nbackup_lostrec_guid_db = 337117261; isc_nbackup_seq_misuse = 337117265; + isc_nbackup_wrong_param = 337117268; + isc_nbackup_clean_hist_misuse = 337117269; + isc_nbackup_clean_hist_missed = 337117270; + isc_nbackup_keep_hist_missed = 337117271; + isc_nbackup_second_keep_switch = 337117272; isc_trace_conflict_acts = 337182750; isc_trace_act_notfound = 337182751; isc_trace_switch_once = 337182752; diff --git a/src/jrd/idx.h b/src/jrd/idx.h index 1ab8fb4e17..7960bd7e33 100644 --- a/src/jrd/idx.h +++ b/src/jrd/idx.h @@ -52,6 +52,7 @@ using Jrd::idx_metadata; using Jrd::idx_numeric; using Jrd::idx_string; using Jrd::idx_descending; +using Jrd::idx_timestamp_tz; #define INDEX(id, rel, unique, count) {(id), (UCHAR) (rel), (unique), (count), { #define SEGMENT(fld, type) {(fld), (type)} @@ -304,6 +305,10 @@ static const struct ini_idx_t indices[] = SEGMENT(f_pubtab_tab_name, idx_string), // table name SEGMENT(f_pubtab_pub_name, idx_string) // publication name }}, + // define index RDB$INDEX_57 for RDB$BACKUP_HISTORY RDB$TIMESTAMP; + INDEX(57, rel_backup_history, idx_descending, 1) + SEGMENT(f_backup_time, idx_timestamp_tz) // backup timestamp + }}, }; #define SYSTEM_INDEX_COUNT FB_NELEM(indices) diff --git a/src/jrd/svc.cpp b/src/jrd/svc.cpp index 80934a2e5e..fba0364ee1 100644 --- a/src/jrd/svc.cpp +++ b/src/jrd/svc.cpp @@ -2606,6 +2606,8 @@ bool Service::process_switches(ClumpletReader& spb, string& switches) string nbk_database, nbk_file, nbk_guid; int nbk_level = -1; + bool cleanHistory = false, keepHistory = false; + bool val_database = false; bool found = false; string::size_type userPos = string::npos; @@ -2676,6 +2678,30 @@ bool Service::process_switches(ClumpletReader& spb, string& switches) get_action_svc_string(spb, switches); break; + case isc_spb_nbk_clean_history: + if (cleanHistory) + { + (Arg::Gds(isc_unexp_spb_form) << Arg::Str("only one isc_spb_nbk_clean_history")).raise(); + } + if (!get_action_svc_parameter(spb.getClumpTag(), nbackup_action_in_sw_table, switches)) + { + return false; + } + cleanHistory = true; + break; + + case isc_spb_nbk_keep_days: + case isc_spb_nbk_keep_rows: + if (keepHistory) + { + (Arg::Gds(isc_unexp_spb_form) << Arg::Str("only one isc_spb_nbk_keep_days or isc_spb_nbk_keep_rows")).raise(); + } + switches += "-KEEP "; + get_action_svc_data(spb, switches, false); + switches += spb.getClumpTag() == isc_spb_nbk_keep_days ? "DAYS " : "ROWS "; + keepHistory = true; + break; + default: return false; } @@ -3175,6 +3201,16 @@ bool Service::process_switches(ClumpletReader& spb, string& switches) } else switches += nbk_guid; + + if (!cleanHistory && keepHistory) + { + (Arg::Gds(isc_missing_required_spb) << Arg::Str("isc_spb_nbk_clean_history")).raise(); + } + + if (cleanHistory && !keepHistory) + { + (Arg::Gds(isc_missing_required_spb) << Arg::Str("isc_spb_nbk_keep_days or isc_spb_nbk_keep_rows")).raise(); + } } switches += nbk_database; switches += nbk_file; diff --git a/src/jrd/vio.cpp b/src/jrd/vio.cpp index 9a35d25948..dd24ec6fb8 100644 --- a/src/jrd/vio.cpp +++ b/src/jrd/vio.cpp @@ -2005,7 +2005,6 @@ bool VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) { case rel_database: case rel_log: - case rel_backup_history: case rel_global_auth_mapping: protect_system_table_delupd(tdbb, relation, "DELETE", true); break; @@ -2293,6 +2292,11 @@ bool VIO_erase(thread_db* tdbb, record_param* rpb, jrd_tra* transaction) DFW_post_work(transaction, dfw_grant, &desc, id); break; + case rel_backup_history: + if (!tdbb->getAttachment()->locksmith(tdbb, USE_NBACKUP_UTILITY)) + protect_system_table_delupd(tdbb, relation, "DELETE", true); + break; + case rel_pub_tables: protect_system_table_delupd(tdbb, relation, "DELETE"); DFW_post_work(transaction, dfw_change_repl_state, "", 1); diff --git a/src/utilities/fbsvcmgr/fbsvcmgr.cpp b/src/utilities/fbsvcmgr/fbsvcmgr.cpp index 4632fea670..2a87e80624 100644 --- a/src/utilities/fbsvcmgr/fbsvcmgr.cpp +++ b/src/utilities/fbsvcmgr/fbsvcmgr.cpp @@ -559,6 +559,9 @@ const SvcSwitches nbackOptions[] = {"nbk_guid", putStringArgument, 0, isc_spb_nbk_guid, 0}, {"nbk_no_triggers", putOption, 0, isc_spb_nbk_no_triggers, 0}, {"nbk_direct", putStringArgument, 0, isc_spb_nbk_direct, 0}, + {"nbk_clean_history", putSingleTag, 0, isc_spb_nbk_clean_history, 0}, + {"nbk_keep_days", putIntArgument, 0, isc_spb_nbk_keep_days, 0}, + {"nbk_keep_rows", putIntArgument, 0, isc_spb_nbk_keep_rows, 0}, {0, 0, 0, 0, 0} }; diff --git a/src/utilities/nbackup/nbackup.cpp b/src/utilities/nbackup/nbackup.cpp index 26a1542173..eb35da6a5f 100644 --- a/src/utilities/nbackup/nbackup.cpp +++ b/src/utilities/nbackup/nbackup.cpp @@ -271,13 +271,17 @@ struct inc_header class NBackup { public: + enum CLEAN_HISTORY_KIND { NONE, DAYS, ROWS }; + NBackup(UtilSvc* _uSvc, const PathName& _database, const string& _username, const string& _role, - const string& _password, bool _run_db_triggers, bool _direct_io, const string& _deco) + const string& _password, bool _run_db_triggers, bool _direct_io, const string& _deco, + CLEAN_HISTORY_KIND cleanHistKind, int keepHistValue) : uSvc(_uSvc), newdb(0), trans(0), database(_database), username(_username), role(_role), password(_password), run_db_triggers(_run_db_triggers), direct_io(_direct_io), dbase(INVALID_HANDLE_VALUE), backup(INVALID_HANDLE_VALUE), - decompress(_deco), childId(0), db_size_pages(0), + decompress(_deco), m_cleanHistKind(cleanHistKind), m_keepHistValue(keepHistValue), + childId(0), db_size_pages(0), m_odsNumber(0), m_silent(false), m_printed(false), m_flash_map(false) { // Recognition of local prefix allows to work with @@ -334,6 +338,8 @@ private: FILE_HANDLE dbase; FILE_HANDLE backup; string decompress; + const CLEAN_HISTORY_KIND m_cleanHistKind; + const int m_keepHistValue; #ifdef WIN_NT HANDLE childId; HANDLE childStdErr; @@ -361,6 +367,7 @@ private: void attach_database(); void detach_database(); string to_system(const PathName& from); + void cleanHistory(); // Create/open database and backup void open_database_write(bool exclusive = false); @@ -576,11 +583,16 @@ void NBackup::create_database() void NBackup::close_database() { + if (dbase == INVALID_HANDLE_VALUE) + return; + #ifdef WIN_NT CloseHandle(dbase); #else close(dbase); #endif + + dbase = INVALID_HANDLE_VALUE; } string NBackup::to_system(const PathName& from) @@ -793,6 +805,10 @@ void NBackup::close_backup() { if (bakname == "stdout") return; + + if (backup == INVALID_HANDLE_VALUE) + return; + #ifdef WIN_NT CloseHandle(backup); if (childId != 0) @@ -817,6 +833,7 @@ void NBackup::close_backup() childId = 0; } #endif + backup = INVALID_HANDLE_VALUE; } void NBackup::fixup_database(bool repl_seq, bool set_readonly) @@ -1056,6 +1073,31 @@ void NBackup::internal_lock_database() pr_error(status, "begin backup: commit"); } +void NBackup::cleanHistory() +{ + if (m_cleanHistKind == NONE) + return; + + string sql; + if (m_cleanHistKind == DAYS) + { + sql.printf( + "DELETE FROM RDB$BACKUP_HISTORY WHERE RDB$TIMESTAMP < DATEADD(1 - %i DAY TO CURRENT_DATE)", + m_keepHistValue); + } + else + { + sql.printf( + "DELETE FROM RDB$BACKUP_HISTORY WHERE RDB$TIMESTAMP <= " + "(SELECT RDB$TIMESTAMP FROM RDB$BACKUP_HISTORY ORDER BY RDB$TIMESTAMP DESC " + "OFFSET %i ROWS FETCH FIRST 1 ROW ONLY)", + m_keepHistValue); + } + + if (isc_dsql_execute_immediate(status, &newdb, &trans, 0, sql.c_str(), SQL_DIALECT_CURRENT, NULL)) + pr_error(status, "execute history delete"); +} + void NBackup::get_database_size() { db_size_pages = 0; @@ -1243,7 +1285,7 @@ void NBackup::backup_database(int level, Guid& guid, const PathName& fname) default: pr_error(status, "fetch history query"); } - isc_dsql_free_statement(status, &stmt, DSQL_close); + isc_dsql_free_statement(status, &stmt, DSQL_drop); if (isc_commit_transaction(status, &trans)) pr_error(status, "commit history query"); } @@ -1560,6 +1602,9 @@ void NBackup::backup_database(int level, Guid& guid, const PathName& fname) in_sqlda->sqlvar[3].sqlind = &null_flag; if (isc_dsql_execute(status, &trans, &stmt, 1, in_sqlda)) pr_error(status, "execute history insert"); + + cleanHistory(); + isc_dsql_free_statement(status, &stmt, DSQL_drop); if (isc_commit_transaction(status, &trans)) pr_error(status, "commit history insert"); @@ -1871,6 +1916,9 @@ void nbackup(UtilSvc* uSvc) Guid guid; bool print_size = false, version = false, inc_rest = false, repl_seq = false; string onOff; + bool cleanHistory = false; + NBackup::CLEAN_HISTORY_KIND cleanHistKind = NBackup::CLEAN_HISTORY_KIND::NONE; + int keepHistValue = 0; const Switches switches(nbackup_action_in_sw_table, FB_NELEM(nbackup_action_in_sw_table), false, true); @@ -2054,6 +2102,37 @@ void nbackup(UtilSvc* uSvc) repl_seq = true; break; + case IN_SW_NBK_CLEAN_HISTORY: + cleanHistory = true; + break; + + case IN_SW_NBK_KEEP: + if (cleanHistKind != NBackup::CLEAN_HISTORY_KIND::NONE) + usage(uSvc, isc_nbackup_second_keep_switch); + + if (++itr >= argc) + missingParameterForSwitch(uSvc, argv[itr - 1]); + + keepHistValue = atoi(argv[itr]); + if (keepHistValue < 1) + usage(uSvc, isc_nbackup_wrong_param, argv[itr - 1]); + + if (++itr >= argc) + missingParameterForSwitch(uSvc, argv[itr - 1]); + + { // scope + string keepUnit = argv[itr]; + keepUnit.upper(); + + if (string("DAYS").find(keepUnit) == 0) + cleanHistKind = NBackup::CLEAN_HISTORY_KIND::DAYS; + else if (string("ROWS").find(keepUnit) == 0) + cleanHistKind = NBackup::CLEAN_HISTORY_KIND::ROWS; + else + usage(uSvc, isc_nbackup_wrong_param, argv[itr - 2]); + } + break; + default: usage(uSvc, isc_nbackup_unknown_switch, argv[itr]); break; @@ -2084,7 +2163,22 @@ void nbackup(UtilSvc* uSvc) usage(uSvc, isc_nbackup_seq_misuse); } - NBackup nbk(uSvc, database, username, role, password, run_db_triggers, direct_io, decompress); + if (cleanHistory) + { + // CLEAN_HISTORY could be used with BACKUP only + if (op != nbBackup) + usage(uSvc, isc_nbackup_clean_hist_misuse); + + if (cleanHistKind == NBackup::CLEAN_HISTORY_KIND::NONE) + usage(uSvc, isc_nbackup_keep_hist_missed); + } + else if (cleanHistKind != NBackup::CLEAN_HISTORY_KIND::NONE) + { + usage(uSvc, isc_nbackup_clean_hist_missed); + } + + NBackup nbk(uSvc, database, username, role, password, run_db_triggers, direct_io, + decompress, cleanHistKind, keepHistValue); try { switch (op) diff --git a/src/utilities/nbackup/nbkswi.h b/src/utilities/nbackup/nbkswi.h index 8daa971c97..363aa21707 100644 --- a/src/utilities/nbackup/nbkswi.h +++ b/src/utilities/nbackup/nbkswi.h @@ -49,6 +49,8 @@ const int IN_SW_NBK_DECOMPRESS = 14; const int IN_SW_NBK_ROLE = 15; const int IN_SW_NBK_INPLACE = 16; const int IN_SW_NBK_SEQUENCE = 17; +const int IN_SW_NBK_CLEAN_HISTORY = 18; +const int IN_SW_NBK_KEEP = 19; static const struct Switches::in_sw_tab_t nbackup_in_sw_table [] = @@ -75,6 +77,8 @@ static const struct Switches::in_sw_tab_t nbackup_action_in_sw_table [] = {IN_SW_NBK_SIZE, 0, "SIZE", 0, 0, 0, false, false, 17, 1, NULL, nboSpecial}, {IN_SW_NBK_DECOMPRESS, 0, "DECOMPRESS", 0, 0, 0, false, false, 74, 2, NULL, nboSpecial}, {IN_SW_NBK_SEQUENCE, 0, "SEQUENCE", 0, 0, 0, false, false, 80, 3, NULL, nboSpecial}, + {IN_SW_NBK_CLEAN_HISTORY, isc_spb_nbk_clean_history, "CLEAN_HISTORY", 0, 0, 0, false, false, 82, 10, NULL, nboSpecial}, + {IN_SW_NBK_KEEP, 0, "KEEP", 0, 0, 0, false, false, 83, 1, NULL, nboSpecial}, {IN_SW_NBK_NODBTRIG, 0, "T", 0, 0, 0, false, false, 0, 1, NULL, nboGeneral}, {IN_SW_NBK_NODBTRIG, 0, "NODBTRIGGERS", 0, 0, 0, false, false, 16, 3, NULL, nboGeneral}, {IN_SW_NBK_USER_NAME, 0, "USER", 0, 0, 0, false, false, 13, 1, NULL, nboGeneral},