Hi,
We are upgrading our EOS (Version 4.8.105 with CentOS 7.9) to current latest version 5.2.28 (on AlmaLinux 9.4). So, we are taking a backup/checkpoint of Quarkdb by redis-cli -p 7777 raft-checkpoint --path /locallog/backup and it’s go ok.
Now when we are validate those backup/checkpoint through quarkdb-validate-checkpoint, it show lots of error and at last it show error like " Aborted (Signal sent by tkill() 8772 0)
Aborted (core dumped)."
Then, are there any option to repair those corrupted checkpoint? Are there any possibility to move or migrate old EOS Quarkdb to Newest version EOS Quarkdb?
+++++++++++++++++
Error shown on quarkdb-validate-checkpoint:
[root@eos-slave metadata]# quarkdb-validate-checkpoint --path backup_1730807151
[1730812017028] INFO: Attempting to open ShardDirectory…
[1730812017029] INFO: — OK!
[1730812017029] INFO: Attempting to open StateMachine…
[1730812017029] INFO: Openning state machine ‘backup_1730807151/current/state-machine’.
[1730812017097] INFO: — OK! LAST-APPLIED: 1223974219
[1730812017097] INFO: Attempting to open RaftJournal…
[1730812017097] INFO: Opening raft journal ‘backup_1730807151/current/raft-journal’
terminate called after throwing an instance of ‘quarkdb::FatalException’
what(): Error while opening journal in backup_1730807151/current/raft-journal:IO error: While open directory: backup_1730807151/current/raft-journal: Too many open files ----- Stack trace (most recent call last):
#7 Object ", at 0xffffffffffffffff, in
#6 Object "/usr/bin/quarkdb-validate-checkpoint, at 0x411490, in _start
#5 Object "/usr/lib64/libc-2.17.so, at 0x7f124d926554, in __libc_start_main
#4 Object “/usr/bin/quarkdb-validate-checkpoint, at 0x410ac4, in main
Source “/usr/src/debug/quarkdb-0.4.2/tools/quarkdb-validate-checkpoint.cc”, line 142, in main [0x410ac4]
139: // Yes, let’s open it
140: //----------------------------------------------------------------------------
141: qdb_info(“Attempting to open RaftJournal…”);
> 142: quarkdb::RaftJournal *raftJournal = shardDirectory.getRaftJournal();
143: qdb_info(”— OK! LOG-SIZE: " << raftJournal->getLogSize() << ", COMMIT-INDEX: " << raftJournal->getCommitIndex() << ", LOG-START: " << raftJournal->getLogStart());
144:
145: //----------------------------------------------------------------------------
#3 Object "/usr/lib64/libXrdQuarkDB.so, at 0x7f1250534f3d, in quarkdb::ShardDirectory::getRaftJournal()
Source “/usr/src/debug/quarkdb-0.4.2/src/ShardDirectory.cc”, line 120, in getRaftJournal [0x7f1250534f3d]
117: qdb_throw("Cannot open raft journal: " << suberr);
118: }
119:
> 120: journalptr = new RaftJournal(raftJournalPath());
121: return journalptr;
122: }
#2 Object "/usr/lib64/libXrdQuarkDB.so, at 0x7f125057f2d9, in quarkdb::RaftJournal::RaftJournal(std::string const&)
Source “/usr/src/debug/quarkdb-0.4.2/src/raft/RaftJournal.cc”, line 205, in RaftJournal [0x7f125057f2d9]
202: }
203:
204: RaftJournal::RaftJournal(const std::string &filename) {
> 205: openDB(filename);
206: ensureFsyncPolicyInitialized();
207: initialize();
208: }
#1 Object "/usr/lib64/libXrdQuarkDB.so, at 0x7f125057bc59, in quarkdb::RaftJournal::openDB(std::string const&)
Source “/usr/src/debug/quarkdb-0.4.2/src/raft/RaftJournal.cc”, line 186, in openDB [0x7f125057bc59]
183: options.create_if_missing = true;
184: options.max_manifest_file_size = 1024 * 1024;
185: rocksdb::Status status = rocksdb::DB::Open(options, path, &db);
> 186: if(!status.ok()) qdb_throw("Error while opening journal in " << path << “:” << status.ToString());
187: }
188:
189: RaftJournal::RaftJournal(const std::string &filename, RaftClusterID clusterID, const std::vector &nodes, LogIndex startIndex, FsyncPolicy fsyncPolicy) {
#0 Object “/usr/lib64/libXrdQuarkDB.so, at 0x7f12504f5dd8, in quarkdb::errorStacktrace(bool)
| Source “/usr/src/debug/quarkdb-0.4.2/src/Utils.cc”, line 51, in getStacktrace
| 49: }
| 50:
| > 51: return SSTR(” ----- " << getStacktrace() << suffixMessage);
| 52: }
Source “/usr/src/debug/quarkdb-0.4.2/src/utils/Stacktrace.hh”, line 36, in errorStacktrace [0x7f12504f5dd8]
33: inline std::string getStacktrace() {
34: std::ostringstream ss;
35:
> 36: backward::StackTrace st; st.load_here(32);
37: backward::Printer p;
38: p.object = true;
39: p.color_mode = backward::ColorMode::always;
Stack trace (most recent call last):
#12 Object ", at 0xffffffffffffffff, in
#11 Object "/usr/bin/quarkdb-validate-checkpoint, at 0x411490, in _start
#10 Object “/usr/lib64/libc-2.17.so, at 0x7f124d926554, in __libc_start_main
#9 Source “/usr/src/debug/quarkdb-0.4.2/tools/quarkdb-validate-checkpoint.cc”, line 142, in main [0x410ac4]
139: // Yes, let’s open it
140: //----------------------------------------------------------------------------
141: qdb_info(“Attempting to open RaftJournal…”);
> 142: quarkdb::RaftJournal *raftJournal = shardDirectory.getRaftJournal();
143: qdb_info(”— OK! LOG-SIZE: " << raftJournal->getLogSize() << ", COMMIT-INDEX: " << raftJournal->getCommitIndex() << ", LOG-START: " << raftJournal->getLogStart());
144:
145: //----------------------------------------------------------------------------
#8 Source “/usr/src/debug/quarkdb-0.4.2/src/ShardDirectory.cc”, line 120, in getRaftJournal [0x7f1250534f3d]
117: qdb_throw("Cannot open raft journal: " << suberr);
118: }
119:
> 120: journalptr = new RaftJournal(raftJournalPath());
121: return journalptr;
122: }
#7 Source “/usr/src/debug/quarkdb-0.4.2/src/raft/RaftJournal.cc”, line 205, in RaftJournal [0x7f125057f2d9]
202: }
203:
204: RaftJournal::RaftJournal(const std::string &filename) {
> 205: openDB(filename);
206: ensureFsyncPolicyInitialized();
207: initialize();
208: }
#6 Source “/usr/src/debug/quarkdb-0.4.2/src/raft/RaftJournal.cc”, line 186, in openDB [0x7f125057bcd0]
183: options.create_if_missing = true;
184: options.max_manifest_file_size = 1024 * 1024;
185: rocksdb::Status status = rocksdb::DB::Open(options, path, &db);
> 186: if(!status.ok()) qdb_throw("Error while opening journal in " << path << “:” << status.ToString());
187: }
188:
189: RaftJournal::RaftJournal(const std::string &filename, RaftClusterID clusterID, const std::vector &nodes, LogIndex startIndex, FsyncPolicy fsyncPolicy) {
#5 Object "/usr/lib64/libstdc++.so.6.0.19, at 0x7f124e248c52, in __cxa_throw
#4 Object "/usr/lib64/libstdc++.so.6.0.19, at 0x7f124e248a32, in std::terminate()
#3 Object "/usr/lib64/libstdc++.so.6.0.19, at 0x7f124e248a05, in
#2 Object "/usr/lib64/libstdc++.so.6.0.19, at 0x7f124e24aa94, in __gnu_cxx::__verbose_terminate_handler()
#1 Object "/usr/lib64/libc-2.17.so, at 0x7f124d93ba77, in abort
#0 Object "/usr/lib64/libc-2.17.so, at 0x7f124d93a387, in raise
Aborted (Signal sent by tkill() 29097 0)
Aborted (core dumped)
[root@eos-slave metadata]#
+++++++++++++++++
Regards
Prasun