Ceph » CephFS

pdonnell@vossi04 /teuthology$ grep 'ceph_assert(_raw' *-2024-09-*-fs-*/*/teu*
²pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913565/teuthology.log:2024-09-21T07:33:51.595 INFO:tasks.ceph.mds.c.smithi045.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913565/teuthology.log:2024-09-21T07:33:51.597 INFO:tasks.ceph.mds.c.smithi045.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913565/teuthology.log:2024-09-21T07:33:51.625 INFO:tasks.ceph.mds.c.smithi045.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
pdonnell-2024-09-20_23:48:22-fs-wip-pdonnell-testing-20240920.212106-debug-distro-default-smithi/7913565/teuthology.log:2024-09-21T07:33:51.644 INFO:tasks.ceph.mds.c.smithi045.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917333/teuthology.log:2024-09-24T05:20:11.220 INFO:tasks.ceph.mds.a.smithi100.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917333/teuthology.log:2024-09-24T05:20:11.222 INFO:tasks.ceph.mds.a.smithi100.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917333/teuthology.log:2024-09-24T05:20:11.247 INFO:tasks.ceph.mds.a.smithi100.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917333/teuthology.log:2024-09-24T05:20:11.259 INFO:tasks.ceph.mds.a.smithi100.stderr:./src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917542/teuthology.log:2024-09-24T09:24:03.080 INFO:tasks.ceph.mds.a.smithi098.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-5115-g531c2373/rpm/el9/BUILD/ceph-19.3.0-5115-g531c2373/src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917542/teuthology.log:2024-09-24T09:24:03.082 INFO:tasks.ceph.mds.a.smithi098.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-5115-g531c2373/rpm/el9/BUILD/ceph-19.3.0-5115-g531c2373/src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917542/teuthology.log:2024-09-24T09:24:03.108 INFO:tasks.ceph.mds.a.smithi098.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-5115-g531c2373/rpm/el9/BUILD/ceph-19.3.0-5115-g531c2373/src/common/buffer.cc: 455: FAILED ceph_assert(_raw)
vshankar-2024-09-24_04:11:42-fs-wip-vshankar-testing-20240923.114534-debug-testing-default-smithi/7917542/teuthology.log:2024-09-24T09:24:03.121 INFO:tasks.ceph.mds.a.smithi098.stderr:/home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-5115-g531c2373/rpm/el9/BUILD/ceph-19.3.0-5115-g531c2373/src/common/buffer.cc: 455: FAILED ceph_assert(_raw)

[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `ceph-mds -f --cluster ceph -i c'.
Program terminated with signal SIGABRT, Aborted.
#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=140356951737920) at ./nptl/pthread_kill.c:44
44      ./nptl/pthread_kill.c: No such file or directory.
[Current thread is 1 (Thread 0x7fa766400640 (LWP 116762))]
(gdb) bt
#0  __pthread_kill_implementation (no_tid=0, signo=6, threadid=140356951737920) at ./nptl/pthread_kill.c:44
#1  __pthread_kill_internal (signo=6, threadid=140356951737920) at ./nptl/pthread_kill.c:78
#2  __GI___pthread_kill (threadid=140356951737920, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#3  0x00007fa76a642476 in __GI_raise (sig=6) at ../sysdeps/posix/raise.c:26
#4  0x0000557a2a6fa940 in reraise_fatal (signum=signum@entry=6) at global/./src/global/signal_handler.cc:88
#5  0x0000557a2a703276 in handle_oneshot_fatal_signal (signum=6) at global/./src/global/signal_handler.cc:367
#6  <signal handler called>
#7  __pthread_kill_implementation (no_tid=0, signo=6, threadid=140356951737920) at ./nptl/pthread_kill.c:44
#8  __pthread_kill_internal (signo=6, threadid=140356951737920) at ./nptl/pthread_kill.c:78
#9  __GI___pthread_kill (threadid=140356951737920, signo=signo@entry=6) at ./nptl/pthread_kill.c:89
#10 0x00007fa76a642476 in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#11 0x00007fa76a6287f3 in __GI_abort () at ./stdlib/abort.c:79
#12 0x00007fa76b0419c6 in ceph::__ceph_assert_fail(char const*, char const*, int, char const*) () from /usr/lib/ceph/libceph-common.so.2
#13 0x00007fa76b041a98 in ceph::__ceph_assert_fail(ceph::assert_data const&) () from /usr/lib/ceph/libceph-common.so.2
#14 0x00007fa76b2f4e1a in ceph::buffer::v15_2_0::ptr::c_str() const () from /usr/lib/ceph/libceph-common.so.2
#15 0x0000557a2a5f8578 in InodeStoreBase::dump (this=0x557a55480160, f=0x557a5444f180) at mds/./src/mds/CInode.cc:4592
#16 0x0000557a2a606b2a in CInode::dump (this=0x557a55480100, f=0x557a5444f180, flags=-33) at mds/./src/mds/CInode.cc:5135
#17 0x0000557a2a4af69c in MDCache::dump_tree (this=0x557a55278000, in=0x557a55480100, cur_depth=0, max_depth=0, f=0x557a5444f180) at mds/./src/mds/MDCache.cc:12913
#18 0x0000557a2a2e59e7 in operator() (__closure=__closure@entry=0x7fa7663fb9f0, f=f@entry=0x557a5444f180) at mds/./src/mds/MDSRank.cc:3292
#19 0x0000557a2a2e763a in MDSRank::command_dump_tree (this=this@entry=0x557a55272008, cmdmap=std::map with 4 elements = {...}, ss=..., f=f@entry=0x557a5444f180) at mds/./src/mds/MDSRank.cc:3303
#20 0x0000557a2a2ea587 in MDSRankDispatcher::handle_asok_command(std::basic_string_view<char, std::char_traits<char> >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&, ceph::Formatter*, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::basic_string_view<char, std::char_traits<char> >, ceph::buffer::v15_2_0::list&)>) (this=0x557a55272000, command=..., cmdmap=std::map with 4 elements = {...}, f=0x557a5444f180, 
    inbl=..., on_finish=...) at mds/./src/mds/MDSRank.cc:3018
#21 0x0000557a2a2ad905 in MDSDaemon::asok_command(std::basic_string_view<char, std::char_traits<char> >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&, ceph::Formatter*, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::basic_string_view<char, std::char_traits<char> >, ceph::buffer::v15_2_0::list&)>) (this=this@entry=0x557a5521ab00, command=..., cmdmap=std::map with 4 elements = {...}, f=f@entry=0x557a5444f180, 
    inbl=..., on_finish=...) at mds/./src/mds/MDSDaemon.cc:213
#22 0x0000557a2a2adcc4 in MDSSocketHook::call_async(std::basic_string_view<char, std::char_traits<char> >, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > >, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, boost::variant<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, bool, long, double, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >, std::vector<long, std::allocator<long> >, std::vector<double, std::allocator<double> > > > > > const&, ceph::Formatter*, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::basic_string_view<char, std::char_traits<char> >, ceph::buffer::v15_2_0::list&)>) (this=<optimized out>, command=..., cmdmap=std::map with 4 elements = {...}, f=0x557a5444f180, inbl=..., 
    on_finish=...) at mds/./src/mds/MDSDaemon.cc:131
#23 0x00007fa76b0314a0 in AdminSocket::execute_command(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, ceph::buffer::v15_2_0::list const&, std::function<void (int, std::basic_string_view<char, std::char_traits<char> >, ceph::buffer::v15_2_0::list&)>) () from /usr/lib/ceph/libceph-common.so.2
#24 0x00007fa76b0357d6 in AdminSocket::execute_command(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, ceph::buffer::v15_2_0::list const&, std::ostream&, ceph::buffer::v15_2_0::list*) () from /usr/lib/ceph/libceph-common.so.2
#25 0x00007fa76b036f3e in AdminSocket::do_accept() () from /usr/lib/ceph/libceph-common.so.2
#26 0x00007fa76b037ab0 in AdminSocket::entry() () from /usr/lib/ceph/libceph-common.so.2
#27 0x00007fa76b039170 in ?? () from /usr/lib/ceph/libceph-common.so.2
#28 0x00007fa76aadc2b3 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6
#29 0x00007fa76a694b43 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#30 0x00007fa76a726a00 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

real stack trace courtesy of @Brad Hubbard (thank you!!)