Actions
Tasks #73346
closedTasks #63293: Implement fscrypt in libcephfs and cephfs-fuse
Fscrypt LlreadvLlwritevLargeBuffers fails
% Done:
0%
Reviewed:
Affected Versions:
Component(FS):
Labels (FS):
Pull request ID:
Tags (freeform):
Merge Commit:
Fixed In:
Released In:
Upkeep Timestamp:
Description
Stack trace shows len=2147483648 which is higher than INT_MAX due to rounding up read size to FSCRYPT_BLOCK_SIZE
#0 __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=6,
no_tid=no_tid@entry=0) at pthread_kill.c:44
#1 0x00007ffff587f843 in __pthread_kill_internal (threadid=<optimized out>, signo=6)
at pthread_kill.c:78
#2 0x00007ffff58263de in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
#3 0x00007ffff580d942 in __GI_abort () at abort.c:79
#4 0x00007ffff64978c7 in ceph::__ceph_assert_fail (
assertion=0x5555558a97c0 "pos <= (uint64_t) 0x7fffffff", file=<optimized out>, line=1727,
func=<optimized out>) at gsrc/common/assert.cc:101
#5 0x00007ffff6497a1f in ceph::__ceph_assert_fail (ctx=...)
at gsrc/common/assert.cc:110
#6 0x000055555569d1bf in ceph::__ceph_assert_fail<&assert_data_ctx> ()
at gsrc/include/ceph_assert.h:57
#7 0x00005555556ac650 in operator() (__closure=<optimized out>, eval=false)
at gsrc/osdc/ObjectCacher.cc:1727
#8 ObjectCacher::_readx (this=this@entry=0x555555c84010, rd=rd@entry=0x55565c9491b0,
oset=oset@entry=0x7fffd800df08, onfinish=onfinish@entry=0x5556769b1cd0,
external_call=external_call@entry=true, trace=trace@entry=0x7fffffffcc7f,
holes=0x7fffffffcdb0) at gsrc/osdc/ObjectCacher.cc:1727
#9 0x00005555556acb2a in ObjectCacher::readx (this=this@entry=0x555555c84010,
rd=rd@entry=0x55565c9491b0, oset=oset@entry=0x7fffd800df08,
onfinish=onfinish@entry=0x5556769b1cd0, parent_trace=parent_trace@entry=0x0,
holes=holes@entry=0x7fffffffcdb0)
at gsrc/osdc/ObjectCacher.cc:1413
#10 0x0000555555613edc in ObjectCacher::file_read_ex (this=0x555555c84010,
oset=oset@entry=0x7fffd800df08, layout=layout@entry=0x7fffd800db70, snapid=..., offset=0,
len=2147483648, bl=0x7fffffffd230, flags=0, holes=0x7fffffffcdb0, onfinish=0x5556769b1cd0)
at gsrc/osdc/ObjectCacher.h:722
#11 0x0000555555602d15 in Client::_read_async (this=this@entry=0x555555b7d260,
f=f@entry=0x5555559fcc90, off=off@entry=0, len=len@entry=2147483647,
bl=bl@entry=0x7fffffffd230, onfinish=0x555676db2960)
at /usr/include/c++/14/bits/unique_ptr.h:193
#12 0x0000555555603893 in Client::_read (this=this@entry=0x555555b7d260,
f=f@entry=0x5555559fcc90, offset=offset@entry=0, size=2147483647,
bl=bl@entry=0x7fffffffd230, onfinish=onfinish@entry=0x7fffffffd440, read_for_write=false)
at /usr/include/c++/14/bits/unique_ptr.h:193
#13 0x0000555555607670 in Client::_preadv_pwritev_locked (this=this@entry=0x555555b7d260,
fh=fh@entry=0x5555559fcc90, iov=iov@entry=0x7fffffffd300, iovcnt=iovcnt@entry=2,
offset=offset@entry=0, write=write@entry=false, clamp_to_int=true,
onfinish=0x7fffffffd440, blp=0x7fffffffd230, do_fsync=false, syncdataonly=false)
at gsrc/client/Client.cc:12069
#14 0x0000555555608265 in Client::ll_preadv_pwritev (this=0x555555b7d260, fh=0x5555559fcc90,
iov=iov@entry=0x7fffffffd300, iovcnt=iovcnt@entry=2, offset=offset@entry=0,
--Type <RET> for more, q to quit, c to continue without paging--r
write=write@entry=false, onfinish=0x7fffffffd440, bl=0x7fffffffd230, do_fsync=false, syncdataonly=false) at gsrc/client/Client.cc:17188
#15 0x0000555555582b49 in TestClient_LlreadvLlwritevLargeBuffers_Test::TestBody (this=0x555555b78950) at gsrc/test/client/nonblocking.cc:782
#16 0x0000555555693269 in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x555555b78950, method=<optimized out>,
location=location@entry=0x5555558a88a3 "the test body") at gsrc/googletest/googletest/src/gtest.cc:2653
#17 0x000055555569aace in testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x555555b78950, method=&virtual testing::Test::TestBody(),
location=location@entry=0x5555558a88a3 "the test body") at gsrc/googletest/googletest/src/gtest.cc:2689
#18 0x000055555568aac4 in testing::Test::Run (this=this@entry=0x555555b78950) at gsrc/googletest/googletest/src/gtest.cc:2728
#19 0x000055555568ac1a in testing::TestInfo::Run (this=0x5555559f1590) at gsrc/googletest/googletest/src/gtest.cc:2874
#20 0x000055555568c493 in testing::TestSuite::Run (this=0x5555559f9520) at gsrc/googletest/googletest/src/gtest.cc:3052
#21 0x000055555568f118 in testing::internal::UnitTestImpl::RunAllTests (this=0x555555a3df60) at gsrc/googletest/googletest/src/gtest.cc:6004
#22 0x00005555556938a1 in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=object@entry=0x555555a3df60, method=<optimized out>,
location=location@entry=0x5555558ca5f8 "auxiliary test code (environments or event listeners)") at gsrc/googletest/googletest/src/gtest.cc:2653
#23 0x000055555569b02b in testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x555555a3df60,
method=(bool (testing::internal::UnitTestImpl::*)(class testing::internal::UnitTestImpl * const)) 0x55555568ed7e <testing::internal::UnitTestImpl::RunAllTests()>,
location=location@entry=0x5555558ca5f8 "auxiliary test code (environments or event listeners)") at gsrc/googletest/googletest/src/gtest.cc:2689
#24 0x000055555568ac87 in testing::UnitTest::Run (this=0x55555596bd00 <testing::UnitTest::GetInstance()::instance>)
at gsrc/googletest/googletest/src/gtest.cc:5583
#25 0x000055555555cab5 in RUN_ALL_TESTS () at gsrc/googletest/googletest/include/gtest/gtest.h:2334
#26 0x000055555555c9b0 in main (argc=<optimized out>, argv=0x7fffffffd9f8) at gsrc/test/client/main_fscrypt.cc:31
Updated by Christopher Hoffman 5 months ago
- Status changed from New to Resolved
Author: Christopher Hoffman <choffman@redhat.com>
Date: Mon Oct 20 18:15:11 2025 +0000
client: Use nearest fscrypt block when clamping max io size
A max io size can currently be up to INT_MAX. If it is greater,
then clamp the size to INT_MAX. This conflicts with fscrypt io
operations. An fscrypt, op needs to read a whole fscrypt block.
The size of fscrypt block size is 4K, INT_MAX % 4K is not equal
to 0. Therefore, get the nearest multiple of 4K to INT_MAX that
does not go over. In the fscrypt case, this value will be used
for clamping max io size.
Fixes: https://tracker.ceph.com/issues/73346
Signed-off-by: Christopher Hoffman <choffman@redhat.com>
diff --git a/src/client/Client.cc b/src/client/Client.cc
index 048053fdfb6..daa2f8974bc 100644
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -11353,8 +11353,18 @@ int Client::read(int fd, char *buf, loff_t size, loff_t offset)
return -EBADF;
#endif
bufferlist bl;
- /* We can't return bytes written larger than INT_MAX, clamp size to that */
+#if defined(__linux__)
+ /* We can't return bytes written larger than INT_MAX, clamp size to
+ * that or FSCRYPT_MAXIO_SIZE*/
+ Inode *in = f->inode.get();
+ if (in->is_fscrypt_enabled()) {
+ size = std::min(size, (loff_t)FSCRYPT_MAXIO_SIZE);
+ } else {
+ size = std::min(size, (loff_t)INT_MAX);
+ }
+#else
size = std::min(size, (loff_t)INT_MAX);
+#endif
int r = _read(f, offset, size, &bl);
ldout(cct, 3) << "read(" << fd << ", " << (void*)buf << ", " << size << ", " << offset << ") = " << r << dendl;
if (r >= 0) {
@@ -12080,8 +12090,18 @@ int Client::write(int fd, const char *buf, loff_t size, loff_t offset)
if (fh->flags & O_PATH)
return -EBADF;
#endif
- /* We can't return bytes written larger than INT_MAX, clamp size to that */
+#if defined(__linux__)
+ /* We can't return bytes written larger than INT_MAX, clamp size to
+ * that or FSCRYPT_MAXIO_SIZE*/
+ Inode *in = fh->inode.get();
+ if (in->is_fscrypt_enabled()) {
+ size = std::min(size, (loff_t)FSCRYPT_MAXIO_SIZE);
+ } else {
+ size = std::min(size, (loff_t)INT_MAX);
+ }
+#else
size = std::min(size, (loff_t)INT_MAX);
+#endif
bufferlist bl;
bl.append(buf, size);
int r = _write(fh, offset, size, std::move(bl));
@@ -12121,7 +12141,18 @@ int64_t Client::_preadv_pwritev_locked(Fh *fh, const struct iovec *iov,
*/
bufferlist data;
if (clamp_to_int) {
+#if defined(__linux__)
+ /* We can't return bytes written larger than INT_MAX, clamp size to
+ * that or FSCRYPT_MAXIO_SIZE*/
+ Inode *in = fh->inode.get();
+ if (in->is_fscrypt_enabled()) {
+ totallen = std::min(totallen, (size_t)FSCRYPT_MAXIO_SIZE);
+ } else {
+ totallen = std::min(totallen, (size_t)INT_MAX);
+ }
+#else
totallen = std::min(totallen, (size_t)INT_MAX);
+#endif
size_t total_appended = 0;
for (int i = 0; i < iovcnt; i++) {
if (iov[i].iov_len > 0) {
@@ -17064,9 +17095,19 @@ int Client::ll_read(Fh *fh, loff_t off, loff_t len, bufferlist *bl)
return -ENOTCONN;
}
+#if defined(__linux__)
+ /* We can't return bytes written larger than INT_MAX, clamp size to
+ * that or FSCRYPT_MAXIO_SIZE*/
+ Inode *in = fh->inode.get();
+ if (in->is_fscrypt_enabled()) {
+ len = std::min(len, (loff_t)FSCRYPT_MAXIO_SIZE);
+ } else {
/* We can't return bytes written larger than INT_MAX, clamp len to that */
+ len = std::min(len, (loff_t)INT_MAX);
+ }
+#else
len = std::min(len, (loff_t)INT_MAX);
-
+#endif
std::scoped_lock lock(client_lock);
if (fh == NULL || !_ll_fh_exists(fh)) {
ldout(cct, 3) << "(fh)" << fh << " is invalid" << dendl;
@@ -17209,9 +17250,18 @@ int Client::ll_write(Fh *fh, loff_t off, loff_t len, const char *data)
return -ENOTCONN;
}
- /* We can't return bytes written larger than INT_MAX, clamp len to that */
+#if defined(__linux__)
+ /* We can't return bytes written larger than INT_MAX, clamp size to
+ * that or FSCRYPT_MAXIO_SIZE*/
+ Inode *in = fh->inode.get();
+ if (in->is_fscrypt_enabled()) {
+ len = std::min(len, (loff_t)FSCRYPT_MAXIO_SIZE);
+ } else {
+ len = std::min(len, (loff_t)INT_MAX);
+ }
+#else
len = std::min(len, (loff_t)INT_MAX);
-
+#endif
std::scoped_lock lock(client_lock);
if (fh == NULL || !_ll_fh_exists(fh)) {
ldout(cct, 3) << "(fh)" << fh << " is invalid" << dendl;
diff --git a/src/client/FSCrypt.h b/src/client/FSCrypt.h
index 2f8727ee666..b13ec4dbf4f 100644
--- a/src/client/FSCrypt.h
+++ b/src/client/FSCrypt.h
@@ -23,6 +23,10 @@
#define FSCRYPT_DATA_ALIGNMENT 16
+// fscrypt maxio_size cannot be INT_MAX as it is not a multiple of fscrypt
+// block size. Set it to nearest fscrypt block without exceeding INT_MAX
+#define FSCRYPT_MAXIO_SIZE fscrypt_block_start(INT_MAX)
+
static inline uint64_t fscrypt_align_ofs(uint64_t ofs) {
return (ofs + FSCRYPT_DATA_ALIGNMENT - 1) & ~(FSCRYPT_DATA_ALIGNMENT - 1);
}
Actions