RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-03-06 16:28:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #51450 from batrick/i59716

Browse Source 
pybind/rados: keep byte representation if decode fails

Reviewed-by: Samuel Just <sjust@redhat.com>
Reviewed-by: John Mulligan <jmulligan@redhat.com>
Reviewed-by: Dhairya Parmar <dparmar@redhat.com>
This commit is contained in:
Yuri Weinstein 2023-06-21 17:12:08 -04:00 committed by GitHub
commit b1ed114a66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 55 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
PendingReleaseNotes
View File

@ -124,6 +124,9 @@
* RBD: list-watchers C++ API (`Image::list_watchers`) now clears the passed
`std::list` before potentially appending to it, aligning with the semantics
of the corresponding C API (`rbd_watchers_list`).
* The rados python binding is now able to process (opt-in) omap keys as bytes
objects. This enables interacting with RADOS omap keys that are not decodeable as
UTF-8 strings.
* Telemetry: Users who are opted-in to telemetry can also opt-in to
participating in a leaderboard in the telemetry public
dashboards (https://telemetry-public.ceph.com/). Users can now also add a

14
qa/workunits/fs/damage/test-first-damage.sh
View File

@ -6,6 +6,7 @@ FIRST_DAMAGE="first-damage.py"
FS=cephfs
METADATA_POOL=cephfs_meta
MOUNT=~/mnt/mnt.0
PYTHON=python3
function usage {
printf '%s: [--fs=<fs_name>] [--metadata-pool=<pool>] [--first-damage=</path/to/first-damage.py>]\n'
@ -19,6 +20,7 @@ function create {
DIR_INODE=$(stat -c '%i' dir)
touch dir/a
touch dir/"a space"
touch -- $(printf 'dir/\xff')
mkdir dir/.snap/1
mkdir dir/.snap/2
# two snaps
@ -83,9 +85,9 @@ function recover {
sleep 5
cephfs-journal-tool --rank="$FS":0 event recover_dentries summary
cephfs-journal-tool --rank="$FS":0 journal reset
python3 $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL"
python3 $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap "$METADATA_POOL"
python3 $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL"
"$PYTHON" $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL"
"$PYTHON" $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap "$METADATA_POOL"
"$PYTHON" $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL"
ceph fs set "$FS" joinable true
}
@ -123,7 +125,7 @@ function mount {
}
function main {
eval set -- $(getopt --name "$0" --options '' --longoptions 'help,fs:,metadata-pool:,first-damage:,mount:' -- "$@")
eval set -- $(getopt --name "$0" --options '' --longoptions 'help,fs:,metadata-pool:,first-damage:,mount:,python:' -- "$@")
while [ "$#" -gt 0 ]; do
echo "$*"
@ -148,6 +150,10 @@ function main {
FIRST_DAMAGE="$2"
shift 2
;;
--python)
PYTHON="$2"
shift 2
;;
--)
shift
break

39
src/pybind/rados/rados.pyx
View File

@ -76,6 +76,7 @@ MAX_ERRNO = _MAX_ERRNO
ANONYMOUS_AUID = 0xffffffffffffffff
ADMIN_AUID = 0
OMAP_KEY_TYPE = Union[str,bytes]
class Error(Exception):
""" `Error` class, derived from `Exception` """
@ -1369,10 +1370,12 @@ cdef class OmapIterator(object):
"""Omap iterator"""
cdef public Ioctx ioctx
cdef public object omap_key_type
cdef rados_omap_iter_t ctx
def __cinit__(self, Ioctx ioctx):
def __cinit__(self, Ioctx ioctx, omap_key_type):
self.ioctx = ioctx
self.omap_key_type = omap_key_type
def __iter__(self):
return self
@ -1394,7 +1397,7 @@ cdef class OmapIterator(object):
raise make_ex(ret, "error iterating over the omap")
if key_ == NULL:
raise StopIteration()
key = decode_cstr(key_)
key = self.omap_key_type(key_)
val = None
if val_ != NULL:
val = val_[:len_]
@ -1929,7 +1932,7 @@ cdef class WriteOp(object):
with nogil:
rados_write_op_cmpext(self.write_op, _cmp_buf, _cmp_buf_len, _offset, NULL)
def omap_cmp(self, key: str, val: str, cmp_op: int = LIBRADOS_CMPXATTR_OP_EQ):
def omap_cmp(self, key: OMAP_KEY_TYPE, val: OMAP_KEY_TYPE, cmp_op: int = LIBRADOS_CMPXATTR_OP_EQ):
"""
Ensure that an omap key value satisfies comparison
:param key: omap key whose associated value is evaluated for comparison
@ -3605,7 +3608,7 @@ returned %d, but should return zero on success." % (self.name, ret))
"""
read_op.release()
def set_omap(self, write_op: WriteOp, keys: Sequence[str], values: Sequence[bytes]):
def set_omap(self, write_op: WriteOp, keys: Sequence[OMAP_KEY_TYPE], values: Sequence[bytes]):
"""
set keys values to write_op
:para write_op: write_operation object
@ -3752,9 +3755,10 @@ returned %d, but should return zero on success." % (self.name, ret))
def get_omap_vals(self,
read_op: ReadOp,
start_after: str,
filter_prefix: str,
max_return: int) -> Tuple[OmapIterator, int]:
start_after: OMAP_KEY_TYPE,
filter_prefix: OMAP_KEY_TYPE,
max_return: int,
omap_key_type = bytes.decode) -> Tuple[OmapIterator, int]:
"""
get the omap values
:para read_op: read operation object
@ -3776,11 +3780,15 @@ returned %d, but should return zero on success." % (self.name, ret))
with nogil:
rados_read_op_omap_get_vals2(_read_op.read_op, _start_after, _filter_prefix,
_max_return, &iter_addr, NULL, NULL)
it = OmapIterator(self)
it = OmapIterator(self, omap_key_type)
it.ctx = iter_addr
return it, 0 # 0 is meaningless; there for backward-compat
def get_omap_keys(self, read_op: ReadOp, start_after: str, max_return: int) -> Tuple[OmapIterator, int]:
def get_omap_keys(self,
read_op: ReadOp,
start_after: OMAP_KEY_TYPE,
max_return: int,
omap_key_type = bytes.decode) -> Tuple[OmapIterator, int]:
"""
get the omap keys
:para read_op: read operation object
@ -3798,11 +3806,14 @@ returned %d, but should return zero on success." % (self.name, ret))
with nogil:
rados_read_op_omap_get_keys2(_read_op.read_op, _start_after,
_max_return, &iter_addr, NULL, NULL)
it = OmapIterator(self)
it = OmapIterator(self, omap_key_type)
it.ctx = iter_addr
return it, 0 # 0 is meaningless; there for backward-compat
def get_omap_vals_by_keys(self, read_op: ReadOp, keys: Sequence[str]) -> Tuple[OmapIterator, int]:
def get_omap_vals_by_keys(self,
read_op: ReadOp,
keys: Sequence[OMAP_KEY_TYPE],
omap_key_type = bytes.decode) -> Tuple[OmapIterator, int]:
"""
get the omap values by keys
:para read_op: read operation object
@ -3821,13 +3832,13 @@ returned %d, but should return zero on success." % (self.name, ret))
rados_read_op_omap_get_vals_by_keys(_read_op.read_op,
<const char**>_keys,
key_num, &iter_addr, NULL)
it = OmapIterator(self)
it = OmapIterator(self, omap_key_type)
it.ctx = iter_addr
return it, 0 # 0 is meaningless; there for backward-compat
finally:
free(_keys)
def remove_omap_keys(self, write_op: WriteOp, keys: Sequence[str]):
def remove_omap_keys(self, write_op: WriteOp, keys: Sequence[OMAP_KEY_TYPE]):
"""
remove omap keys specifiled
:para write_op: write operation object
@ -3858,7 +3869,7 @@ returned %d, but should return zero on success." % (self.name, ret))
with nogil:
rados_write_op_omap_clear(_write_op.write_op)
def remove_omap_range2(self, write_op: WriteOp, key_begin: str, key_end: str):
def remove_omap_range2(self, write_op: WriteOp, key_begin: OMAP_KEY_TYPE, key_end: OMAP_KEY_TYPE):
"""
Remove key/value pairs from an object whose keys are in the range
[key_begin, key_end)

28
src/test/pybind/test_rados.py
View File

@ -433,30 +433,30 @@ class TestIoctx(object):
self.ioctx.remove_object("inhead")
def test_set_omap(self):
keys = ("1", "2", "3", "4")
values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
keys = ("1", "2", "3", "4", b"\xff")
values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04", b"5")
with WriteOpCtx() as write_op:
self.ioctx.set_omap(write_op, keys, values)
write_op.set_flags(LIBRADOS_OPERATION_SKIPRWLOCKS)
self.ioctx.operate_write_op(write_op, "hw")
with ReadOpCtx() as read_op:
iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4)
iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 5, omap_key_type=bytes)
eq(ret, 0)
self.ioctx.operate_read_op(read_op, "hw")
next(iter)
eq(list(iter), [("2", b"bbb"), ("3", b"ccc"), ("4", b"\x04\x04\x04\x04")])
eq(list(iter), [(b"2", b"bbb"), (b"3", b"ccc"), (b"4", b"\x04\x04\x04\x04"), (b"\xff", b"5")])
with ReadOpCtx() as read_op:
iter, ret = self.ioctx.get_omap_vals(read_op, "2", "", 4)
iter, ret = self.ioctx.get_omap_vals(read_op, b"2", "", 4, omap_key_type=bytes)
eq(ret, 0)
self.ioctx.operate_read_op(read_op, "hw")
eq(("3", b"ccc"), next(iter))
eq(list(iter), [("4", b"\x04\x04\x04\x04")])
eq((b"3", b"ccc"), next(iter))
eq(list(iter), [(b"4", b"\x04\x04\x04\x04"), (b"\xff", b"5")])
with ReadOpCtx() as read_op:
iter, ret = self.ioctx.get_omap_vals(read_op, "", "2", 4)
iter, ret = self.ioctx.get_omap_vals(read_op, "", "2", 4, omap_key_type=bytes)
eq(ret, 0)
read_op.set_flags(LIBRADOS_OPERATION_BALANCE_READS)
self.ioctx.operate_read_op(read_op, "hw")
eq(list(iter), [("2", b"bbb")])
eq(list(iter), [(b"2", b"bbb")])
def test_set_omap_aio(self):
lock = threading.Condition()
@ -534,18 +534,18 @@ class TestIoctx(object):
eq(self.ioctx.read('abc'), b'rzxrzxrzx')
def test_get_omap_vals_by_keys(self):
keys = ("1", "2", "3", "4")
values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04")
keys = ("1", "2", "3", "4", b"\xff")
values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04", b"5")
with WriteOpCtx() as write_op:
self.ioctx.set_omap(write_op, keys, values)
self.ioctx.operate_write_op(write_op, "hw")
with ReadOpCtx() as read_op:
iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",))
iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",b"\xff"), omap_key_type=bytes)
eq(ret, 0)
self.ioctx.operate_read_op(read_op, "hw")
eq(list(iter), [("3", b"ccc"), ("4", b"\x04\x04\x04\x04")])
eq(list(iter), [(b"3", b"ccc"), (b"4", b"\x04\x04\x04\x04"), (b"\xff", b"5")])
with ReadOpCtx() as read_op:
iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",))
iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",), omap_key_type=bytes)
eq(ret, 0)
with assert_raises(ObjectNotFound):
self.ioctx.operate_read_op(read_op, "no_such")

6
src/tools/cephfs/first-damage.py
View File

@ -78,15 +78,15 @@ def traverse(MEMO, ioctx):
with rados.ReadOpCtx() as rctx:
nkey = None
while True:
it = ioctx.get_omap_vals(rctx, nkey, None, 100)[0]
it = ioctx.get_omap_vals(rctx, nkey, None, 100, omap_key_type=bytes)[0]
ioctx.operate_read_op(rctx, o.key)
nkey = None
for (dnk, val) in it:
log.debug('\t%s: val size %d', dnk, len(val))
log.debug(f'\t{dnk}: val size {len(val)}')
(first,) = struct.unpack('<I', val[:4])
if first > NEXT_SNAP:
log.warning(f"found {o.key}:{dnk} first (0x{first:x}) > NEXT_SNAP (0x{NEXT_SNAP:x})")
if REPAIR_NOSNAP and dnk.endswith("_head") and first == CEPH_NOSNAP:
if REPAIR_NOSNAP and dnk.endswith(b"_head") and first == CEPH_NOSNAP:
log.warning(f"repairing first==CEPH_NOSNAP damage, setting to NEXT_SNAP (0x{NEXT_SNAP:x})")
first = NEXT_SNAP
nval = bytearray(val)