diff --git a/PendingReleaseNotes b/PendingReleaseNotes index e9df0a071e6..dabb3f1b6c2 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -124,6 +124,9 @@ * RBD: list-watchers C++ API (`Image::list_watchers`) now clears the passed `std::list` before potentially appending to it, aligning with the semantics of the corresponding C API (`rbd_watchers_list`). +* The rados python binding is now able to process (opt-in) omap keys as bytes + objects. This enables interacting with RADOS omap keys that are not decodeable as + UTF-8 strings. * Telemetry: Users who are opted-in to telemetry can also opt-in to participating in a leaderboard in the telemetry public dashboards (https://telemetry-public.ceph.com/). Users can now also add a diff --git a/qa/workunits/fs/damage/test-first-damage.sh b/qa/workunits/fs/damage/test-first-damage.sh index 5cbae5f37fa..57447b957d7 100755 --- a/qa/workunits/fs/damage/test-first-damage.sh +++ b/qa/workunits/fs/damage/test-first-damage.sh @@ -6,6 +6,7 @@ FIRST_DAMAGE="first-damage.py" FS=cephfs METADATA_POOL=cephfs_meta MOUNT=~/mnt/mnt.0 +PYTHON=python3 function usage { printf '%s: [--fs=] [--metadata-pool=] [--first-damage=]\n' @@ -19,6 +20,7 @@ function create { DIR_INODE=$(stat -c '%i' dir) touch dir/a touch dir/"a space" + touch -- $(printf 'dir/\xff') mkdir dir/.snap/1 mkdir dir/.snap/2 # two snaps @@ -83,9 +85,9 @@ function recover { sleep 5 cephfs-journal-tool --rank="$FS":0 event recover_dentries summary cephfs-journal-tool --rank="$FS":0 journal reset - python3 $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL" - python3 $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap "$METADATA_POOL" - python3 $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL" + "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug1 --memo /tmp/memo1 "$METADATA_POOL" + "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug2 --memo /tmp/memo2 --repair-nosnap "$METADATA_POOL" + "$PYTHON" $FIRST_DAMAGE --debug /tmp/debug3 --memo /tmp/memo3 --remove "$METADATA_POOL" ceph fs set "$FS" joinable true } @@ -123,7 +125,7 @@ function mount { } function main { - eval set -- $(getopt --name "$0" --options '' --longoptions 'help,fs:,metadata-pool:,first-damage:,mount:' -- "$@") + eval set -- $(getopt --name "$0" --options '' --longoptions 'help,fs:,metadata-pool:,first-damage:,mount:,python:' -- "$@") while [ "$#" -gt 0 ]; do echo "$*" @@ -148,6 +150,10 @@ function main { FIRST_DAMAGE="$2" shift 2 ;; + --python) + PYTHON="$2" + shift 2 + ;; --) shift break diff --git a/src/pybind/rados/rados.pyx b/src/pybind/rados/rados.pyx index d7b68b4ac90..15198f6e642 100644 --- a/src/pybind/rados/rados.pyx +++ b/src/pybind/rados/rados.pyx @@ -76,6 +76,7 @@ MAX_ERRNO = _MAX_ERRNO ANONYMOUS_AUID = 0xffffffffffffffff ADMIN_AUID = 0 +OMAP_KEY_TYPE = Union[str,bytes] class Error(Exception): """ `Error` class, derived from `Exception` """ @@ -1369,10 +1370,12 @@ cdef class OmapIterator(object): """Omap iterator""" cdef public Ioctx ioctx + cdef public object omap_key_type cdef rados_omap_iter_t ctx - def __cinit__(self, Ioctx ioctx): + def __cinit__(self, Ioctx ioctx, omap_key_type): self.ioctx = ioctx + self.omap_key_type = omap_key_type def __iter__(self): return self @@ -1394,7 +1397,7 @@ cdef class OmapIterator(object): raise make_ex(ret, "error iterating over the omap") if key_ == NULL: raise StopIteration() - key = decode_cstr(key_) + key = self.omap_key_type(key_) val = None if val_ != NULL: val = val_[:len_] @@ -1929,7 +1932,7 @@ cdef class WriteOp(object): with nogil: rados_write_op_cmpext(self.write_op, _cmp_buf, _cmp_buf_len, _offset, NULL) - def omap_cmp(self, key: str, val: str, cmp_op: int = LIBRADOS_CMPXATTR_OP_EQ): + def omap_cmp(self, key: OMAP_KEY_TYPE, val: OMAP_KEY_TYPE, cmp_op: int = LIBRADOS_CMPXATTR_OP_EQ): """ Ensure that an omap key value satisfies comparison :param key: omap key whose associated value is evaluated for comparison @@ -3605,7 +3608,7 @@ returned %d, but should return zero on success." % (self.name, ret)) """ read_op.release() - def set_omap(self, write_op: WriteOp, keys: Sequence[str], values: Sequence[bytes]): + def set_omap(self, write_op: WriteOp, keys: Sequence[OMAP_KEY_TYPE], values: Sequence[bytes]): """ set keys values to write_op :para write_op: write_operation object @@ -3752,9 +3755,10 @@ returned %d, but should return zero on success." % (self.name, ret)) def get_omap_vals(self, read_op: ReadOp, - start_after: str, - filter_prefix: str, - max_return: int) -> Tuple[OmapIterator, int]: + start_after: OMAP_KEY_TYPE, + filter_prefix: OMAP_KEY_TYPE, + max_return: int, + omap_key_type = bytes.decode) -> Tuple[OmapIterator, int]: """ get the omap values :para read_op: read operation object @@ -3776,11 +3780,15 @@ returned %d, but should return zero on success." % (self.name, ret)) with nogil: rados_read_op_omap_get_vals2(_read_op.read_op, _start_after, _filter_prefix, _max_return, &iter_addr, NULL, NULL) - it = OmapIterator(self) + it = OmapIterator(self, omap_key_type) it.ctx = iter_addr return it, 0 # 0 is meaningless; there for backward-compat - def get_omap_keys(self, read_op: ReadOp, start_after: str, max_return: int) -> Tuple[OmapIterator, int]: + def get_omap_keys(self, + read_op: ReadOp, + start_after: OMAP_KEY_TYPE, + max_return: int, + omap_key_type = bytes.decode) -> Tuple[OmapIterator, int]: """ get the omap keys :para read_op: read operation object @@ -3798,11 +3806,14 @@ returned %d, but should return zero on success." % (self.name, ret)) with nogil: rados_read_op_omap_get_keys2(_read_op.read_op, _start_after, _max_return, &iter_addr, NULL, NULL) - it = OmapIterator(self) + it = OmapIterator(self, omap_key_type) it.ctx = iter_addr return it, 0 # 0 is meaningless; there for backward-compat - def get_omap_vals_by_keys(self, read_op: ReadOp, keys: Sequence[str]) -> Tuple[OmapIterator, int]: + def get_omap_vals_by_keys(self, + read_op: ReadOp, + keys: Sequence[OMAP_KEY_TYPE], + omap_key_type = bytes.decode) -> Tuple[OmapIterator, int]: """ get the omap values by keys :para read_op: read operation object @@ -3821,13 +3832,13 @@ returned %d, but should return zero on success." % (self.name, ret)) rados_read_op_omap_get_vals_by_keys(_read_op.read_op, _keys, key_num, &iter_addr, NULL) - it = OmapIterator(self) + it = OmapIterator(self, omap_key_type) it.ctx = iter_addr return it, 0 # 0 is meaningless; there for backward-compat finally: free(_keys) - def remove_omap_keys(self, write_op: WriteOp, keys: Sequence[str]): + def remove_omap_keys(self, write_op: WriteOp, keys: Sequence[OMAP_KEY_TYPE]): """ remove omap keys specifiled :para write_op: write operation object @@ -3858,7 +3869,7 @@ returned %d, but should return zero on success." % (self.name, ret)) with nogil: rados_write_op_omap_clear(_write_op.write_op) - def remove_omap_range2(self, write_op: WriteOp, key_begin: str, key_end: str): + def remove_omap_range2(self, write_op: WriteOp, key_begin: OMAP_KEY_TYPE, key_end: OMAP_KEY_TYPE): """ Remove key/value pairs from an object whose keys are in the range [key_begin, key_end) diff --git a/src/test/pybind/test_rados.py b/src/test/pybind/test_rados.py index 6334d6ebd78..fe614aaca79 100644 --- a/src/test/pybind/test_rados.py +++ b/src/test/pybind/test_rados.py @@ -433,30 +433,30 @@ class TestIoctx(object): self.ioctx.remove_object("inhead") def test_set_omap(self): - keys = ("1", "2", "3", "4") - values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04") + keys = ("1", "2", "3", "4", b"\xff") + values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04", b"5") with WriteOpCtx() as write_op: self.ioctx.set_omap(write_op, keys, values) write_op.set_flags(LIBRADOS_OPERATION_SKIPRWLOCKS) self.ioctx.operate_write_op(write_op, "hw") with ReadOpCtx() as read_op: - iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 4) + iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 5, omap_key_type=bytes) eq(ret, 0) self.ioctx.operate_read_op(read_op, "hw") next(iter) - eq(list(iter), [("2", b"bbb"), ("3", b"ccc"), ("4", b"\x04\x04\x04\x04")]) + eq(list(iter), [(b"2", b"bbb"), (b"3", b"ccc"), (b"4", b"\x04\x04\x04\x04"), (b"\xff", b"5")]) with ReadOpCtx() as read_op: - iter, ret = self.ioctx.get_omap_vals(read_op, "2", "", 4) + iter, ret = self.ioctx.get_omap_vals(read_op, b"2", "", 4, omap_key_type=bytes) eq(ret, 0) self.ioctx.operate_read_op(read_op, "hw") - eq(("3", b"ccc"), next(iter)) - eq(list(iter), [("4", b"\x04\x04\x04\x04")]) + eq((b"3", b"ccc"), next(iter)) + eq(list(iter), [(b"4", b"\x04\x04\x04\x04"), (b"\xff", b"5")]) with ReadOpCtx() as read_op: - iter, ret = self.ioctx.get_omap_vals(read_op, "", "2", 4) + iter, ret = self.ioctx.get_omap_vals(read_op, "", "2", 4, omap_key_type=bytes) eq(ret, 0) read_op.set_flags(LIBRADOS_OPERATION_BALANCE_READS) self.ioctx.operate_read_op(read_op, "hw") - eq(list(iter), [("2", b"bbb")]) + eq(list(iter), [(b"2", b"bbb")]) def test_set_omap_aio(self): lock = threading.Condition() @@ -534,18 +534,18 @@ class TestIoctx(object): eq(self.ioctx.read('abc'), b'rzxrzxrzx') def test_get_omap_vals_by_keys(self): - keys = ("1", "2", "3", "4") - values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04") + keys = ("1", "2", "3", "4", b"\xff") + values = (b"aaa", b"bbb", b"ccc", b"\x04\x04\x04\x04", b"5") with WriteOpCtx() as write_op: self.ioctx.set_omap(write_op, keys, values) self.ioctx.operate_write_op(write_op, "hw") with ReadOpCtx() as read_op: - iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",)) + iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",b"\xff"), omap_key_type=bytes) eq(ret, 0) self.ioctx.operate_read_op(read_op, "hw") - eq(list(iter), [("3", b"ccc"), ("4", b"\x04\x04\x04\x04")]) + eq(list(iter), [(b"3", b"ccc"), (b"4", b"\x04\x04\x04\x04"), (b"\xff", b"5")]) with ReadOpCtx() as read_op: - iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",)) + iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3","4",), omap_key_type=bytes) eq(ret, 0) with assert_raises(ObjectNotFound): self.ioctx.operate_read_op(read_op, "no_such") diff --git a/src/tools/cephfs/first-damage.py b/src/tools/cephfs/first-damage.py index b7660f590f0..0479dc8cb2a 100644 --- a/src/tools/cephfs/first-damage.py +++ b/src/tools/cephfs/first-damage.py @@ -78,15 +78,15 @@ def traverse(MEMO, ioctx): with rados.ReadOpCtx() as rctx: nkey = None while True: - it = ioctx.get_omap_vals(rctx, nkey, None, 100)[0] + it = ioctx.get_omap_vals(rctx, nkey, None, 100, omap_key_type=bytes)[0] ioctx.operate_read_op(rctx, o.key) nkey = None for (dnk, val) in it: - log.debug('\t%s: val size %d', dnk, len(val)) + log.debug(f'\t{dnk}: val size {len(val)}') (first,) = struct.unpack(' NEXT_SNAP: log.warning(f"found {o.key}:{dnk} first (0x{first:x}) > NEXT_SNAP (0x{NEXT_SNAP:x})") - if REPAIR_NOSNAP and dnk.endswith("_head") and first == CEPH_NOSNAP: + if REPAIR_NOSNAP and dnk.endswith(b"_head") and first == CEPH_NOSNAP: log.warning(f"repairing first==CEPH_NOSNAP damage, setting to NEXT_SNAP (0x{NEXT_SNAP:x})") first = NEXT_SNAP nval = bytearray(val)