qa: use supported releases for featureful_client

... and switch for centos9.

Fixes: https://tracker.ceph.com/issues/65580
Signed-off-by: Venky Shankar <vshankar@redhat.com>

qa/suites/fs/upgrade/featureful_client/old_client/centos_8.stream.yaml

@@ -1 +0,0 @@
-.qa/distros/all/centos_8.stream.yaml

qa/suites/fs/upgrade/featureful_client/old_client/centos_9.stream.yaml

@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml

qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-from/reef.yaml

@@ -1,9 +1,9 @@
 meta:
 - desc: |
-   install ceph/octopus latest
+   install ceph/reef latest
 tasks:
 - install:
-    branch: octopus
+    branch: reef
     exclude_packages:
       - librados3
       - ceph-mgr-dashboard
@@ -13,7 +13,7 @@ tasks:
       - cephadm
       - ceph-volume
     extra_packages: ['librados2']
-- print: "**** done installing octopus"
+- print: "**** done installing reef"
 - ceph:
     log-ignorelist:
       - overall HEALTH_
@@ -35,5 +35,5 @@ tasks:
         ms bind msgr2: false
 - exec:
     osd.0:
-      - ceph osd set-require-min-compat-client octopus
+      - ceph osd set-require-min-compat-client reef
 - print: "**** done ceph"

qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-from/squid.yaml

@@ -1,9 +1,9 @@
 meta:
 - desc: |
-   install ceph/octopus latest
+   install ceph/squid latest
 tasks:
 - install:
-    branch: octopus
+    branch: squid
     exclude_packages:
       - librados3
       - ceph-mgr-dashboard
@@ -13,7 +13,7 @@ tasks:
       - cephadm
       - ceph-volume
     extra_packages: ['librados2']
-- print: "**** done installing octopus"
+- print: "**** done installing squid"
 - ceph:
     log-ignorelist:
       - overall HEALTH_
@@ -35,5 +35,5 @@ tasks:
         ms bind msgr2: false
 - exec:
     osd.0:
-      - ceph osd set-require-min-compat-client octopus
+      - ceph osd set-require-min-compat-client squid
 - print: "**** done ceph"

qa/suites/fs/upgrade/featureful_client/old_client/tasks/1-client.yaml

@@ -1,6 +1,10 @@
 tasks:
 - ceph-fuse:
-- print: "**** done octopus client"
+    client.0:
+      client_feature_range: "[0-13],[15-21]"
+    client.1:
+      client_feature_range: "[0-13],[15-21]"
+- print: "**** done client"
 - workunit:
     clients:
       all:

qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml

@@ -22,7 +22,6 @@ tasks:
 - install.upgrade:
     # upgrade the single cluster node, which is running all the mon/mds/osd/mgr daemons
     mon.a:
-      branch: quincy
 - print: "**** done install.upgrade the host"
 - ceph.restart:
     daemons: [mon.*, mgr.*]
@@ -42,7 +41,7 @@ tasks:
     mon.a:
     - ceph osd dump -f json-pretty
     - ceph versions
-    - ceph osd require-osd-release quincy
+    - ceph osd require-osd-release squid
     - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done
 - ceph.healthy:
 - print: "**** done ceph.restart"

qa/suites/fs/upgrade/featureful_client/upgraded_client/centos_8.stream.yaml

@@ -1 +0,0 @@
-.qa/distros/all/centos_8.stream.yaml

qa/suites/fs/upgrade/featureful_client/upgraded_client/centos_9.stream.yaml

@@ -0,0 +1 @@
+.qa/distros/podman/centos_9.stream.yaml

qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-from/reef.yaml

@@ -0,0 +1,39 @@
+meta:
+- desc: |
+   install ceph/reef latest
+tasks:
+- install:
+    branch: reef
+    exclude_packages:
+      - librados3
+      - ceph-mgr-dashboard
+      - ceph-mgr-diskprediction-local
+      - ceph-mgr-rook
+      - ceph-mgr-cephadm
+      - cephadm
+      - ceph-volume
+    extra_packages: ['librados2']
+- print: "**** done installing reef"
+- ceph:
+    log-ignorelist:
+      - overall HEALTH_
+      - \(FS_
+      - \(MDS_
+      - \(OSD_
+      - \(MON_DOWN\)
+      - \(CACHE_POOL_
+      - \(POOL_
+      - \(MGR_DOWN\)
+      - \(PG_
+      - \(SMALLER_PGP_NUM\)
+      - Monitor daemon marked osd
+      - Behind on trimming
+      - Manager daemon
+    conf:
+      global:
+        mon warn on pool no app: false
+        ms bind msgr2: false
+- exec:
+    osd.0:
+      - ceph osd set-require-min-compat-client reef
+- print: "**** done ceph"

qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-from/squid.yaml

@@ -0,0 +1,39 @@
+meta:
+- desc: |
+   install ceph/squid latest
+tasks:
+- install:
+    branch: squid
+    exclude_packages:
+      - librados3
+      - ceph-mgr-dashboard
+      - ceph-mgr-diskprediction-local
+      - ceph-mgr-rook
+      - ceph-mgr-cephadm
+      - cephadm
+      - ceph-volume
+    extra_packages: ['librados2']
+- print: "**** done installing squid"
+- ceph:
+    log-ignorelist:
+      - overall HEALTH_
+      - \(FS_
+      - \(MDS_
+      - \(OSD_
+      - \(MON_DOWN\)
+      - \(CACHE_POOL_
+      - \(POOL_
+      - \(MGR_DOWN\)
+      - \(PG_
+      - \(SMALLER_PGP_NUM\)
+      - Monitor daemon marked osd
+      - Behind on trimming
+      - Manager daemon
+    conf:
+      global:
+        mon warn on pool no app: false
+        ms bind msgr2: false
+- exec:
+    osd.0:
+      - ceph osd set-require-min-compat-client squid
+- print: "**** done ceph"

qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/1-client.yaml

@@ -3,9 +3,11 @@ overrides:
   nuke-on-error: false
 tasks:
 - ceph-fuse:
-- print: "**** done octopus client"
-#- workunit:
-#    clients:
-#      all:
-#      - suites/fsstress.sh
+    client.1:
+      client_feature_range: "[0-13],[15-21]"
+- print: "**** done client"
+- workunit:
+    clients:
+      all:
+      - suites/fsstress.sh
 - print: "**** done fsstress"

qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml

@@ -22,7 +22,6 @@ tasks:
 - install.upgrade:
     # upgrade the single cluster node, which is running all the mon/mds/osd/mgr daemons
     mon.a:
-      branch: quincy
 - print: "**** done install.upgrade the host"
 - ceph.restart:
     daemons: [mon.*, mgr.*]
@@ -42,7 +41,7 @@ tasks:
     mon.a:
     - ceph versions
     - ceph osd dump -f json-pretty
-    - ceph osd require-osd-release quincy
+    - ceph osd require-osd-release squid
     - for f in `ceph osd pool ls` ; do ceph osd pool set $f pg_autoscale_mode off ; done
 - ceph.healthy:
 - print: "**** done ceph.restart"

qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/3-client-upgrade.yaml

@@ -1,7 +1,6 @@
 tasks:
 - install.upgrade:
     client.0:
-      branch: quincy
 - print: "**** done install.upgrade on client.0"
 - ceph-fuse:
     client.0:

qa/tasks/ceph_fuse.py

@@ -4,6 +4,7 @@ Ceph FUSE client task
 
 import contextlib
 import logging
+import re
 
 from teuthology import misc
 from tasks.cephfs.fuse_mount import FuseMount
@@ -86,6 +87,32 @@ def task(ctx, config):
             client.1:
               mount_subvol_num: 1
 
+    Example for client mount with custom client feature set
+
+        tasks:
+        - ceph:
+        - ceph-fuse:
+            client.0:
+              client_feature_range: 21 # everything including CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK
+
+        OR
+
+        tasks:
+        - ceph:
+        - ceph-fuse:
+            client.0:
+              client_feature_range: "[0-13],[15-21]" # all features except metric_collect (bit 14)
+
+        OR
+
+        tasks:
+        - ceph:
+        - ceph-fuse:
+            client.0:
+              client_feature_range: "[0-13],16,19,[20-21]" # all features except metric_collect,alternate_name, op_getvxattr, 32bit_retry_fwd
+
+        client_feature_range can have repetitive and overlapping ranges/values - the parsed feature bits would not have duplicates and is sorted. Decreasing ranges are silently ignored.
+
     :param ctx: Context
     :param config: Configuration
     """
@@ -161,12 +188,50 @@ def task(ctx, config):
     for remote in remotes:
         FuseMount.cleanup_stale_netnses_and_bridge(remote)
 
+    def parse_client_feature_range(client_feature_range):
+        def intify(val):
+            try:
+                return int(val)
+            except ValueError:
+                log.warn(f'failed to decode feature bit {val}')
+                raise
+        feature_bits = []
+        pvalue = re.compile(r'(\d+)')
+        prange = re.compile(r'\[(\d+)\-(\d+)\]')
+        if (isinstance(client_feature_range, int)):
+            # everything upto (and including) this feature bit
+            feature_bits.extend(range(0, client_feature_range+1))
+        elif isinstance(client_feature_range, str):
+            for feat in client_feature_range.split(','):
+                m = pvalue.match(feat)
+                if m:
+                    feature_bits.append(intify(m.group(1)))
+                    continue
+                m = prange.match(feat)
+                if m:
+                    feature_bits.extend(range(intify(m.group(1)), intify(m.group(2))+1))
+                    continue
+                raise ValueError(f'Invalid feature range or value "{feat}"')
+        else:
+            raise TypeError("client_feature_range must be of type int or str")
+        return sorted(set(feature_bits))
+
     # Mount any clients we have been asked to (default to mount all)
     log.info('Mounting ceph-fuse clients...')
     for info in mounted_by_me.values():
         config = info["config"]
         mount_x = info['mount']
-        mount_x.mount(mntopts=config.get('mntopts', []), mntargs=config.get('mntargs', []))
+
+        # apply custom client feature set
+        client_features = []
+        client_feature_range = config.get("client_feature_range", None)
+        if client_feature_range is not None:
+            client_features = ",".join(str(i) for i in parse_client_feature_range(client_feature_range))
+        mntargs = config.get('mntargs', [])
+        if client_features:
+            mntargs.append(f"--client_debug_inject_features={client_features}")
+        log.debug(f"passing mntargs={mntargs}")
+        mount_x.mount(mntopts=config.get('mntopts', []), mntargs=mntargs)
 
     for info in mounted_by_me.values():
         info["mount"].wait_until_mounted()