diff --git a/qa/suites/fs/thrash/msgr-failures/osd-mds-delay.yaml b/qa/suites/fs/thrash/msgr-failures/osd-mds-delay.yaml index 68802961957..ec583e6d3f3 100644 --- a/qa/suites/fs/thrash/msgr-failures/osd-mds-delay.yaml +++ b/qa/suites/fs/thrash/msgr-failures/osd-mds-delay.yaml @@ -6,5 +6,6 @@ overrides: ms inject delay type: osd mds ms inject delay probability: .005 ms inject delay max: 1 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/fsx/striping/default/msgr-failures/few.yaml b/qa/suites/krbd/fsx/striping/default/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/krbd/fsx/striping/default/msgr-failures/few.yaml +++ b/qa/suites/krbd/fsx/striping/default/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/fsx/striping/default/msgr-failures/many.yaml b/qa/suites/krbd/fsx/striping/default/msgr-failures/many.yaml index 4caedaebdca..31edc91599c 100644 --- a/qa/suites/krbd/fsx/striping/default/msgr-failures/many.yaml +++ b/qa/suites/krbd/fsx/striping/default/msgr-failures/many.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 500 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/fsx/striping/fancy/msgr-failures/few.yaml b/qa/suites/krbd/fsx/striping/fancy/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/krbd/fsx/striping/fancy/msgr-failures/few.yaml +++ b/qa/suites/krbd/fsx/striping/fancy/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/rbd-nomount/msgr-failures/few.yaml b/qa/suites/krbd/rbd-nomount/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/krbd/rbd-nomount/msgr-failures/few.yaml +++ b/qa/suites/krbd/rbd-nomount/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/rbd-nomount/msgr-failures/many.yaml b/qa/suites/krbd/rbd-nomount/msgr-failures/many.yaml index 4caedaebdca..31edc91599c 100644 --- a/qa/suites/krbd/rbd-nomount/msgr-failures/many.yaml +++ b/qa/suites/krbd/rbd-nomount/msgr-failures/many.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 500 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/rbd/msgr-failures/few.yaml b/qa/suites/krbd/rbd/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/krbd/rbd/msgr-failures/few.yaml +++ b/qa/suites/krbd/rbd/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/rbd/msgr-failures/many.yaml b/qa/suites/krbd/rbd/msgr-failures/many.yaml index 4caedaebdca..31edc91599c 100644 --- a/qa/suites/krbd/rbd/msgr-failures/many.yaml +++ b/qa/suites/krbd/rbd/msgr-failures/many.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 500 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/singleton/msgr-failures/few.yaml b/qa/suites/krbd/singleton/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/krbd/singleton/msgr-failures/few.yaml +++ b/qa/suites/krbd/singleton/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/singleton/msgr-failures/many.yaml b/qa/suites/krbd/singleton/msgr-failures/many.yaml index 4caedaebdca..31edc91599c 100644 --- a/qa/suites/krbd/singleton/msgr-failures/many.yaml +++ b/qa/suites/krbd/singleton/msgr-failures/many.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 500 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/krbd/wac/wac/verify/many-resets.yaml b/qa/suites/krbd/wac/wac/verify/many-resets.yaml index d69f65031de..fab0abd658e 100644 --- a/qa/suites/krbd/wac/wac/verify/many-resets.yaml +++ b/qa/suites/krbd/wac/wac/verify/many-resets.yaml @@ -3,6 +3,7 @@ overrides: conf: global: ms inject socket failures: 500 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME tasks: diff --git a/qa/suites/rados/basic/msgr-failures/few.yaml b/qa/suites/rados/basic/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rados/basic/msgr-failures/few.yaml +++ b/qa/suites/rados/basic/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/basic/msgr-failures/many.yaml b/qa/suites/rados/basic/msgr-failures/many.yaml index f4bb065b5f3..dcf7a471f6d 100644 --- a/qa/suites/rados/basic/msgr-failures/many.yaml +++ b/qa/suites/rados/basic/msgr-failures/many.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 1500 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/monthrash/msgr-failures/few.yaml b/qa/suites/rados/monthrash/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rados/monthrash/msgr-failures/few.yaml +++ b/qa/suites/rados/monthrash/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/monthrash/msgr-failures/mon-delay.yaml b/qa/suites/rados/monthrash/msgr-failures/mon-delay.yaml index fcd8ca7c293..c4bc304fc73 100644 --- a/qa/suites/rados/monthrash/msgr-failures/mon-delay.yaml +++ b/qa/suites/rados/monthrash/msgr-failures/mon-delay.yaml @@ -7,6 +7,7 @@ overrides: ms inject delay probability: .005 ms inject delay max: 1 ms inject internal delays: .002 + mon client directed command retry: 5 mgr: debug monc: 10 log-whitelist: diff --git a/qa/suites/rados/multimon/msgr-failures/few.yaml b/qa/suites/rados/multimon/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rados/multimon/msgr-failures/few.yaml +++ b/qa/suites/rados/multimon/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/multimon/msgr-failures/many.yaml b/qa/suites/rados/multimon/msgr-failures/many.yaml index ffeb5f6857c..b7104d3137c 100644 --- a/qa/suites/rados/multimon/msgr-failures/many.yaml +++ b/qa/suites/rados/multimon/msgr-failures/many.yaml @@ -3,6 +3,7 @@ overrides: conf: global: ms inject socket failures: 1000 + mon client directed command retry: 5 mon mgr beacon grace: 90 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/singleton/msgr-failures/few.yaml b/qa/suites/rados/singleton/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rados/singleton/msgr-failures/few.yaml +++ b/qa/suites/rados/singleton/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/singleton/msgr-failures/many.yaml b/qa/suites/rados/singleton/msgr-failures/many.yaml index 20aeb4df28b..7185596c2e0 100644 --- a/qa/suites/rados/singleton/msgr-failures/many.yaml +++ b/qa/suites/rados/singleton/msgr-failures/many.yaml @@ -5,6 +5,7 @@ overrides: ms inject socket failures: 1000 mon mgr beacon grace: 90 mon client hunt interval max multiple: 2 + mon client directed command retry: 5 mgr: debug monc: 10 log-whitelist: diff --git a/qa/suites/rados/thrash-old-clients/msgr-failures/fastclose.yaml b/qa/suites/rados/thrash-old-clients/msgr-failures/fastclose.yaml index 02121726ef4..5e0b180c61e 100644 --- a/qa/suites/rados/thrash-old-clients/msgr-failures/fastclose.yaml +++ b/qa/suites/rados/thrash-old-clients/msgr-failures/fastclose.yaml @@ -4,5 +4,6 @@ overrides: global: ms inject socket failures: 2500 ms tcp read timeout: 5 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/thrash-old-clients/msgr-failures/few.yaml b/qa/suites/rados/thrash-old-clients/msgr-failures/few.yaml index 527eadb450b..0bd752f215f 100644 --- a/qa/suites/rados/thrash-old-clients/msgr-failures/few.yaml +++ b/qa/suites/rados/thrash-old-clients/msgr-failures/few.yaml @@ -3,6 +3,7 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 osd: osd heartbeat use min delay socket: true log-whitelist: diff --git a/qa/suites/rados/thrash-old-clients/msgr-failures/osd-delay.yaml b/qa/suites/rados/thrash-old-clients/msgr-failures/osd-delay.yaml index 91c14725635..b3f2f37f487 100644 --- a/qa/suites/rados/thrash-old-clients/msgr-failures/osd-delay.yaml +++ b/qa/suites/rados/thrash-old-clients/msgr-failures/osd-delay.yaml @@ -7,5 +7,6 @@ overrides: ms inject delay probability: .005 ms inject delay max: 1 ms inject internal delays: .002 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/thrash/msgr-failures/fastclose.yaml b/qa/suites/rados/thrash/msgr-failures/fastclose.yaml index 02121726ef4..5e0b180c61e 100644 --- a/qa/suites/rados/thrash/msgr-failures/fastclose.yaml +++ b/qa/suites/rados/thrash/msgr-failures/fastclose.yaml @@ -4,5 +4,6 @@ overrides: global: ms inject socket failures: 2500 ms tcp read timeout: 5 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/thrash/msgr-failures/few.yaml b/qa/suites/rados/thrash/msgr-failures/few.yaml index 527eadb450b..0bd752f215f 100644 --- a/qa/suites/rados/thrash/msgr-failures/few.yaml +++ b/qa/suites/rados/thrash/msgr-failures/few.yaml @@ -3,6 +3,7 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 osd: osd heartbeat use min delay socket: true log-whitelist: diff --git a/qa/suites/rados/thrash/msgr-failures/osd-delay.yaml b/qa/suites/rados/thrash/msgr-failures/osd-delay.yaml index 91c14725635..b3f2f37f487 100644 --- a/qa/suites/rados/thrash/msgr-failures/osd-delay.yaml +++ b/qa/suites/rados/thrash/msgr-failures/osd-delay.yaml @@ -7,5 +7,6 @@ overrides: ms inject delay probability: .005 ms inject delay max: 1 ms inject internal delays: .002 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rados/verify/msgr-failures/few.yaml b/qa/suites/rados/verify/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rados/verify/msgr-failures/few.yaml +++ b/qa/suites/rados/verify/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rbd/basic/msgr-failures/few.yaml b/qa/suites/rbd/basic/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rbd/basic/msgr-failures/few.yaml +++ b/qa/suites/rbd/basic/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rbd/cli/msgr-failures/few.yaml b/qa/suites/rbd/cli/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rbd/cli/msgr-failures/few.yaml +++ b/qa/suites/rbd/cli/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rbd/cli_v1/msgr-failures/few.yaml b/qa/suites/rbd/cli_v1/msgr-failures/few.yaml index 4326fe23eb9..55b148cd1c7 100644 --- a/qa/suites/rbd/cli_v1/msgr-failures/few.yaml +++ b/qa/suites/rbd/cli_v1/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rbd/librbd/msgr-failures/few.yaml b/qa/suites/rbd/librbd/msgr-failures/few.yaml index 55b6df536ec..c1b4047d1ea 100644 --- a/qa/suites/rbd/librbd/msgr-failures/few.yaml +++ b/qa/suites/rbd/librbd/msgr-failures/few.yaml @@ -3,5 +3,7 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - - but it is still running + - but it is still running + - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rbd/qemu/msgr-failures/few.yaml b/qa/suites/rbd/qemu/msgr-failures/few.yaml index 9349b4f9a30..2891ec15c10 100644 --- a/qa/suites/rbd/qemu/msgr-failures/few.yaml +++ b/qa/suites/rbd/qemu/msgr-failures/few.yaml @@ -3,6 +3,7 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - but it is still running - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rbd/thrash/msgr-failures/few.yaml b/qa/suites/rbd/thrash/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rbd/thrash/msgr-failures/few.yaml +++ b/qa/suites/rbd/thrash/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/rgw/verify/msgr-failures/few.yaml b/qa/suites/rgw/verify/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/rgw/verify/msgr-failures/few.yaml +++ b/qa/suites/rgw/verify/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/smoke/basic/tasks/mon_thrash.yaml b/qa/suites/smoke/basic/tasks/mon_thrash.yaml index 2144afd2fe4..18670457ae6 100644 --- a/qa/suites/smoke/basic/tasks/mon_thrash.yaml +++ b/qa/suites/smoke/basic/tasks/mon_thrash.yaml @@ -21,6 +21,7 @@ overrides: ms inject delay type: mon ms inject internal delays: 0.002 ms inject socket failures: 2500 + mon client directed command retry: 5 tasks: - install: null - ceph: diff --git a/qa/suites/smoke/basic/tasks/rados_bench.yaml b/qa/suites/smoke/basic/tasks/rados_bench.yaml index 493858d37d5..2ce8982d588 100644 --- a/qa/suites/smoke/basic/tasks/rados_bench.yaml +++ b/qa/suites/smoke/basic/tasks/rados_bench.yaml @@ -7,6 +7,7 @@ overrides: ms inject delay type: osd ms inject internal delays: 0.002 ms inject socket failures: 2500 + mon client directed command retry: 5 tasks: - install: null - ceph: diff --git a/qa/suites/smoke/basic/tasks/rbd_fsx.yaml b/qa/suites/smoke/basic/tasks/rbd_fsx.yaml index 3297ef5b8aa..3af52c85260 100644 --- a/qa/suites/smoke/basic/tasks/rbd_fsx.yaml +++ b/qa/suites/smoke/basic/tasks/rbd_fsx.yaml @@ -17,6 +17,7 @@ overrides: rbd cache: true global: ms inject socket failures: 5000 + mon client directed command retry: 5 tasks: - install: null - ceph: diff --git a/qa/suites/tgt/basic/msgr-failures/few.yaml b/qa/suites/tgt/basic/msgr-failures/few.yaml index 4326fe23eb9..dea65948b72 100644 --- a/qa/suites/tgt/basic/msgr-failures/few.yaml +++ b/qa/suites/tgt/basic/msgr-failures/few.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 5000 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME diff --git a/qa/suites/tgt/basic/msgr-failures/many.yaml b/qa/suites/tgt/basic/msgr-failures/many.yaml index 4caedaebdca..31edc91599c 100644 --- a/qa/suites/tgt/basic/msgr-failures/many.yaml +++ b/qa/suites/tgt/basic/msgr-failures/many.yaml @@ -3,5 +3,6 @@ overrides: conf: global: ms inject socket failures: 500 + mon client directed command retry: 5 log-whitelist: - \(OSD_SLOW_PING_TIME