"""
Special regression test for tracker #11184
Synopsis: osd/SnapMapper.cc: 282: FAILED assert(check(oid))
This is accomplished by moving a pg that wasn't part of split and still include
divergent priors.
"""
import logging
import time
from cStringIO import StringIO
from teuthology import misc as teuthology
from util.rados import rados
import os
log = logging.getLogger(__name__)
def task(ctx, config):
"""
Test handling of divergent entries during export / import
to regression test tracker #11184
overrides:
ceph:
conf:
osd:
debug osd: 5
Requires 3 osds on a single test node.
"""
if config is None:
config = {}
assert isinstance(config, dict), \
'divergent_priors task only accepts a dict for configuration'
while len(ctx.manager.get_osd_status()['up']) < 3:
time.sleep(10)
ctx.manager.raw_cluster_cmd('tell', 'osd.0', 'flush_pg_stats')
ctx.manager.raw_cluster_cmd('tell', 'osd.1', 'flush_pg_stats')
ctx.manager.raw_cluster_cmd('tell', 'osd.2', 'flush_pg_stats')
ctx.manager.raw_cluster_cmd('osd', 'set', 'noout')
ctx.manager.raw_cluster_cmd('osd', 'set', 'noin')
ctx.manager.raw_cluster_cmd('osd', 'set', 'nodown')
ctx.manager.wait_for_clean()
# something that is always there
dummyfile = '/etc/fstab'
dummyfile2 = '/etc/resolv.conf'
testdir = teuthology.get_testdir(ctx)
# create 1 pg pool
log.info('creating foo')
ctx.manager.raw_cluster_cmd('osd', 'pool', 'create', 'foo', '1')
osds = [0, 1, 2]
for i in osds:
ctx.manager.set_config(i, osd_min_pg_log_entries=10)
ctx.manager.set_config(i, osd_max_pg_log_entries=10)
ctx.manager.set_config(i, osd_pg_log_trim_min=5)
# determine primary
divergent = ctx.manager.get_pg_primary('foo', 0)
log.info("primary and soon to be divergent is %d", divergent)
non_divergent = list(osds)
non_divergent.remove(divergent)
log.info('writing initial objects')
first_mon = teuthology.get_first_mon(ctx, config)
(mon,) = ctx.cluster.only(first_mon).remotes.iterkeys()
# write 100 objects
for i in range(100):
rados(ctx, mon, ['-p', 'foo', 'put', 'existing_%d' % i, dummyfile])
ctx.manager.wait_for_clean()
# blackhole non_divergent
log.info("blackholing osds %s", str(non_divergent))
for i in non_divergent:
ctx.manager.set_config(i, filestore_blackhole=1)
DIVERGENT_WRITE = 5
DIVERGENT_REMOVE = 5
# Write some soon to be divergent
log.info('writing divergent objects')
for i in range(DIVERGENT_WRITE):
rados(ctx, mon, ['-p', 'foo', 'put', 'existing_%d' % i,
dummyfile2], wait=False)
# Remove some soon to be divergent
log.info('remove divergent objects')
for i in range(DIVERGENT_REMOVE):
rados(ctx, mon, ['-p', 'foo', 'rm',
'existing_%d' % (i + DIVERGENT_WRITE)], wait=False)
time.sleep(10)
mon.run(
args=['killall', '-9', 'rados'],
wait=True,
check_status=False)
# kill all the osds but leave divergent in
log.info('killing all the osds')
for i in osds:
ctx.manager.kill_osd(i)
for i in osds:
ctx.manager.mark_down_osd(i)
for i in non_divergent:
ctx.manager.mark_out_osd(i)
# bring up non-divergent
log.info("bringing up non_divergent %s", str(non_divergent))
for i in non_divergent:
ctx.manager.revive_osd(i)
for i in non_divergent:
ctx.manager.mark_in_osd(i)
# write 1 non-divergent object (ensure that old divergent one is divergent)
objname = "existing_%d" % (DIVERGENT_WRITE + DIVERGENT_REMOVE)
log.info('writing non-divergent object ' + objname)
rados(ctx, mon, ['-p', 'foo', 'put', objname, dummyfile2])
ctx.manager.wait_for_recovery()
# ensure no recovery of up osds first
log.info('delay recovery')
for i in non_divergent:
ctx.manager.wait_run_admin_socket(
'osd', i, ['set_recovery_delay', '100000'])
# bring in our divergent friend
log.info("revive divergent %d", divergent)
ctx.manager.raw_cluster_cmd('osd', 'set', 'noup')
ctx.manager.revive_osd(divergent)
log.info('delay recovery divergent')
ctx.manager.wait_run_admin_socket(
'osd', divergent, ['set_recovery_delay', '100000'])
ctx.manager.raw_cluster_cmd('osd', 'unset', 'noup')
while len(ctx.manager.get_osd_status()['up']) < 3:
time.sleep(10)
log.info('wait for peering')
rados(ctx, mon, ['-p', 'foo', 'put', 'foo', dummyfile])
# At this point the divergent_priors should have been detected
log.info("killing divergent %d", divergent)
ctx.manager.kill_osd(divergent)
# Split pgs for pool foo
ctx.manager.raw_cluster_cmd('osd', 'pool', 'set', 'foo', 'pg_num', '2')
time.sleep(5)
# Export a pg
(exp_remote,) = ctx.\
cluster.only('osd.{o}'.format(o=divergent)).remotes.iterkeys()
FSPATH = ctx.manager.get_filepath()
JPATH = os.path.join(FSPATH, "journal")
prefix = ("sudo adjust-ulimits ceph-objectstore-tool "
"--data-path {fpath} --journal-path {jpath} "
"--log-file="
"/var/log/ceph/objectstore_tool.$$.log ".
format(fpath=FSPATH, jpath=JPATH))
pid = os.getpid()
expfile = os.path.join(testdir, "exp.{pid}.out".format(pid=pid))
cmd = ((prefix + "--op export --pgid 1.0 --file {file}").
format(id=divergent, file=expfile))
proc = exp_remote.run(args=cmd, wait=True,
check_status=False, stdout=StringIO())
assert proc.exitstatus == 0
# Remove the same pg that was exported
cmd = ((prefix + "--op remove --pgid 1.0").
format(id=divergent, file=expfile))
proc = exp_remote.run(args=cmd, wait=True,
check_status=False, stdout=StringIO())
assert proc.exitstatus == 0
# Kill one of non-divergent OSDs
log.info('killing osd.%d' % non_divergent[1])
ctx.manager.kill_osd(non_divergent[1])
ctx.manager.mark_down_osd(non_divergent[1])
# ctx.manager.mark_out_osd(non_divergent[1])
cmd = ((prefix + "--op import --file {file}").
format(id=non_divergent[1], file=expfile))
proc = exp_remote.run(args=cmd, wait=True,
check_status=False, stdout=StringIO())
assert proc.exitstatus == 0
# bring in our divergent friend and other node
log.info("revive divergent %d", divergent)
ctx.manager.revive_osd(divergent)
ctx.manager.mark_in_osd(divergent)
log.info("revive %d", non_divergent[1])
ctx.manager.revive_osd(non_divergent[1])
while len(ctx.manager.get_osd_status()['up']) < 3:
time.sleep(10)
log.info('delay recovery divergent')
ctx.manager.set_config(divergent, osd_recovery_delay_start=100000)
log.info('mark divergent in')
ctx.manager.mark_in_osd(divergent)
log.info('wait for peering')
rados(ctx, mon, ['-p', 'foo', 'put', 'foo', dummyfile])
log.info("killing divergent %d", divergent)
ctx.manager.kill_osd(divergent)
log.info("reviving divergent %d", divergent)
ctx.manager.revive_osd(divergent)
time.sleep(3)
log.info('allowing recovery')
# Set osd_recovery_delay_start back to 0 and kick the queue
for i in osds:
ctx.manager.raw_cluster_cmd('tell', 'osd.%d' % i, 'debug',
'kick_recovery_wq', ' 0')
log.info('reading divergent objects')
for i in range(DIVERGENT_WRITE + DIVERGENT_REMOVE):
exit_status = rados(ctx, mon, ['-p', 'foo', 'get', 'existing_%d' % i,
'-o', '/tmp/existing'])
assert exit_status is 0
(remote,) = ctx.\
cluster.only('osd.{o}'.format(o=divergent)).remotes.iterkeys()
msg = "dirty_divergent_priors: true, divergent_priors: %d" \
% (DIVERGENT_WRITE + DIVERGENT_REMOVE)
cmd = 'grep "{msg}" /var/log/ceph/ceph-osd.{osd}.log'\
.format(msg=msg, osd=divergent)
proc = remote.run(args=cmd, wait=True, check_status=False)
assert proc.exitstatus == 0
cmd = 'rm {file}'.format(file=expfile)
remote.run(args=cmd, wait=True)
log.info("success")