RepoMirrors/ceph
1
0
Fork 0
mirror of https://github.com/ceph/ceph synced 2025-02-06 10:23:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #23376 from dzafman/wip-25108

Browse Source 
object errors found in be_select_auth_object() aren't logged the same

Reviewed-by: Kefu Chai <kchai@redhat.com>
This commit is contained in:
David Zafman 2018-08-23 13:23:55 -07:00 committed by GitHub
commit b0d2c64d6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 299 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
qa/standalone/scrub/osd-scrub-repair.sh
View File

@ -512,6 +512,7 @@ function TEST_corrupt_scrub_replicated() {
done
local pg=$(get_pg $poolname ROBJ0)
local primary=$(get_primary $poolname ROBJ0)
# Compute an old omap digest and save oi
CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \
@ -642,6 +643,39 @@ function TEST_corrupt_scrub_replicated() {
pg_scrub $pg
ERRORS=0
declare -a s_err_strings
err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : object info inconsistent "
err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : candidate size 1 info size 7 mismatch"
err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: void SnapSet::decode(ceph::buffer::list::const_iterator&) no longer understand old encoding version 3 < 97"
err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 18/18 objects, 0/0 clones, 17/18 dirty, 17/18 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 113/120 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 1 missing, 7 inconsistent objects"
err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 17 errors"
for err_string in "${err_strings[@]}"
do
if ! grep -q "$err_string" $dir/osd.${primary}.log
then
echo "Missing log message '$err_string'"
ERRORS=$(expr $ERRORS + 1)
fi
done
rados list-inconsistent-pg $poolname > $dir/json || return 1
# Check pg count
test $(jq '. | length' $dir/json) = "1" || return 1
@ -1503,6 +1537,56 @@ EOF
inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1
pg_deep_scrub $pg
err_strings=()
err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key"
err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)], object info inconsistent "
err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)]"
err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : failed to pick suitable auth object"
err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset"
err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key"
err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info"
err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'"
err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:87abbf36:::ROBJ11:head : candidate had a read error"
err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]"
err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8aa5320e:::ROBJ17:head : failed to pick suitable auth object"
err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0xefced57a != omap_digest 0x6a73cc07 from shard 1"
err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0x6a73cc07 != omap_digest 0xefced57a from auth oi 3:8b55fa4b:::ROBJ7:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [[]0 0 0[]][)]"
err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:a53c12e8:::ROBJ6:head : omap_digest 0x689ee887 != omap_digest 0x179c919f from shard 1, omap_digest 0x689ee887 != omap_digest 0x179c919f from auth oi 3:a53c12e8:::ROBJ6:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [[]0 0 0[]][)]"
err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]"
err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:b1f19cbd:::ROBJ10:head : failed to pick suitable auth object"
err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error"
err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key"
err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info"
err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info"
err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch"
err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from shard 0, data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:65 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0"
err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:d60617f9:::ROBJ13:head : candidate had a read error"
err_strings[25]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error"
err_strings[26]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:d60617f9:::ROBJ13:head : failed to pick suitable object info"
err_strings[27]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:e97ce31e:::ROBJ2:head : data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from shard 1, data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from auth oi 3:e97ce31e:::ROBJ2:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [[]0 0 0[]][)]"
err_strings[28]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing"
err_strings[29]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:f4981d31:::ROBJ4:head : omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from shard 1, omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from auth oi 3:f4981d31:::ROBJ4:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [[]0 0 0[]][)]"
err_strings[30]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x1a862a41 != omap_digest 0x6cac8f6 from shard 1"
err_strings[31]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x6cac8f6 != omap_digest 0x1a862a41 from auth oi 3:f4bfd4d1:::ROBJ5:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [[]0 0 0[]][)]"
err_strings[32]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : candidate size 3 info size 7 mismatch"
err_strings[33]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent "
err_strings[34]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr"
err_strings[35]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: void SnapSet::decode[(]ceph::buffer::list::const_iterator&[)] no longer understand old encoding version 3 < 97"
err_strings[36]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub : stat mismatch, got 18/18 objects, 0/0 clones, 17/18 dirty, 17/18 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 115/116 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
err_strings[37]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 1 missing, 11 inconsistent objects"
err_strings[38]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 35 errors"
for err_string in "${err_strings[@]}"
do
if ! grep -q "$err_string" $dir/osd.${primary}.log
then
echo "Missing log message '$err_string'"
ERRORS=$(expr $ERRORS + 1)
fi
done
rados list-inconsistent-pg $poolname > $dir/json || return 1
# Check pg count
test $(jq '. | length' $dir/json) = "1" || return 1
@ -2916,6 +3000,12 @@ EOF
diff -q $dir/new.ROBJ18 $dir/robj18.out || return 1
rm -f $dir/new.ROBJ18 $dir/robj18.out || return 1
if [ $ERRORS != "0" ];
then
echo "TEST FAILED WITH $ERRORS ERRORS"
return 1
fi
rados rmpool $poolname $poolname --yes-i-really-really-mean-it
teardown $dir || return 1
}
@ -5207,6 +5297,7 @@ function TEST_corrupt_snapset_scrub_rep() {
done
local pg=$(get_pg $poolname ROBJ0)
local primary=$(get_primary $poolname ROBJ0)
rados -p $poolname mksnap snap1
echo -n head_of_snapshot_data > $dir/change
@ -5436,6 +5527,30 @@ EOF
jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1
fi
ERRORS=0
declare -a err_strings
err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ1:head : snapset inconsistent"
err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ2:head : snapset inconsistent"
err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 [0-9]*:.*:::ROBJ1:1 : is an unexpected clone"
err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes."
err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 0 missing, 2 inconsistent objects"
err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 4 errors"
for err_string in "${err_strings[@]}"
do
if ! grep -q "$err_string" $dir/osd.${primary}.log
then
echo "Missing log message '$err_string'"
ERRORS=$(expr $ERRORS + 1)
fi
done
if [ $ERRORS != "0" ];
then
echo "TEST FAILED WITH $ERRORS ERRORS"
return 1
fi
rados rmpool $poolname $poolname --yes-i-really-really-mean-it
teardown $dir || return 1
}

56
qa/standalone/scrub/osd-scrub-snaps.sh
View File

@ -689,28 +689,28 @@ EOF
kill_daemons $dir || return 1
declare -a err_strings
err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj10:.* is missing in clone_overlap"
err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 no '_' attr"
err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 is an unexpected clone"
err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:4 on disk size [(]4608[)] does not match object info size [(]512[)] adjusted for ondisk to [(]512[)]"
err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:2"
err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head expected clone .*:::obj5:1"
err_strings[6]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj5:head 2 missing clone[(]s[)]"
err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj8:head snaps.seq not set"
err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:1 is an unexpected clone"
err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj3:head on disk size [(]3840[)] does not match object info size [(]768[)] adjusted for ondisk to [(]768[)]"
err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj6:1 is an unexpected clone"
err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:head no 'snapset' attr"
err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:7 clone ignored due to missing snapset"
err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:4 clone ignored due to missing snapset"
err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj4:head expected clone .*:::obj4:7"
err_strings[15]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj4:head 1 missing clone[(]s[)]"
err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj1:1 is an unexpected clone"
err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj9:1 is missing in clone_size"
err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj11:1 is an unexpected clone"
err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj14:1 size 1032 != clone_size 1033"
err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj10:.* : is missing in clone_overlap"
err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 : no '_' attr"
err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:7 : is an unexpected clone"
err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*::obj5:4 : on disk size [(]4608[)] does not match object info size [(]512[)] adjusted for ondisk to [(]512[)]"
err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head : expected clone .*:::obj5:2"
err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj5:head : expected clone .*:::obj5:1"
err_strings[6]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj5:head : 2 missing clone[(]s[)]"
err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj8:head : snaps.seq not set"
err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj7:1 : is an unexpected clone"
err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj3:head : on disk size [(]3840[)] does not match object info size [(]768[)] adjusted for ondisk to [(]768[)]"
err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj6:1 : is an unexpected clone"
err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:head : no 'snapset' attr"
err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:7 : clone ignored due to missing snapset"
err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj2:4 : clone ignored due to missing snapset"
err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj4:head : expected clone .*:::obj4:7"
err_strings[15]="log_channel[(]cluster[)] log [[]INF[]] : scrub [0-9]*[.]0 .*:::obj4:head : 1 missing clone[(]s[)]"
err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj1:1 : is an unexpected clone"
err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj9:1 : is missing in clone_size"
err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj11:1 : is an unexpected clone"
err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj14:1 : size 1032 != clone_size 1033"
err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 20 errors"
err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj15:head can't decode 'snapset' attr buffer"
err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj15:head : can't decode 'snapset' attr buffer"
for err_string in "${err_strings[@]}"
do
@ -1142,13 +1142,13 @@ fi
kill_daemons $dir || return 1
declare -a err_strings
err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj4:7"
err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1]: soid .*:::obj3:head size 3840 != size 768 from auth oi"
err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj5:1"
err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj5:2"
err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1]: soid .*:::obj5:4 size 4608 != size 512 from auth oi"
err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid .*:::obj5:7: failed to pick suitable object info"
err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] missing .*:::obj1:head"
err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] .*:::obj4:7 : missing"
err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] soid .*:::obj3:head : size 3840 != size 768 from auth oi"
err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] .*:::obj5:1 : missing"
err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] .*:::obj5:2 : missing"
err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] soid .*:::obj5:4 : size 4608 != size 512 from auth oi"
err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid .*:::obj5:7 : failed to pick suitable object info"
err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard [0-1] .*:::obj1:head : missing"
err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub ${scruberrors} errors"
for err_string in "${err_strings[@]}"

1
qa/suites/rados/basic/tasks/repair_test.yaml
View File

@ -17,6 +17,7 @@ overrides:
- 'size 1 != size'
- attr name mismatch
- Regular scrub request, deep-scrub details will be lost
- candidate size [0-9]+ info size [0-9]+ mismatch
- overall HEALTH_
- \(OSDMAP_FLAGS\)
- \(OSD_

3
qa/suites/rados/basic/tasks/scrub_test.yaml
View File

@ -9,12 +9,13 @@ overrides:
- 'deep-scrub [0-9]+ errors'
- 'repair 0 missing, 1 inconsistent objects'
- 'repair [0-9]+ errors, [0-9]+ fixed'
- 'shard [0-9]+ missing'
- 'shard [0-9]+ .* : missing'
- 'deep-scrub 1 missing, 1 inconsistent objects'
- 'does not match object info size'
- 'attr name mistmatch'
- 'deep-scrub 1 missing, 0 inconsistent objects'
- 'failed to pick suitable auth object'
- 'candidate size [0-9]+ info size [0-9]+ mismatch'
- overall HEALTH_
- \(OSDMAP_FLAGS\)
- \(OSD_

1
qa/tasks/repair_test.py
View File

@ -276,6 +276,7 @@ def task(ctx, config):
- 'size 1 != size'
- 'attr name mismatch'
- 'Regular scrub request, deep-scrub details will be lost'
- 'candidate size [0-9]+ info size [0-9]+ mismatch'
conf:
osd:
filestore debug inject read err: true

3
qa/tasks/scrub_test.py
View File

@ -342,12 +342,13 @@ def task(ctx, config):
- deep-scrub [0-9]+ errors
- repair 0 missing, 1 inconsistent objects
- repair [0-9]+ errors, [0-9]+ fixed
- shard [0-9]+ missing
- shard [0-9]+ .* : missing
- deep-scrub 1 missing, 1 inconsistent objects
- does not match object info size
- attr name mistmatch
- deep-scrub 1 missing, 0 inconsistent objects
- failed to pick suitable auth object
- candidate size [0-9]+ info size [0-9]+ mismatch
conf:
osd:
osd deep scrub update digest min age: 0

193
src/osd/PGBackend.cc
View File

@ -624,18 +624,10 @@ bool PGBackend::be_compare_scrub_objects(
const ScrubMap::object &candidate,
shard_info_wrapper &shard_result,
inconsistent_obj_wrapper &obj_result,
ostream &errorstream)
ostream &errorstream,
bool has_snapset)
{
enum { CLEAN, FOUND_ERROR } error = CLEAN;
if (candidate.stat_error) {
assert(shard_result.has_stat_error());
error = FOUND_ERROR;
errorstream << "candidate had a stat error";
}
if (candidate.read_error || candidate.ec_hash_mismatch || candidate.ec_size_mismatch) {
error = FOUND_ERROR;
errorstream << "candidate had a read error";
}
if (auth.digest_present && candidate.digest_present) {
if (auth.digest != candidate.digest) {
if (error != CLEAN)
@ -684,6 +676,67 @@ bool PGBackend::be_compare_scrub_objects(
}
if (candidate.stat_error)
return error == FOUND_ERROR;
if (!shard_result.has_info_missing()
&& !shard_result.has_info_corrupted()) {
bufferlist can_bl, auth_bl;
auto can_attr = candidate.attrs.find(OI_ATTR);
auto auth_attr = auth.attrs.find(OI_ATTR);
assert(auth_attr != auth.attrs.end());
assert(can_attr != candidate.attrs.end());
can_bl.push_back(can_attr->second);
auth_bl.push_back(auth_attr->second);
if (!can_bl.contents_equal(auth_bl)) {
if (error != CLEAN)
errorstream << ", ";
error = FOUND_ERROR;
obj_result.set_object_info_inconsistency();
errorstream << "object info inconsistent ";
}
}
if (has_snapset) {
if (!shard_result.has_snapset_missing()
&& !shard_result.has_snapset_corrupted()) {
bufferlist can_bl, auth_bl;
auto can_attr = candidate.attrs.find(SS_ATTR);
auto auth_attr = auth.attrs.find(SS_ATTR);
assert(auth_attr != auth.attrs.end());
assert(can_attr != candidate.attrs.end());
can_bl.push_back(can_attr->second);
auth_bl.push_back(auth_attr->second);
if (!can_bl.contents_equal(auth_bl)) {
if (error != CLEAN)
errorstream << ", ";
error = FOUND_ERROR;
obj_result.set_snapset_inconsistency();
errorstream << "snapset inconsistent ";
}
}
}
if (parent->get_pool().is_erasure()) {
if (!shard_result.has_hinfo_missing()
&& !shard_result.has_hinfo_corrupted()) {
bufferlist can_bl, auth_bl;
auto can_hi = candidate.attrs.find(ECUtil::get_hinfo_key());
auto auth_hi = auth.attrs.find(ECUtil::get_hinfo_key());
assert(auth_hi != auth.attrs.end());
assert(can_hi != candidate.attrs.end());
can_bl.push_back(can_hi->second);
auth_bl.push_back(auth_hi->second);
if (!can_bl.contents_equal(auth_bl)) {
if (error != CLEAN)
errorstream << ", ";
error = FOUND_ERROR;
obj_result.set_hinfo_inconsistency();
errorstream << "hinfo inconsistent ";
}
}
}
uint64_t oi_size = be_get_ondisk_size(auth_oi.size);
if (oi_size != candidate.size) {
if (error != CLEAN)
@ -756,11 +809,11 @@ map<pg_shard_t, ScrubMap *>::const_iterator
const map<pg_shard_t,ScrubMap*> &maps,
object_info_t *auth_oi,
map<pg_shard_t, shard_info_wrapper> &shard_map,
inconsistent_obj_wrapper &object_error,
bool &digest_match)
bool &digest_match,
spg_t pgid,
ostream &errorstream)
{
eversion_t auth_version;
bufferlist first_oi_bl, first_ss_bl, first_hk_bl;
// Create list of shards with primary first so it will be auth copy all
// other things being equal.
@ -777,27 +830,37 @@ map<pg_shard_t, ScrubMap *>::const_iterator
map<pg_shard_t, ScrubMap *>::const_iterator auth = maps.end();
digest_match = true;
for (auto &l : shards) {
ostringstream shard_errorstream;
bool error = false;
map<pg_shard_t, ScrubMap *>::const_iterator j = maps.find(l);
map<hobject_t, ScrubMap::object>::iterator i =
j->second->objects.find(obj);
if (i == j->second->objects.end()) {
continue;
}
string error_string;
auto& shard_info = shard_map[j->first];
if (j->first == get_parent()->whoami_shard())
shard_info.primary = true;
if (i->second.read_error) {
shard_info.set_read_error();
error_string += " read_error";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a read error";
}
if (i->second.ec_hash_mismatch) {
shard_info.set_ec_hash_mismatch();
error_string += " ec_hash_mismatch";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had an ec hash mismatch";
}
if (i->second.ec_size_mismatch) {
shard_info.set_ec_size_mismatch();
error_string += " ec_size_mismatch";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had an ec size mismatch";
}
object_info_t oi;
@ -808,7 +871,10 @@ map<pg_shard_t, ScrubMap *>::const_iterator
if (i->second.stat_error) {
shard_info.set_stat_error();
error_string += " stat_error";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a stat error";
// With stat_error no further checking
// We don't need to also see a missing_object_info_attr
goto out;
@ -820,22 +886,22 @@ map<pg_shard_t, ScrubMap *>::const_iterator
k = i->second.attrs.find(SS_ATTR);
if (k == i->second.attrs.end()) {
shard_info.set_snapset_missing();
error_string += " snapset_missing";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a missing snapset key";
} else {
ss_bl.push_back(k->second);
try {
auto bliter = ss_bl.cbegin();
decode(ss, bliter);
if (first_ss_bl.length() == 0) {
first_ss_bl.append(ss_bl);
} else if (!object_error.has_snapset_inconsistency() && !ss_bl.contents_equal(first_ss_bl)) {
object_error.set_snapset_inconsistency();
error_string += " snapset_inconsistency";
}
} catch (...) {
// invalid snapset, probably corrupt
shard_info.set_snapset_corrupted();
error_string += " snapset_corrupted";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a corrupt snapset";
}
}
}
@ -845,22 +911,22 @@ map<pg_shard_t, ScrubMap *>::const_iterator
k = i->second.attrs.find(ECUtil::get_hinfo_key());
if (k == i->second.attrs.end()) {
shard_info.set_hinfo_missing();
error_string += " hinfo_key_missing";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a missing hinfo key";
} else {
hk_bl.push_back(k->second);
try {
auto bliter = hk_bl.cbegin();
decode(hi, bliter);
if (first_hk_bl.length() == 0) {
first_hk_bl.append(hk_bl);
} else if (!object_error.has_hinfo_inconsistency() && !hk_bl.contents_equal(first_hk_bl)) {
object_error.set_hinfo_inconsistency();
error_string += " hinfo_inconsistency";
}
} catch (...) {
// invalid snapset, probably corrupt
shard_info.set_hinfo_corrupted();
error_string += " hinfo_corrupted";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a corrupt hinfo";
}
}
}
@ -869,7 +935,10 @@ map<pg_shard_t, ScrubMap *>::const_iterator
if (k == i->second.attrs.end()) {
// no object info on object, probably corrupt
shard_info.set_info_missing();
error_string += " info_missing";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a missing info key";
goto out;
}
bl.push_back(k->second);
@ -879,24 +948,23 @@ map<pg_shard_t, ScrubMap *>::const_iterator
} catch (...) {
// invalid object info, probably corrupt
shard_info.set_info_corrupted();
error_string += " info_corrupted";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate had a corrupt info";
goto out;
}
// This is automatically corrected in PG::_repair_oinfo_oid()
assert(oi.soid == obj);
if (first_oi_bl.length() == 0) {
first_oi_bl.append(bl);
} else if (!object_error.has_object_info_inconsistency() && !bl.contents_equal(first_oi_bl)) {
object_error.set_object_info_inconsistency();
error_string += " object_info_inconsistency";
}
if (i->second.size != be_get_ondisk_size(oi.size)) {
dout(5) << __func__ << " size " << i->second.size << " oi size " << oi.size << dendl;
shard_info.set_obj_size_info_mismatch();
error_string += " obj_size_info_mismatch";
if (error)
shard_errorstream << ", ";
error = true;
shard_errorstream << "candidate size " << i->second.size << " info size "
<< oi.size << " mismatch";
}
// digest_match will only be true if computed digests are the same
@ -923,13 +991,9 @@ map<pg_shard_t, ScrubMap *>::const_iterator
}
out:
// Check error_string because some errors already generated messages
if (error_string != "") {
dout(10) << __func__ << ": error(s) osd " << j->first
<< " for obj " << obj
<< "," << error_string
<< dendl;
}
if (error)
errorstream << pgid.pgid << " shard " << l << " soid " << obj
<< " : " << shard_errorstream.str() << "\n";
// Keep scanning other shards
}
dout(10) << __func__ << ": selecting osd " << auth->first
@ -967,8 +1031,8 @@ void PGBackend::be_compare_scrubmaps(
bool digest_match;
map<pg_shard_t, ScrubMap *>::const_iterator auth =
be_select_auth_object(*k, maps, &auth_oi, shard_map, object_error,
digest_match);
be_select_auth_object(*k, maps, &auth_oi, shard_map, digest_match,
pgid, errorstream);
list<pg_shard_t> auth_list;
set<pg_shard_t> object_errors;
@ -982,7 +1046,7 @@ void PGBackend::be_compare_scrubmaps(
++shallow_errors;
store->add_object_error(k->pool, object_error);
errorstream << pgid.pgid << " soid " << *k
<< ": failed to pick suitable object info\n";
<< " : failed to pick suitable object info\n";
continue;
}
object_error.set_version(auth_oi.user_version);
@ -1004,7 +1068,8 @@ void PGBackend::be_compare_scrubmaps(
j->second->objects[*k],
shard_map[j->first],
object_error,
ss);
ss,
k->has_snapset());
dout(20) << __func__ << (repair ? " repair " : " ") << (parent->get_pool().is_replicated() ? "replicated " : "")
<< (j == auth ? "auth" : "") << "shards " << shard_map.size() << (digest_match ? " digest_match " : " ")
@ -1019,7 +1084,7 @@ void PGBackend::be_compare_scrubmaps(
fix_digest = true;
// Clear the error
shard_map[j->first].clear_data_digest_mismatch_info();
errorstream << pgid << " : soid " << *k << " repairing object info data_digest" << "\n";
errorstream << pgid << " soid " << *k << " : repairing object info data_digest" << "\n";
}
// Some errors might have already been set in be_select_auth_object()
if (shard_map[j->first].errors != 0) {
@ -1031,14 +1096,13 @@ void PGBackend::be_compare_scrubmaps(
// Only true if be_compare_scrub_objects() found errors and put something
// in ss.
if (found)
errorstream << pgid << " shard " << j->first << ": soid " << *k
<< " " << ss.str() << "\n";
} else if (object_error.errors != 0) {
errorstream << pgid << " shard " << j->first << " soid " << *k
<< " : " << ss.str() << "\n";
} else if (found) {
// Track possible shard to use as authoritative, if needed
// There are errors, without identifying the shard
object_errors.insert(j->first);
if (found)
errorstream << pgid << " : soid " << *k << " " << ss.str() << "\n";
errorstream << pgid << " soid " << *k << " : " << ss.str() << "\n";
} else {
// XXX: The auth shard might get here that we don't know
// that it has the "correct" data.
@ -1050,8 +1114,7 @@ void PGBackend::be_compare_scrubmaps(
shard_map[j->first].primary = (j->first == get_parent()->whoami_shard());
// Can't have any other errors if there is no information available
++shallow_errors;
errorstream << pgid << " shard " << j->first << " missing " << *k
<< "\n";
errorstream << pgid << " shard " << j->first << " " << *k << " : missing\n";
}
object_error.add_shard(j->first, shard_map[j->first]);
}
@ -1059,7 +1122,7 @@ void PGBackend::be_compare_scrubmaps(
if (auth_list.empty()) {
if (object_errors.empty()) {
errorstream << pgid.pgid << " soid " << *k
<< ": failed to pick suitable auth object\n";
<< " : failed to pick suitable auth object\n";
goto out;
}
// Object errors exist and nothing in auth_list

8
src/osd/PGBackend.h
View File

@ -570,14 +570,16 @@ typedef std::shared_ptr<const OSDMap> OSDMapRef;
const ScrubMap::object &candidate,
shard_info_wrapper& shard_error,
inconsistent_obj_wrapper &result,
ostream &errorstream);
ostream &errorstream,
bool has_snapset);
map<pg_shard_t, ScrubMap *>::const_iterator be_select_auth_object(
const hobject_t &obj,
const map<pg_shard_t,ScrubMap*> &maps,
object_info_t *auth_oi,
map<pg_shard_t, shard_info_wrapper> &shard_map,
inconsistent_obj_wrapper &object_error,
bool &digest_match);
bool &digest_match,
spg_t pgid,
ostream &errorstream);
void be_compare_scrubmaps(
const map<pg_shard_t,ScrubMap*> &maps,
const set<hobject_t> &master_set,

34
src/osd/PrimaryLogPG.cc
View File

@ -14536,7 +14536,7 @@ void PrimaryLogPG::log_missing(unsigned missing,
<< " skipped " << missing << " clone(s) in cache tier" << dendl;
} else {
clog->info() << mode << " " << pgid << " " << head.get()
<< " " << missing << " missing clone(s)";
<< " : " << missing << " missing clone(s)";
}
}
@ -14563,7 +14563,7 @@ unsigned PrimaryLogPG::process_clones_to(const boost::optional<hobject_t> &head,
if (!allow_incomplete_clones) {
next_clone.snap = **curclone;
clog->error() << mode << " " << pgid << " " << head.get()
<< " expected clone " << next_clone << " " << missing
<< " : expected clone " << next_clone << " " << missing
<< " missing";
++scrubber.shallow_errors;
e.set_clone_missing(next_clone.snap);
@ -14642,7 +14642,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
if (p->second.attrs.count(OI_ATTR) == 0) {
oi = boost::none;
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " no '" << OI_ATTR << "' attr";
<< " : no '" << OI_ATTR << "' attr";
++scrubber.shallow_errors;
soid_error.set_info_missing();
} else {
@ -14654,7 +14654,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
} catch (buffer::error& e) {
oi = boost::none;
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " can't decode '" << OI_ATTR << "' attr " << e.what();
<< " : can't decode '" << OI_ATTR << "' attr " << e.what();
++scrubber.shallow_errors;
soid_error.set_info_corrupted();
soid_error.set_info_missing(); // Not available too
@ -14664,7 +14664,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
if (oi) {
if (pgbackend->be_get_ondisk_size(oi->size) != p->second.size) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " on disk size (" << p->second.size
<< " : on disk size (" << p->second.size
<< ") does not match object info size ("
<< oi->size << ") adjusted for ondisk to ("
<< pgbackend->be_get_ondisk_size(oi->size)
@ -14732,10 +14732,10 @@ void PrimaryLogPG::scrub_snapshot_metadata(
// If we couldn't read the head's snapset, just ignore clones
if (head && !snapset) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " clone ignored due to missing snapset";
<< " : clone ignored due to missing snapset";
} else {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " is an unexpected clone";
<< " : is an unexpected clone";
}
++scrubber.shallow_errors;
soid_error.set_headless();
@ -14767,7 +14767,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
if (p->second.attrs.count(SS_ATTR) == 0) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " no '" << SS_ATTR << "' attr";
<< " : no '" << SS_ATTR << "' attr";
++scrubber.shallow_errors;
snapset = boost::none;
head_error.set_snapset_missing();
@ -14782,7 +14782,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
} catch (buffer::error& e) {
snapset = boost::none;
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " can't decode '" << SS_ATTR << "' attr " << e.what();
<< " : can't decode '" << SS_ATTR << "' attr " << e.what();
++scrubber.shallow_errors;
head_error.set_snapset_corrupted();
}
@ -14796,7 +14796,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
dout(20) << " snapset " << snapset.get() << dendl;
if (snapset->seq == 0) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " snaps.seq not set";
<< " : snaps.seq not set";
++scrubber.shallow_errors;
head_error.set_snapset_error();
}
@ -14812,13 +14812,13 @@ void PrimaryLogPG::scrub_snapshot_metadata(
if (snapset->clone_size.count(soid.snap) == 0) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " is missing in clone_size";
<< " : is missing in clone_size";
++scrubber.shallow_errors;
soid_error.set_size_mismatch();
} else {
if (oi && oi->size != snapset->clone_size[soid.snap]) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " size " << oi->size << " != clone_size "
<< " : size " << oi->size << " != clone_size "
<< snapset->clone_size[*curclone];
++scrubber.shallow_errors;
soid_error.set_size_mismatch();
@ -14826,7 +14826,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
if (snapset->clone_overlap.count(soid.snap) == 0) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " is missing in clone_overlap";
<< " : is missing in clone_overlap";
++scrubber.shallow_errors;
soid_error.set_size_mismatch();
} else {
@ -14849,7 +14849,7 @@ void PrimaryLogPG::scrub_snapshot_metadata(
if (bad_interval_set) {
osd->clog->error() << mode << " " << info.pgid << " " << soid
<< " bad interval_set in clone_overlap";
<< " : bad interval_set in clone_overlap";
++scrubber.shallow_errors;
soid_error.set_size_mismatch();
} else {
@ -14897,8 +14897,8 @@ void PrimaryLogPG::scrub_snapshot_metadata(
continue;
} else if (obc->obs.oi.soid != p->first) {
osd->clog->error() << info.pgid << " " << mode
<< " object " << p->first
<< " has a valid oi attr with a mismatched name, "
<< " " << p->first
<< " : object has a valid oi attr with a mismatched name, "
<< " obc->obs.oi.soid: " << obc->obs.oi.soid;
continue;
}
@ -14980,7 +14980,7 @@ void PrimaryLogPG::_scrub_finish()
scrub_cstat.sum.num_whiteouts != info.stats.stats.sum.num_whiteouts ||
scrub_cstat.sum.num_bytes != info.stats.stats.sum.num_bytes) {
osd->clog->error() << info.pgid << " " << mode
<< " stat mismatch, got "
<< " : stat mismatch, got "
<< scrub_cstat.sum.num_objects << "/" << info.stats.stats.sum.num_objects << " objects, "
<< scrub_cstat.sum.num_object_clones << "/" << info.stats.stats.sum.num_object_clones << " clones, "
<< scrub_cstat.sum.num_objects_dirty << "/" << info.stats.stats.sum.num_objects_dirty << " dirty, "