ceph/qa/standalone/erasure-code/test-erasure-eio.sh

#!/usr/bin/env bash
#
# Copyright (C) 2015 Red Hat <contact@redhat.com>
#
#
# Author: Kefu Chai <kchai@redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Library Public License as published by
# the Free Software Foundation; either version 2, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Library Public License for more details.
#

source $CEPH_ROOT/qa/standalone/ceph-helpers.sh

function run() {
    local dir=$1
    shift

    export CEPH_MON="127.0.0.1:7112" # git grep '\<7112\>' : there must be only one
    export CEPH_ARGS
    CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
    CEPH_ARGS+="--mon-host=$CEPH_MON "

    local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
    for func in $funcs ; do
        setup $dir || return 1
        run_mon $dir a || return 1
	run_mgr $dir x || return 1
	create_pool rbd 4 || return 1

        # check that erasure code plugins are preloaded
        CEPH_ARGS='' ceph --admin-daemon $(get_asok_path mon.a) log flush || return 1
        grep 'load: jerasure.*lrc' $dir/mon.a.log || return 1
        $func $dir || return 1
        teardown $dir || return 1
    done
}

function setup_osds() {
    local count=$1
    shift

    for id in $(seq 0 $(expr $count - 1)) ; do
        run_osd $dir $id || return 1
    done

    # check that erasure code plugins are preloaded
    CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.0) log flush || return 1
    grep 'load: jerasure.*lrc' $dir/osd.0.log || return 1
}

function get_state() {
    local pgid=$1
    local sname=state
    ceph --format json pg dump pgs 2>/dev/null | \
        jq -r ".pg_stats | .[] | select(.pgid==\"$pgid\") | .$sname"
}

function create_erasure_coded_pool() {
    local poolname=$1
    shift
    local k=$1
    shift
    local m=$1
    shift

    ceph osd erasure-code-profile set myprofile \
        plugin=jerasure \
        k=$k m=$m \
        crush-failure-domain=osd || return 1
    create_pool $poolname 1 1 erasure myprofile \
        || return 1
    wait_for_clean || return 1
}

function delete_erasure_coded_pool() {
    local poolname=$1
    ceph osd pool delete $poolname $poolname --yes-i-really-really-mean-it
    ceph osd erasure-code-profile rm myprofile
}

function rados_put() {
    local dir=$1
    local poolname=$2
    local objname=${3:-SOMETHING}

    for marker in AAA BBB CCCC DDDD ; do
        printf "%*s" 1024 $marker
    done > $dir/ORIGINAL
    #
    # get and put an object, compare they are equal
    #
    rados --pool $poolname put $objname $dir/ORIGINAL || return 1
}

function rados_get() {
    local dir=$1
    local poolname=$2
    local objname=${3:-SOMETHING}
    local expect=${4:-ok}

    #
    # Expect a failure to get object
    #
    if [ $expect = "fail" ];
    then
        ! rados --pool $poolname get $objname $dir/COPY
        return
    fi
    #
    # get an object, compare with $dir/ORIGINAL
    #
    rados --pool $poolname get $objname $dir/COPY || return 1
    diff $dir/ORIGINAL $dir/COPY || return 1
    rm $dir/COPY
}


function inject_remove() {
    local pooltype=$1
    shift
    local which=$1
    shift
    local poolname=$1
    shift
    local objname=$1
    shift
    local dir=$1
    shift
    local shard_id=$1
    shift

    local -a initial_osds=($(get_osds $poolname $objname))
    local osd_id=${initial_osds[$shard_id]}
    objectstore_tool $dir $osd_id $objname remove || return 1
}

# Test with an inject error
function rados_put_get_data() {
    local inject=$1
    shift
    local dir=$1
    shift
    local shard_id=$1
    shift
    local arg=$1

    # inject eio to speificied shard
    #
    local poolname=pool-jerasure
    local objname=obj-$inject-$$-$shard_id
    rados_put $dir $poolname $objname || return 1
    inject_$inject ec data $poolname $objname $dir $shard_id || return 1
    rados_get $dir $poolname $objname || return 1

    if [ "$arg" = "recovery" ];
    then
        #
        # take out the last OSD used to store the object,
        # bring it back, and check for clean PGs which means
        # recovery didn't crash the primary.
        #
        local -a initial_osds=($(get_osds $poolname $objname))
        local last_osd=${initial_osds[-1]}
        # Kill OSD
        kill_daemons $dir TERM osd.${last_osd} >&2 < /dev/null || return 1
        ceph osd out ${last_osd} || return 1
        ! get_osds $poolname $objname | grep '\<'${last_osd}'\>' || return 1
        ceph osd in ${last_osd} || return 1
        activate_osd $dir ${last_osd} || return 1
        wait_for_clean || return 1
        # Won't check for eio on get here -- recovery above might have fixed it
    else
        shard_id=$(expr $shard_id + 1)
        inject_$inject ec data $poolname $objname $dir $shard_id || return 1
        rados_get $dir $poolname $objname fail || return 1
        rm $dir/ORIGINAL
    fi

}

# Change the size of speificied shard
#
function set_size() {
    local objname=$1
    shift
    local dir=$1
    shift
    local shard_id=$1
    shift
    local bytes=$1
    shift
    local mode=${1}

    local poolname=pool-jerasure
    local -a initial_osds=($(get_osds $poolname $objname))
    local osd_id=${initial_osds[$shard_id]}
    ceph osd set noout
    if [ "$mode" = "add" ];
    then
      objectstore_tool $dir $osd_id $objname get-bytes $dir/CORRUPT || return 1
      dd if=/dev/urandom bs=$bytes count=1 >> $dir/CORRUPT
    elif [ "$bytes" = "0" ];
    then
      touch $dir/CORRUPT
    else
      dd if=/dev/urandom bs=$bytes count=1 of=$dir/CORRUPT
    fi
    objectstore_tool $dir $osd_id $objname set-bytes $dir/CORRUPT || return 1
    rm -f $dir/CORRUPT
    ceph osd unset noout
}

function rados_get_data_bad_size() {
    local dir=$1
    shift
    local shard_id=$1
    shift
    local bytes=$1
    shift
    local mode=${1:-set}

    local poolname=pool-jerasure
    local objname=obj-size-$$-$shard_id-$bytes
    rados_put $dir $poolname $objname || return 1

    # Change the size of speificied shard
    #
    set_size $objname $dir $shard_id $bytes $mode || return 1

    rados_get $dir $poolname $objname || return 1

    # Leave objname and modify another shard
    shard_id=$(expr $shard_id + 1)
    set_size $objname $dir $shard_id $bytes $mode || return 1
    rados_get $dir $poolname $objname fail || return 1
    rm $dir/ORIGINAL
}

#
# These two test cases try to validate the following behavior:
#  For object on EC pool, if there is one shard having read error (
#  either primary or replica), client can still read object.
#
# If 2 shards have read errors the client will get an error.
#
function TEST_rados_get_subread_eio_shard_0() {
    local dir=$1
    setup_osds 4 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 2 1 || return 1
    # inject eio on primary OSD (0) and replica OSD (1)
    local shard_id=0
    rados_put_get_data eio $dir $shard_id || return 1
    delete_erasure_coded_pool $poolname
}

function TEST_rados_get_subread_eio_shard_1() {
    local dir=$1
    setup_osds 4 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 2 1 || return 1
    # inject eio into replicas OSD (1) and OSD (2)
    local shard_id=1
    rados_put_get_data eio $dir $shard_id || return 1
    delete_erasure_coded_pool $poolname
}

# We don't remove the object from the primary because
# that just causes it to appear to be missing

function TEST_rados_get_subread_missing() {
    local dir=$1
    setup_osds 4 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 2 1 || return 1
    # inject remove into replicas OSD (1) and OSD (2)
    local shard_id=1
    rados_put_get_data remove $dir $shard_id || return 1
    delete_erasure_coded_pool $poolname
}

#
#
# These two test cases try to validate that following behavior:
#  For object on EC pool, if there is one shard which an incorrect
# size this will cause an internal read error, client can still read object.
#
# If 2 shards have incorrect size the client will get an error.
#
function TEST_rados_get_bad_size_shard_0() {
    local dir=$1
    setup_osds 4 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 2 1 || return 1
    # Set incorrect size into primary OSD (0) and replica OSD (1)
    local shard_id=0
    rados_get_data_bad_size $dir $shard_id 10 || return 1
    rados_get_data_bad_size $dir $shard_id 0 || return 1
    rados_get_data_bad_size $dir $shard_id 256 add || return 1
    delete_erasure_coded_pool $poolname
}

function TEST_rados_get_bad_size_shard_1() {
    local dir=$1
    setup_osds 4 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 2 1 || return 1
    # Set incorrect size into replicas OSD (1) and OSD (2)
    local shard_id=1
    rados_get_data_bad_size $dir $shard_id 10 || return 1
    rados_get_data_bad_size $dir $shard_id 0 || return 1
    rados_get_data_bad_size $dir $shard_id 256 add || return 1
    delete_erasure_coded_pool $poolname
}

function TEST_rados_get_with_subreadall_eio_shard_0() {
    local dir=$1
    local shard_id=0

    setup_osds 4 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 2 1 || return 1
    # inject eio on primary OSD (0)
    rados_put_get_data eio $dir $shard_id recovery || return 1

    delete_erasure_coded_pool $poolname
}

function TEST_rados_get_with_subreadall_eio_shard_1() {
    local dir=$1
    local shard_id=1

    setup_osds 4 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 2 1 || return 1
    # inject eio on replica OSD (1)
    rados_put_get_data eio $dir $shard_id recovery || return 1

    delete_erasure_coded_pool $poolname
}

# Test recovery the object attr read error
function TEST_ec_object_attr_read_error() {
    local dir=$1
    local objname=myobject

    setup_osds 7 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 3 2 || return 1

    local primary_osd=$(get_primary $poolname $objname)
    # Kill primary OSD
    kill_daemons $dir TERM osd.${primary_osd} >&2 < /dev/null || return 1

    # Write data
    rados_put $dir $poolname $objname || return 1

    # Inject eio, shard 1 is the one read attr
    inject_eio ec mdata $poolname $objname $dir 1 || return 1

    # Restart OSD
    activate_osd $dir ${primary_osd} || return 1

    # Cluster should recover this object
    wait_for_clean || return 1

    rados_get $dir $poolname myobject || return 1

    delete_erasure_coded_pool $poolname
}

# Test recovery the first k copies aren't all available
function TEST_ec_single_recovery_error() {
    local dir=$1
    local objname=myobject

    setup_osds 7 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 3 2 || return 1

    rados_put $dir $poolname $objname || return 1
    inject_eio ec data $poolname $objname $dir 0 || return 1

    local -a initial_osds=($(get_osds $poolname $objname))
    local last_osd=${initial_osds[-1]}
    # Kill OSD
    kill_daemons $dir TERM osd.${last_osd} >&2 < /dev/null || return 1
    ceph osd down ${last_osd} || return 1
    ceph osd out ${last_osd} || return 1

    # Cluster should recover this object
    wait_for_clean || return 1

    rados_get $dir $poolname myobject || return 1

    delete_erasure_coded_pool $poolname
}

# Test recovery when repeated reads are needed due to EIO
function TEST_ec_recovery_multiple_errors() {
    local dir=$1
    local objname=myobject

    setup_osds 9 || return 1

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 4 4 || return 1

    rados_put $dir $poolname $objname || return 1
    inject_eio ec data $poolname $objname $dir 0 || return 1
    # first read will try shards 0,1,2 when 0 gets EIO, shard 3 gets
    # tried as well. Make that fail to test multiple-EIO handling.
    inject_eio ec data $poolname $objname $dir 3 || return 1
    inject_eio ec data $poolname $objname $dir 4 || return 1

    local -a initial_osds=($(get_osds $poolname $objname))
    local last_osd=${initial_osds[-1]}
    # Kill OSD
    kill_daemons $dir TERM osd.${last_osd} >&2 < /dev/null || return 1
    ceph osd down ${last_osd} || return 1
    ceph osd out ${last_osd} || return 1

    # Cluster should recover this object
    wait_for_clean || return 1

    rados_get $dir $poolname myobject || return 1

    delete_erasure_coded_pool $poolname
}

# Test recovery when there's only one shard to recover, but multiple
# objects recovering in one RecoveryOp
function TEST_ec_recovery_multiple_objects() {
    local dir=$1
    local objname=myobject

    ORIG_ARGS=$CEPH_ARGS
    CEPH_ARGS+=' --osd-recovery-max-single-start 3 --osd-recovery-max-active 3 '
    setup_osds 7 || return 1
    CEPH_ARGS=$ORIG_ARGS

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 3 2 || return 1

    rados_put $dir $poolname test1
    rados_put $dir $poolname test2
    rados_put $dir $poolname test3

    ceph osd out 0 || return 1

    # Cluster should recover these objects all at once
    wait_for_clean || return 1

    rados_get $dir $poolname test1
    rados_get $dir $poolname test2
    rados_get $dir $poolname test3

    delete_erasure_coded_pool $poolname
}

# test multi-object recovery when the one missing shard gets EIO
function TEST_ec_recovery_multiple_objects_eio() {
    local dir=$1
    local objname=myobject

    ORIG_ARGS=$CEPH_ARGS
    CEPH_ARGS+=' --osd-recovery-max-single-start 3 --osd-recovery-max-active 3 '
    setup_osds 7 || return 1
    CEPH_ARGS=$ORIG_ARGS

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 3 2 || return 1

    rados_put $dir $poolname test1
    rados_put $dir $poolname test2
    rados_put $dir $poolname test3

    # can't read from this shard anymore
    inject_eio ec data $poolname $objname $dir 0 || return 1
    ceph osd out 0 || return 1

    # Cluster should recover these objects all at once
    wait_for_clean || return 1

    rados_get $dir $poolname test1
    rados_get $dir $poolname test2
    rados_get $dir $poolname test3

    delete_erasure_coded_pool $poolname
}

# Test backfill with unfound object
function TEST_ec_backfill_unfound() {
    local dir=$1
    local objname=myobject
    local lastobj=300
    # Must be between 1 and $lastobj
    local testobj=obj250

    ORIG_ARGS=$CEPH_ARGS
    CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10'
    setup_osds 5 || return 1
    CEPH_ARGS=$ORIG_ARGS

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 3 2 || return 1

    ceph pg dump pgs

    rados_put $dir $poolname $objname || return 1

    local -a initial_osds=($(get_osds $poolname $objname))
    local last_osd=${initial_osds[-1]}
    kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1
    ceph osd down ${last_osd} || return 1
    ceph osd out ${last_osd} || return 1

    ceph pg dump pgs

    dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4
    for i in $(seq 1 $lastobj)
    do
      rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1
    done

    inject_eio ec data $poolname $testobj $dir 0 || return 1
    inject_eio ec data $poolname $testobj $dir 1 || return 1

    activate_osd $dir ${last_osd} || return 1
    ceph osd in ${last_osd} || return 1

    sleep 15

    for tmp in $(seq 1 100); do
      state=$(get_state 2.0)
      echo $state | grep backfill_unfound
      if [ "$?" = "0" ]; then
        break
      fi
      echo $state
      sleep 1
    done

    ceph pg dump pgs
    ceph pg 2.0 list_unfound | grep -q $testobj || return 1

    # Command should hang because object is unfound
    timeout 5 rados -p $poolname get $testobj $dir/CHECK
    test $? = "124" || return 1

    ceph pg 2.0 mark_unfound_lost delete

    wait_for_clean || return 1

    for i in $(seq 1 $lastobj)
    do
      if [ obj${i} = "$testobj" ]; then
        # Doesn't exist anymore
        ! rados -p $poolname get $testobj $dir/CHECK || return 1
      else
        rados --pool $poolname get obj${i} $dir/CHECK || return 1
        diff -q $dir/ORIGINAL $dir/CHECK || return 1
      fi
    done

    rm -f ${dir}/ORIGINAL ${dir}/CHECK

    delete_erasure_coded_pool $poolname
}

# Test recovery with unfound object
function TEST_ec_recovery_unfound() {
    local dir=$1
    local objname=myobject
    local lastobj=100
    # Must be between 1 and $lastobj
    local testobj=obj75

    ORIG_ARGS=$CEPH_ARGS
    CEPH_ARGS+=' --osd-recovery-max-single-start 3 --osd-recovery-max-active 3 '
    CEPH_ARGS+=' --osd_min_pg_log_entries=5 --osd_max_pg_log_entries=10'
    setup_osds 5 || return 1
    CEPH_ARGS=$ORIG_ARGS

    local poolname=pool-jerasure
    create_erasure_coded_pool $poolname 3 2 || return 1

    ceph pg dump pgs

    rados_put $dir $poolname $objname || return 1

    local -a initial_osds=($(get_osds $poolname $objname))
    local last_osd=${initial_osds[-1]}
    kill_daemons $dir TERM osd.${last_osd} 2>&2 < /dev/null || return 1
    ceph osd down ${last_osd} || return 1
    ceph osd out ${last_osd} || return 1

    ceph pg dump pgs

    dd if=/dev/urandom of=${dir}/ORIGINAL bs=1024 count=4
    for i in $(seq 1 $lastobj)
    do
      rados --pool $poolname put obj${i} $dir/ORIGINAL || return 1
    done

    inject_eio ec data $poolname $testobj $dir 0 || return 1
    inject_eio ec data $poolname $testobj $dir 1 || return 1

    activate_osd $dir ${last_osd} || return 1
    ceph osd in ${last_osd} || return 1

    sleep 15

    for tmp in $(seq 1 100); do
      state=$(get_state 2.0)
      echo $state | grep recovery_unfound
      if [ "$?" = "0" ]; then
        break
      fi
      echo "$state "
      sleep 1
    done

    ceph pg dump pgs
    ceph pg 2.0 list_unfound | grep -q $testobj || return 1

    # Command should hang because object is unfound
    timeout 5 rados -p $poolname get $testobj $dir/CHECK
    test $? = "124" || return 1

    ceph pg 2.0 mark_unfound_lost delete

    wait_for_clean || return 1

    for i in $(seq 1 $lastobj)
    do
      if [ obj${i} = "$testobj" ]; then
        # Doesn't exist anymore
        ! rados -p $poolname get $testobj $dir/CHECK || return 1
      else
        rados --pool $poolname get obj${i} $dir/CHECK || return 1
        diff -q $dir/ORIGINAL $dir/CHECK || return 1
      fi
    done

    rm -f ${dir}/ORIGINAL ${dir}/CHECK

    delete_erasure_coded_pool $poolname
}

main test-erasure-eio "$@"

# Local Variables:
# compile-command: "cd ../.. ; make -j4 && test/erasure-code/test-erasure-eio.sh"
# End: