diff --git a/qa/workunits/hadoop/internal-tests.sh b/qa/workunits/hadoop/internal-tests.sh deleted file mode 100755 index 48a9224dac1..00000000000 --- a/qa/workunits/hadoop/internal-tests.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -e - -# bail if $TESTDIR is not set as this test will fail in that scenario -[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } - -# configure CEPH_CONF and LD_LIBRARY_PATH if they're not already set -conf="$CEPH_CONF" -if [ -z "$conf" ] ; then - echo "Setting conf to /etc/ceph/ceph.conf" - conf="/etc/ceph/ceph.conf" -else - echo "conf is set to $conf" -fi - -ld_lib_path="$LD_LIBRARY_PATH" -if [ -z "$ld_lib_path" ] ; then - echo "Setting ld_lib_path to /usr/lib/jni" - ld_lib_path="/usr/lib/jni" -else - echo "ld_lib_path was set to $ld_lib_path" -fi - -POOL_SIZES=`seq 1 8` -POOL_BASE=hadoop -POOL_NAMES=`echo -n $POOL_SIZES | sed "s/\([0-9]*\)/$POOL_BASE\1/g" | sed "s/ /,/g"` - -function gen_hadoop_conf() { -local outfile=$1 -local poolnames=$2 -local conf=$3 -cat << EOF > $outfile - - - - - ceph.conf.file - $conf - - - ceph.data.pools - $poolnames - - -EOF -} - -echo creating hadoop test pools -for size in $POOL_SIZES; do - name=${POOL_BASE}$size - echo creating pool $name - ceph osd pool create $name 100 100 - ceph osd pool set $name size $size - - echo making pool $name a data pool - poolid=`ceph osd dump | sed -n "s/^pool \([0-9]*\) '$name'.*/\1/p"` - ceph mds add_data_pool $poolid -done - -def_repl_conf=`mktemp` -echo generating default replication hadoop config $def_repl_conf -gen_hadoop_conf $def_repl_conf "" $conf - -cust_repl_conf=`mktemp` -echo generating custom replication hadoop config $cust_repl_conf -gen_hadoop_conf $cust_repl_conf $POOL_NAMES $conf - -echo running default replication hadoop tests -java -Dhadoop.conf.file=$def_repl_conf -Djava.library.path=$ld_lib_path -cp /usr/share/java/junit4.jar:$TESTDIR/apache_hadoop/build/hadoop-core-1.0.4-SNAPSHOT.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs-test.jar:$TESTDIR/apache_hadoop/build/hadoop-test-1.0.4-SNAPSHOT.jar:$TESTDIR/apache_hadoop/build/ivy/lib/Hadoop/common/commons-logging-1.1.1.jar:/usr/share/java/libcephfs.jar org.junit.runner.JUnitCore org.apache.hadoop.fs.ceph.TestCephDefaultReplication - -echo running custom replication hadoop tests -java -Dhadoop.conf.file=$cust_repl_conf -Djava.library.path=$ld_lib_path -cp /usr/share/java/junit4.jar:$TESTDIR/apache_hadoop/build/hadoop-core-1.0.4-SNAPSHOT.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs.jar:$TESTDIR/inktank_hadoop/build/hadoop-cephfs-test.jar:$TESTDIR/apache_hadoop/build/hadoop-test-1.0.4-SNAPSHOT.jar:$TESTDIR/apache_hadoop/build/ivy/lib/Hadoop/common/commons-logging-1.1.1.jar:/usr/share/java/libcephfs.jar org.junit.runner.JUnitCore org.apache.hadoop.fs.ceph.TestCephCustomReplication - -echo "completed hadoop-internal-tests tests" -exit 0 diff --git a/qa/workunits/hadoop/repl.sh b/qa/workunits/hadoop/repl.sh new file mode 100755 index 00000000000..2284cce0003 --- /dev/null +++ b/qa/workunits/hadoop/repl.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e +set -x + +# bail if $TESTDIR is not set as this test will fail in that scenario +[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; } + +# if HADOOP_PREFIX is not set, use default +[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; } + +export JAVA_HOME=/usr/lib/jvm/default-java + +# create pools with different replication factors +for repl in 2 3 7 8 9; do + name=hadoop.$repl + ceph osd pool create $name 8 8 + ceph osd pool set $name size $repl + + id=`ceph osd dump | sed -n "s/^pool \([0-9]*\) '$name'.*/\1/p"` + ceph mds add_data_pool $id +done + +# create a file in each of the pools +for repl in 2 3 7 8 9; do + name=hadoop.$repl + $HADOOP_PREFIX/bin/hadoop fs -rm -f /$name.dat + dd if=/dev/zero bs=1048576 count=1 | \ + $HADOOP_PREFIX/bin/hadoop fs -Dceph.data.pools="$name" \ + -put - /$name.dat +done + +# check that hadoop reports replication matching +# that of the pool the file was written into +for repl in 2 3 7 8 9; do + name=hadoop.$repl + repl2=$($HADOOP_PREFIX/bin/hadoop fs -ls /$name.dat | awk '{print $2}') + if [ $repl -ne $repl2 ]; then + echo "replication factors didn't match!" + exit 1 + fi +done + +exit 0