mirror of
https://github.com/ceph/ceph
synced 2024-12-12 06:28:31 +00:00
de6b53aba7
This brings the hadoop wordcount up-to-date with the new teuthology hadoop task. Signed-off-by: Noah Watkins <noahwatkins@gmail.com>
38 lines
1001 B
Bash
Executable File
38 lines
1001 B
Bash
Executable File
#!/bin/bash
|
|
|
|
set -e
|
|
set -x
|
|
|
|
WC_INPUT=/wc_input
|
|
WC_OUTPUT=/wc_output
|
|
DATA_INPUT=$(mktemp -d)
|
|
|
|
echo "starting hadoop-wordcount test"
|
|
|
|
# bail if $TESTDIR is not set as this test will fail in that scenario
|
|
[ -z $TESTDIR ] && { echo "\$TESTDIR needs to be set, but is not. Exiting."; exit 1; }
|
|
|
|
# if HADOOP_PREFIX is not set, use default
|
|
[ -z $HADOOP_PREFIX ] && { HADOOP_PREFIX=$TESTDIR/hadoop; }
|
|
|
|
export JAVA_HOME=/usr/lib/jvm/default-java
|
|
|
|
# Nuke hadoop directories
|
|
$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true
|
|
|
|
# Fetch and import testing data set
|
|
curl http://ceph.com/qa/hadoop_input_files.tar | tar xf - -C $DATA_INPUT
|
|
$HADOOP_PREFIX/bin/hadoop fs -copyFromLocal $DATA_INPUT $WC_INPUT
|
|
rm -rf $DATA_INPUT
|
|
|
|
# Run the job
|
|
$HADOOP_PREFIX/bin/hadoop jar \
|
|
$HADOOP_PREFIX/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar \
|
|
wordcount $WC_INPUT $WC_OUTPUT
|
|
|
|
# Cleanup
|
|
$HADOOP_PREFIX/bin/hadoop fs -rm -r $WC_INPUT $WC_OUTPUT || true
|
|
|
|
echo "completed hadoop-wordcount test"
|
|
exit 0
|