PageRenderTime 13ms CodeModel.GetById 11ms app.highlight 0ms RepoModel.GetById 1ms app.codeStats 0ms

/tags/release-0.0.0-rc0/hive/external/testutils/compute_stats

#
#! | 36 lines | 33 code | 3 blank | 0 comment | 0 complexity | ddbc4146ca76bb4285a5f753325961ba MD5 | raw file
 1
 2if [ "$1" = '' ]; then
 3echo "Usage: compute_stats '<table_name>(<comma separated column list>)'"
 4exit 1
 5fi
 6d=`mktemp -d compute_stats_XXXX`
 7echo -n "Removing hive temp tables from previous runs if any..."
 8test=`/mnt/vol/hive/stable/cluster/bin/hadoop dfs -ls compute_stats_test 2>>$d/err`
 9if [ "$test" != '' ]; then
10  /mnt/vol/hive/stable/cluster/bin/hadoop dfs -rmr compute_stats_test >>$d/err 2>&1
11  /mnt/vol/hive/stable/cluster/bin/hadoop dfs -rmr /Trash/Current/compute_stats_test >> $d/err 2>&1
12fi
13echo "Done"
14
15echo -n "Project out columns to test table using hive..."
16/mnt/vol/hive/stable/cluster/bin/hadoop jar /mnt/vol/hive/stable/lib/hive/fb_hive.jar com.facebook.hive.streaming.HiveStreaming -select $1 -sample 1/1024 -output compute_stats_test -numReduceTasks 0 >> $d/err 2>&1
17echo "Done"
18
19test=`/mnt/vol/hive/stable/cluster/bin/hadoop dfs -ls compute_stats_test 2>>$d/err`
20if [ "$test" != '' ]; then
21  echo -n "Generate single local file out of hdfs table..."
22  /mnt/vol/hive/stable/cluster/bin/hadoop dfs -cat /user/`whoami`/compute_stats_test/* 2>>$d/err | gzip > $d/sample.gz
23  echo "Done"
24  n=`zcat $d/sample.gz | head -1 | tr ',' '\n' | wc -l`
25  echo "Generate stats for each of the projected columns..."
26  for ((i=1;i<=$n;i+=1)); do 
27    echo "  column $i"
28    zcat $d/sample.gz | cut -f$i -d',' | sort | uniq -c > $d/$i.stats; 
29    count=`cat $d/$i.stats | wc -l`; 
30    echo $i $count >> $d/overall.stats; 
31  done
32  echo "Done"
33  echo "Stats in $d"
34else
35  echo "Error extracting columns from table!";
36fi