/Misc/build.sh

http://unladen-swallow.googlecode.com/ · Shell · 296 lines · 197 code · 31 blank · 68 comment · 21 complexity · 5f4231f4d571823c918465badac5dda2 MD5 · raw file

  1. #!/bin/sh
  2. ## Script to build and test the latest python from svn. It basically
  3. ## does this:
  4. ## svn up ; ./configure ; make ; make test ; make install ; cd Doc ; make
  5. ##
  6. ## Logs are kept and rsync'ed to the host. If there are test failure(s),
  7. ## information about the failure(s) is mailed.
  8. ##
  9. ## This script is run on the PSF's machine as user neal via crontab.
  10. ##
  11. ## Yes, this script would probably be easier in python, but then
  12. ## there's a bootstrap problem. What if Python doesn't build?
  13. ##
  14. ## This script should be fairly clean Bourne shell, ie not too many
  15. ## bash-isms. We should try to keep it portable to other Unixes.
  16. ## Even though it will probably only run on Linux. I'm sure there are
  17. ## several GNU-isms currently (date +%s and readlink).
  18. ##
  19. ## Perhaps this script should be broken up into 2 (or more) components.
  20. ## Building doc is orthogonal to the rest of the python build/test.
  21. ##
  22. ## FIXME: we should detect test hangs (eg, if they take more than 45 minutes)
  23. ## FIXME: we should run valgrind
  24. ## FIXME: we should run code coverage
  25. ## Utilities invoked in this script include:
  26. ## basename, date, dirname, expr, grep, readlink, uname
  27. ## cksum, make, mutt, rsync, svn
  28. ## remember where did we started from
  29. DIR=`dirname $0`
  30. if [ "$DIR" = "" ]; then
  31. DIR="."
  32. fi
  33. ## make directory absolute
  34. DIR=`readlink -f $DIR`
  35. FULLPATHNAME="$DIR/`basename $0`"
  36. ## we want Misc/..
  37. DIR=`dirname $DIR`
  38. ## Configurable options
  39. FAILURE_SUBJECT="Python Regression Test Failures"
  40. #FAILURE_MAILTO="YOUR_ACCOUNT@gmail.com"
  41. FAILURE_MAILTO="python-checkins@python.org"
  42. #FAILURE_CC="optional--uncomment and set to desired address"
  43. REMOTE_SYSTEM="neal@dinsdale.python.org"
  44. REMOTE_DIR="/data/ftp.python.org/pub/www.python.org/doc/current"
  45. REMOTE_DIR_DIST="/data/ftp.python.org/pub/python/doc/current"
  46. RESULT_FILE="$DIR/build/index.html"
  47. INSTALL_DIR="/tmp/python-test-2.6/local"
  48. RSYNC_OPTS="-aC -e ssh"
  49. # Always run the installed version of Python.
  50. PYTHON=$INSTALL_DIR/bin/python
  51. # Python options and regression test program that should always be run.
  52. REGRTEST_ARGS="-E -tt $INSTALL_DIR/lib/python2.6/test/regrtest.py"
  53. REFLOG="build/reflog.txt.out"
  54. # These tests are not stable and falsely report leaks sometimes.
  55. # The entire leak report will be mailed if any test not in this list leaks.
  56. # Note: test_XXX (none currently) really leak, but are disabled
  57. # so we don't send spam. Any test which really leaks should only
  58. # be listed here if there are also test cases under Lib/test/leakers.
  59. LEAKY_TESTS="test_(asynchat|cmd_line|docxmlrpc|dumbdbm|file|ftplib|httpservers|imaplib|popen2|socket|smtplib|sys|telnetlib|threadedtempfile|threading|threadsignals|xmlrpc)"
  60. # Skip these tests altogether when looking for leaks. These tests
  61. # do not need to be stored above in LEAKY_TESTS too.
  62. # test_compiler almost never finishes with the same number of refs
  63. # since it depends on other modules, skip it.
  64. # test_logging causes hangs, skip it.
  65. LEAKY_SKIPS="-x test_compiler test_logging"
  66. # Change this flag to "yes" for old releases to only update/build the docs.
  67. BUILD_DISABLED="yes"
  68. ## utility functions
  69. current_time() {
  70. date +%s
  71. }
  72. update_status() {
  73. now=`current_time`
  74. time=`expr $now - $3`
  75. echo "<li><a href=\"$2\">$1</a> <font size=\"-1\">($time seconds)</font></li>" >> $RESULT_FILE
  76. }
  77. place_summary_first() {
  78. testf=$1
  79. sed -n '/^[0-9][0-9]* tests OK\./,$p' < $testf \
  80. | egrep -v '\[[0-9]+ refs\]' > $testf.tmp
  81. echo "" >> $testf.tmp
  82. cat $testf >> $testf.tmp
  83. mv $testf.tmp $testf
  84. }
  85. count_failures () {
  86. testf=$1
  87. n=`grep -ic " failed:" $testf`
  88. if [ $n -eq 1 ] ; then
  89. n=`grep " failed:" $testf | sed -e 's/ .*//'`
  90. fi
  91. echo $n
  92. }
  93. mail_on_failure() {
  94. if [ "$NUM_FAILURES" != "0" ]; then
  95. dest=$FAILURE_MAILTO
  96. # FAILURE_CC is optional.
  97. if [ "$FAILURE_CC" != "" ]; then
  98. dest="$dest -c $FAILURE_CC"
  99. fi
  100. if [ "x$3" != "x" ] ; then
  101. (echo "More important issues:"
  102. echo "----------------------"
  103. egrep -v "$3" < $2
  104. echo ""
  105. echo "Less important issues:"
  106. echo "----------------------"
  107. egrep "$3" < $2)
  108. else
  109. cat $2
  110. fi | mutt -s "$FAILURE_SUBJECT $1 ($NUM_FAILURES)" $dest
  111. fi
  112. }
  113. ## setup
  114. cd $DIR
  115. mkdir -p build
  116. rm -f $RESULT_FILE build/*.out
  117. rm -rf $INSTALL_DIR
  118. ## create results file
  119. TITLE="Automated Python Build Results"
  120. echo "<html>" >> $RESULT_FILE
  121. echo " <head>" >> $RESULT_FILE
  122. echo " <title>$TITLE</title>" >> $RESULT_FILE
  123. echo " <meta http-equiv=\"refresh\" content=\"43200\">" >> $RESULT_FILE
  124. echo " </head>" >> $RESULT_FILE
  125. echo "<body>" >> $RESULT_FILE
  126. echo "<h2>Automated Python Build Results</h2>" >> $RESULT_FILE
  127. echo "<table>" >> $RESULT_FILE
  128. echo " <tr>" >> $RESULT_FILE
  129. echo " <td>Built on:</td><td>`date`</td>" >> $RESULT_FILE
  130. echo " </tr><tr>" >> $RESULT_FILE
  131. echo " <td>Hostname:</td><td>`uname -n`</td>" >> $RESULT_FILE
  132. echo " </tr><tr>" >> $RESULT_FILE
  133. echo " <td>Platform:</td><td>`uname -srmpo`</td>" >> $RESULT_FILE
  134. echo " </tr>" >> $RESULT_FILE
  135. echo "</table>" >> $RESULT_FILE
  136. echo "<ul>" >> $RESULT_FILE
  137. ## update, build, and test
  138. ORIG_CHECKSUM=`cksum $FULLPATHNAME`
  139. F=svn-update.out
  140. start=`current_time`
  141. svn update >& build/$F
  142. err=$?
  143. update_status "Updating" "$F" $start
  144. if [ $err = 0 -a "$BUILD_DISABLED" != "yes" ]; then
  145. ## FIXME: we should check if this file has changed.
  146. ## If it has changed, we should re-run the script to pick up changes.
  147. if [ "$ORIG_CHECKSUM" != "$ORIG_CHECKSUM" ]; then
  148. exec $FULLPATHNAME $@
  149. fi
  150. F=svn-stat.out
  151. start=`current_time`
  152. svn stat >& build/$F
  153. ## ignore some of the diffs
  154. NUM_DIFFS=`egrep -vc '^. (@test|db_home|Lib/test/(regrtest\.py|db_home))$' build/$F`
  155. update_status "svn stat ($NUM_DIFFS possibly important diffs)" "$F" $start
  156. F=configure.out
  157. start=`current_time`
  158. ./configure --prefix=$INSTALL_DIR --with-pydebug >& build/$F
  159. err=$?
  160. update_status "Configuring" "$F" $start
  161. if [ $err = 0 ]; then
  162. F=make.out
  163. start=`current_time`
  164. make >& build/$F
  165. err=$?
  166. warnings=`grep warning build/$F | egrep -vc "te?mpnam(_r|)' is dangerous,"`
  167. update_status "Building ($warnings warnings)" "$F" $start
  168. if [ $err = 0 ]; then
  169. ## make install
  170. F=make-install.out
  171. start=`current_time`
  172. make install >& build/$F
  173. update_status "Installing" "$F" $start
  174. if [ ! -x $PYTHON ]; then
  175. ln -s ${PYTHON}2.* $PYTHON
  176. fi
  177. ## make and run basic tests
  178. F=make-test.out
  179. start=`current_time`
  180. $PYTHON $REGRTEST_ARGS -u urlfetch >& build/$F
  181. NUM_FAILURES=`count_failures build/$F`
  182. place_summary_first build/$F
  183. update_status "Testing basics ($NUM_FAILURES failures)" "$F" $start
  184. mail_on_failure "basics" build/$F
  185. F=make-test-opt.out
  186. start=`current_time`
  187. $PYTHON -O $REGRTEST_ARGS -u urlfetch >& build/$F
  188. NUM_FAILURES=`count_failures build/$F`
  189. place_summary_first build/$F
  190. update_status "Testing opt ($NUM_FAILURES failures)" "$F" $start
  191. mail_on_failure "opt" build/$F
  192. ## run the tests looking for leaks
  193. F=make-test-refleak.out
  194. start=`current_time`
  195. ## ensure that the reflog exists so the grep doesn't fail
  196. touch $REFLOG
  197. $PYTHON $REGRTEST_ARGS -R 4:3:$REFLOG -u network,urlfetch $LEAKY_SKIPS >& build/$F
  198. LEAK_PAT="($LEAKY_TESTS|sum=0)"
  199. NUM_FAILURES=`egrep -vc "$LEAK_PAT" $REFLOG`
  200. place_summary_first build/$F
  201. update_status "Testing refleaks ($NUM_FAILURES failures)" "$F" $start
  202. mail_on_failure "refleak" $REFLOG "$LEAK_PAT"
  203. ## now try to run all the tests
  204. F=make-testall.out
  205. start=`current_time`
  206. ## skip curses when running from cron since there's no terminal
  207. ## skip sound since it's not setup on the PSF box (/dev/dsp)
  208. $PYTHON $REGRTEST_ARGS -uall -x test_curses test_linuxaudiodev test_ossaudiodev >& build/$F
  209. NUM_FAILURES=`count_failures build/$F`
  210. place_summary_first build/$F
  211. update_status "Testing all except curses and sound ($NUM_FAILURES failures)" "$F" $start
  212. mail_on_failure "all" build/$F
  213. fi
  214. fi
  215. fi
  216. ## make doc
  217. cd $DIR/Doc
  218. F="make-doc.out"
  219. start=`current_time`
  220. # XXX(nnorwitz): For now, keep the code that checks for a conflicted file until
  221. # after the first release of 2.6a1 or 3.0a1. At that point, it will be clear
  222. # if there will be a similar problem with the new doc system.
  223. # Doc/commontex/boilerplate.tex is expected to always have an outstanding
  224. # modification for the date. When a release is cut, a conflict occurs.
  225. # This allows us to detect this problem and not try to build the docs
  226. # which will definitely fail with a conflict.
  227. #CONFLICTED_FILE=commontex/boilerplate.tex
  228. #conflict_count=`grep -c "<<<" $CONFLICTED_FILE`
  229. make clean
  230. conflict_count=0
  231. if [ $conflict_count != 0 ]; then
  232. echo "Conflict detected in $CONFLICTED_FILE. Doc build skipped." > ../build/$F
  233. err=1
  234. else
  235. make checkout update html >& ../build/$F
  236. err=$?
  237. fi
  238. update_status "Making doc" "$F" $start
  239. if [ $err != 0 ]; then
  240. NUM_FAILURES=1
  241. mail_on_failure "doc" ../build/$F
  242. fi
  243. F="make-doc-dist.out"
  244. start=`current_time`
  245. if [ $conflict_count == 0 ]; then
  246. make dist >& ../build/$F
  247. err=$?
  248. fi
  249. update_status "Making downloadable doc" "$F" $start
  250. if [ $err != 0 ]; then
  251. NUM_FAILURES=1
  252. mail_on_failure "doc dist" ../build/$F
  253. fi
  254. echo "</ul>" >> $RESULT_FILE
  255. echo "</body>" >> $RESULT_FILE
  256. echo "</html>" >> $RESULT_FILE
  257. ## copy results
  258. rsync $RSYNC_OPTS build/html/* $REMOTE_SYSTEM:$REMOTE_DIR
  259. rsync $RSYNC_OPTS dist/* $REMOTE_SYSTEM:$REMOTE_DIR_DIST
  260. cd ../build
  261. rsync $RSYNC_OPTS index.html *.out $REMOTE_SYSTEM:$REMOTE_DIR/results/