PageRenderTime 36ms CodeModel.GetById 15ms RepoModel.GetById 0ms app.codeStats 0ms

/testing/tools/grabber/getpages.sh

https://bitbucket.org/Jiten/mozilla-central
Shell | 54 lines | 26 code | 8 blank | 20 comment | 2 complexity | 3db5c5cfae35b90664f8ca6c90bb612a MD5 | raw file
Possible License(s): JSON, Apache-2.0, 0BSD, LGPL-3.0, BSD-2-Clause, AGPL-1.0, MPL-2.0, GPL-2.0, LGPL-2.1, MIT, MPL-2.0-no-copyleft-exception, BSD-3-Clause
  1. #! /bin/bash
  2. # This Source Code Form is subject to the terms of the Mozilla Public
  3. # License, v. 2.0. If a copy of the MPL was not distributed with this
  4. # file, You can obtain one at http://mozilla.org/MPL/2.0/.
  5. #original author: Alice Nodelman
  6. #contributor: Darin Fisher
  7. #
  8. #takes two inputs, $1 = file containing list of web pages of form http://pagename
  9. # $2 = output file where list of index files is dumped - useful for places list of links into scripts
  10. #
  11. # web pages are dropped in directories named for their urls
  12. if [ $# != 2 ]; then
  13. echo 'missing command line arguments'
  14. echo
  15. echo 'usage: getpages.sh inputfile outputfile'
  16. echo ' inputfile: file containing one url per line of the form http://url'
  17. echo ' outputfile: file to be created during execution, contains a list of index files one per url'
  18. exit
  19. fi
  20. # generates the list of files to be cleansed (exclude image files)
  21. # disables any existing call-outs to the live web
  22. # provided by Darin Fisher
  23. cleanse_files() {
  24. find "$1" -type f -a -print0 ! -iname \*.jpg -a ! -iname \*.gif -a ! -iname \*.png -a ! -name \*.bak | xargs -0 perl -pi -e 's/[a-zA-Z0-9_]*.writeln/void/g;' -e 's/[a-zA-Z0-9_]*.write/void/g;' -e 's/[a-zA-Z0-9_]*.open/void/g;' -e 's/"https/"httpsdisabled/gi;' -e 's/"http/"httpdisabled/gi;' -e 's/<object/<objectdisabled/gi;' -e 's/<embed/<embeddisabled/gi;' -e 's/load/loaddisabled/g;'
  25. }
  26. mkdir testpages
  27. cd testpages
  28. for URL in $(cat ../$1); do
  29. #strip the leading http:// from the url
  30. CLEANURL=$(echo $URL | sed -e 's/http:\/\/\(.*\)/\1/')
  31. #create a directory with the cleaned url as the name
  32. echo "grabbing "$CLEANURL
  33. mkdir $CLEANURL
  34. cd $CLEANURL
  35. ../../wget-1.10-css-parser/src/wget -p -k -H -E -erobots=off --no-check-certificate -U "Mozilla/5.0 (firefox)" --restrict-file-names=windows $URL -o outputlog.txt
  36. #figure out where/what the index file for this page is from the wget output log
  37. FILENAME=$(grep "saved" outputlog.txt | head -1 | sed -e "s/.*\`\(.*\)\'.*/\1/")
  38. rm outputlog.txt
  39. cd ..
  40. #do the final cleanup of any dangling urls
  41. #with thanks to Darin Fisher for the code
  42. cleanse_files $CLEANURL
  43. #add the index file link to the list of index files being generated
  44. echo $CLEANURL/$FILENAME >> $2
  45. done
  46. cd ..