/testmode.go

https://code.google.com/p/redwood-filter/ · Go · 135 lines · 112 code · 20 blank · 3 comment · 29 complexity · 089daee5c2dac15f69464d2d0ad003cf MD5 · raw file

  1. package main
  2. import (
  3. "code.google.com/p/go.net/html/charset"
  4. "fmt"
  5. "io/ioutil"
  6. "net/http"
  7. "net/url"
  8. "strings"
  9. )
  10. // support for running "redwood -test http://example.com"
  11. // runURLTest prints debugging information about how the URL and its content would be rated.
  12. func runURLTest(u string) {
  13. URL, err := url.Parse(u)
  14. if err != nil {
  15. fmt.Println("Could not parse the URL.")
  16. return
  17. }
  18. if URL.Scheme == "" {
  19. url2, err := url.Parse("http://" + u)
  20. if err == nil {
  21. URL = url2
  22. }
  23. }
  24. fmt.Println("URL:", URL)
  25. fmt.Println()
  26. sc := scorecard{
  27. tally: URLRules.MatchingRules(URL),
  28. }
  29. sc.calculate("")
  30. if len(sc.tally) == 0 {
  31. fmt.Println("No URL rules match.")
  32. } else {
  33. fmt.Println("The following URL rules match:")
  34. for s, _ := range sc.tally {
  35. fmt.Println(s)
  36. }
  37. }
  38. if len(sc.scores) > 0 {
  39. fmt.Println()
  40. fmt.Println("The request has the following category scores:")
  41. printSortedTally(sc.scores)
  42. }
  43. if len(sc.blocked) > 0 {
  44. fmt.Println()
  45. fmt.Println("The request is blocked by the following categories:")
  46. for _, c := range sc.blocked {
  47. fmt.Println(c)
  48. }
  49. fmt.Println()
  50. fmt.Println("But we'll check the content too anyway.")
  51. }
  52. if changeQuery(URL) {
  53. fmt.Println()
  54. fmt.Println("URL modified to:", URL)
  55. }
  56. fmt.Println()
  57. fmt.Println("Downloading content...")
  58. resp, err := http.Get(URL.String())
  59. if err != nil {
  60. fmt.Println(err)
  61. return
  62. }
  63. defer resp.Body.Close()
  64. fmt.Println()
  65. contentType, action := checkContentType(resp)
  66. switch action {
  67. case ALLOW:
  68. fmt.Println("The content doesn't seem to be text, so not running a phrase scan.")
  69. return
  70. case BLOCK:
  71. fmt.Println("The content has a banned MIME type:", contentType)
  72. return
  73. }
  74. content, err := ioutil.ReadAll(resp.Body)
  75. if err != nil {
  76. fmt.Println("Error while reading response body:", err)
  77. return
  78. }
  79. modified := false
  80. _, cs, _ := charset.DetermineEncoding(content, resp.Header.Get("Content-Type"))
  81. if strings.Contains(contentType, "html") {
  82. modified = pruneContent(URL, &content, cs)
  83. }
  84. if modified {
  85. cs = "utf-8"
  86. fmt.Println("Performed content pruning.")
  87. fmt.Println()
  88. }
  89. scanContent(content, contentType, cs, sc.tally)
  90. sc.calculate("")
  91. if len(sc.tally) == 0 {
  92. fmt.Println("No content phrases match.")
  93. } else {
  94. fmt.Println("The following rules match:")
  95. printSortedTally(stringTally(sc.tally))
  96. }
  97. if len(sc.scores) > 0 {
  98. fmt.Println()
  99. fmt.Println("The response has the following category scores:")
  100. printSortedTally(sc.scores)
  101. }
  102. if len(sc.blocked) > 0 {
  103. fmt.Println()
  104. fmt.Println("The page is blocked by the following categories:")
  105. for _, c := range sc.blocked {
  106. fmt.Println(c)
  107. }
  108. }
  109. }
  110. // printSortedTally prints tally's keys and values in descending order by value.
  111. func printSortedTally(tally map[string]int) {
  112. for _, rule := range sortedKeys(tally) {
  113. fmt.Println(rule, tally[rule])
  114. }
  115. }