PageRenderTime 25ms CodeModel.GetById 23ms RepoModel.GetById 0ms app.codeStats 0ms

/threading_url.py

https://bitbucket.org/cheng123/mytools
Python | 114 lines | 111 code | 1 blank | 2 comment | 0 complexity | 7681a1d8a64e30ca8b0ec470d1c94e09 MD5 | raw file
    

  1. #encoding: utf-8

    2. 

  2. 

    3. 

  3. import time,math,os,re,urllib,urllib2,cookielib

    4. 

  4. from BeautifulSoup import BeautifulSoup

    5. 

  5. import time

    6. 

  6. """ 自动抓取百度图片 """

    7. 

  7. class BaiduImage:

    8. 

  8.     image_links = []

    9. 

  9.     image_dir = 'image'

    10. 

  10.     current_page = ''

    11. 

  11.     ext_page = ''

    12. 

  12.     image_count = 0

    13. 

  13.     def __init__(self):

    14. 

  14.         self.cj =cookielib.LWPCookieJar()

    15. 

  15.         try:

    16. 

  16.             self.cj.revert('baiduimage.cookie')

    17. 

  17.         except:

    18. 

  18.             None       

    19. 

  19.         self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))

    20. 

  20.         urllib2.install_opener(self.opener)

    21. 

  21.         self.opener.addheaders = [

    22. 

  22.             ("User-agent", "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.9.1) Gecko/20090704 Firefox/3.5"),

    23. 

  23.             ("Accept", "*/*")]

    24. 

  24.  

    25. 

  25.  

    26. 

  26.     """ 得到当前页面中图片的链接地址 """

    27. 

  27.     def get_image_links(self):

    28. 

  28.         try:

    29. 

  29.             html = self.opener.open(self.current_page).read()

    30. 

  30.         except Exception,e:

    31. 

  31.             self.write_log(e)

    32. 

  32.             return

    33. 

  33.              

    34. 

  34.         soup = BeautifulSoup(html)

    35. 

  35.         self.image_links = []

    36. 

  36.         for link in soup.findAll('a',{'href':re.compile('^./img')}):

    37. 

  37.             if 'src=http://' in str(link):

    38. 

  38.                 l = re.findall(r'src=(http://.*)',link['href'])[0]

    39. 

  39.                 self.image_links.append(l)

    40. 

  40.                  

    41. 

  41.                  

    42. 

  42.     """ 得到下一页地址 """

    43. 

  43.     def get_next_page(self):

    44. 

  44.         html = self.opener.open(self.current_page).read()

    45. 

  45.         soup = BeautifulSoup(html)

    46. 

  46.         spans = soup.findAll('span')

    47. 

  47.         for span in spans:

    48. 

  48.             span_html = str(span)

    49. 

  49.             if '下一页' in span_html:

    50. 

  50.                 self.current_page = str('http://wap.baidu.com')+str(BeautifulSoup

    51. 

  51.  

    52. 

  52. (span_html).find('a')['href'])

    53. 

  53.                 self.write_log('Going next page...')

    54. 

  54.                 return               

    55. 

  55.  

    56. 

  56.         self.write_log('This is the latest page')

    57. 

  57.         self.next_page = ''

    58. 

  58.         return False

    59. 

  59.          

    60. 

  60.  

    61. 

  61.     """ 下载self.image_links中的图片 """

    62. 

  62.     def download(self):

    63. 

  63.         if not self.image_links:

    64. 

  64.             return False

    65. 

  65.          

    66. 

  66.         self.write_log('Current page - %s' %self.current_page)

    67. 

  67.  

    68. 

  68.         for link in self.image_links:

    69. 

  69.             try:

    70. 

  70.                 data = urllib.urlopen(link).read()

    71. 

  71.             except Exception,e:

    72. 

  72.                 self.write_log('Connect error:%s' %e)

    73. 

  73.                 return

    74. 

  74.             self.write_log('Downloading... - %s' %link)                

    75. 

  75.             file_name = str(int(time.time()))+'.jpg'

    76. 

  76.             file_path = os.path.join(self.image_dir,file_name)

    77. 

  77.             image = open(file_path,'wb')

    78. 

  78.              

    79. 

  79.             try:                

    80. 

  80.                 image.write(data)

    81. 

  81.             except Exception,e:

    82. 

  82.                 self.write_log('Download faild:%s' %e)

    83. 

  83.             else:

    84. 

  84.                 self.write_log('Download Success!-%s' %link)

    85. 

  85.              

    86. 

  86.             self.image_count += 1

    87. 

  87.             image.close()

    88. 

  88.             del image

    89. 

  89.             time.sleep(2)

    90. 

  90.  

    91. 

  91.     def write_log(self,text):

    92. 

  92.         os.system('cls')

    93. 

  93.         print text

    94. 

  94.         log = open('log.txt','a')

    95. 

  95.         log.write(text)

    96. 

  96.         log.write('\n')

    97. 

  97.         log.close()          

    98. 

  98.  

    99. 

  99.      

    100. 

  100.     """ 给出wap起始页开始下载 """

    101. 

  101.     def run(self,start_page):

    102. 

  102.         self.current_page = start_page

    103. 

  103.  

    104. 

  104.         while True:

    105. 

  105.             # 获取后下载首页图片

    106. 

  106.             self.get_image_links()

    107. 

  107.             self.download()

    108. 

  108.             self.get_next_page()

    109. 

  109.             self.write_log('Image total:%d' %self.image_count)

    110. 

  110.             time.sleep(1)

    111. 

  111.              

    112. 

  112.  

    113. 

  113. app = BaiduImage()

    114. 

  114. app.run(start_page='http://wap.baidu.com/ssid=0/from=0/bd_page_type=1/uid=wiaui_1315707661_2623/pu=sz%40224_220%2Cusm%401/img?tn=bdwis&word=%E8%8B%8D%E4%BA%95%E7%A9%BA&pn=12&dw=w240&bs=176_208&pinf=6_6_0_@bdwis_@%E8%8B%8D%E4%BA%95%E7%A9%BA_@176_208_@w240&sp=&mid=w240')

    115. 

  115.