/WebCrawler/mgstage.py

https://github.com/yoshiko2/AV_Data_Capture · Python · 129 lines · 121 code · 2 blank · 6 comment · 5 complexity · 5cbb3650a3a68442d58c6ab786e8efa7 MD5 · raw file 

    

  1. import sys
    2. 

  2. sys.path.append('../')
    3. 

  3. import re
    4. 

  4. from lxml import etree
    5. 

  5. import json
    6. 

  6. from bs4 import BeautifulSoup
    7. 

  7. from ADC_function import *
    8. 

  8. # import sys
    9. 

  9. # import io
    10. 

  10. # sys.stdout = io.TextIOWrapper(sys.stdout.buffer, errors = 'replace', line_buffering = True)
    11. 

  11. 

  12. def getTitle(a):
    13. 

  13.     try:
    14. 

  14.         html = etree.fromstring(a, etree.HTMLParser())
    15. 

  15.         result = str(html.xpath('//*[@id="center_column"]/div[1]/h1/text()')).strip(" ['']")
    16. 

  16.         return result.replace('/', ',')
    17. 

  17.     except:
    18. 

  18.         return ''
    19. 

  19. def getActor(a): #//*[@id="center_column"]/div[2]/div[1]/div/table/tbody/tr[1]/td/text()
    20. 

  20.     html = etree.fromstring(a, etree.HTMLParser()) #//table/tr[1]/td[1]/text()
    21. 

  21.     result1=str(html.xpath('//th[contains(text(),"出演：")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip('\\n')
    22. 

  22.     result2=str(html.xpath('//th[contains(text(),"出演：")]/../td/text()')).strip(" ['']").strip('\\n    ').strip('\\n')
    23. 

  23.     return str(result1+result2).strip('+').replace("', '",'').replace('"','').replace('/',',')
    24. 

  24. def getStudio(a):
    25. 

  25.     html = etree.fromstring(a, etree.HTMLParser()) #//table/tr[1]/td[1]/text()
    26. 

  26.     result1=str(html.xpath('//th[contains(text(),"メーカー：")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip('\\n')
    27. 

  27.     result2=str(html.xpath('//th[contains(text(),"メーカー：")]/../td/text()')).strip(" ['']").strip('\\n    ').strip('\\n')
    28. 

  28.     return str(result1+result2).strip('+').replace("', '",'').replace('"','')
    29. 

  29. def getRuntime(a):
    30. 

  30.     html = etree.fromstring(a, etree.HTMLParser())  # //table/tr[1]/td[1]/text()
    31. 

  31.     result1 = str(html.xpath('//th[contains(text(),"収録時間：")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip('\\n')
    32. 

  32.     result2 = str(html.xpath('//th[contains(text(),"収録時間：")]/../td/text()')).strip(" ['']").strip('\\n    ').strip('\\n')
    33. 

  33.     return str(result1 + result2).strip('+').rstrip('mi')
    34. 

  34. def getLabel(a):
    35. 

  35.     html = etree.fromstring(a, etree.HTMLParser())  # //table/tr[1]/td[1]/text()
    36. 

  36.     result1 = str(html.xpath('//th[contains(text(),"シリーズ：")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip(
    37. 

  37.         '\\n')
    38. 

  38.     result2 = str(html.xpath('//th[contains(text(),"シリーズ：")]/../td/text()')).strip(" ['']").strip('\\n    ').strip(
    39. 

  39.         '\\n')
    40. 

  40.     return str(result1 + result2).strip('+').replace("', '",'').replace('"','')
    41. 

  41. def getNum(a):
    42. 

  42.     html = etree.fromstring(a, etree.HTMLParser())  # //table/tr[1]/td[1]/text()
    43. 

  43.     result1 = str(html.xpath('//th[contains(text(),"品番：")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip(
    44. 

  44.         '\\n')
    45. 

  45.     result2 = str(html.xpath('//th[contains(text(),"品番：")]/../td/text()')).strip(" ['']").strip('\\n    ').strip(
    46. 

  46.         '\\n')
    47. 

  47.     return str(result1 + result2).strip('+')
    48. 

  48. def getYear(getRelease):
    49. 

  49.     try:
    50. 

  50.         result = str(re.search('\d{4}',getRelease).group())
    51. 

  51.         return result
    52. 

  52.     except:
    53. 

  53.         return getRelease
    54. 

  54. def getRelease(a):
    55. 

  55.     html = etree.fromstring(a, etree.HTMLParser())  # //table/tr[1]/td[1]/text()
    56. 

  56.     result1 = str(html.xpath('//th[contains(text(),"配信開始日：")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip(
    57. 

  57.         '\\n')
    58. 

  58.     result2 = str(html.xpath('//th[contains(text(),"配信開始日：")]/../td/text()')).strip(" ['']").strip('\\n    ').strip(
    59. 

  59.         '\\n')
    60. 

  60.     return str(result1 + result2).strip('+').replace('/','-')
    61. 

  61. def getTag(a):
    62. 

  62.     html = etree.fromstring(a, etree.HTMLParser())  # //table/tr[1]/td[1]/text()
    63. 

  63.     result1 = str(html.xpath('//th[contains(text(),"ジャンル：")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip(
    64. 

  64.         '\\n')
    65. 

  65.     result2 = str(html.xpath('//th[contains(text(),"ジャンル：")]/../td/text()')).strip(" ['']").strip('\\n    ').strip(
    66. 

  66.         '\\n')
    67. 

  67.     result = str(result1 + result2).strip('+').replace("', '\\n",",").replace("', '","").replace('"','').replace(',,','').split(',')
    68. 

  68.     total = []
    69. 

  69.     for i in result:
    70. 

  70.         try:
    71. 

  71.             total.append(translateTag_to_sc(i))
    72. 

  72.         except:
    73. 

  73.             pass
    74. 

  74.     return total
    75. 

  75. def getCover(htmlcode):
    76. 

  76.     html = etree.fromstring(htmlcode, etree.HTMLParser())
    77. 

  77.     result = str(html.xpath('//*[@id="center_column"]/div[1]/div[1]/div/div/h2/img/@src')).strip(" ['']")
    78. 

  78.     #                    /html/body/div[2]/article[2]/div[1]/div[1]/div/div/h2/img/@src
    79. 

  79.     return result
    80. 

  80. def getDirector(a):
    81. 

  81.     html = etree.fromstring(a, etree.HTMLParser())  # //table/tr[1]/td[1]/text()
    82. 

  82.     result1 = str(html.xpath('//th[contains(text(),"シリーズ")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip(
    83. 

  83.         '\\n')
    84. 

  84.     result2 = str(html.xpath('//th[contains(text(),"シリーズ")]/../td/text()')).strip(" ['']").strip('\\n    ').strip(
    85. 

  85.         '\\n')
    86. 

  86.     return str(result1 + result2).strip('+').replace("', '",'').replace('"','')
    87. 

  87. def getOutline(htmlcode):
    88. 

  88.     html = etree.fromstring(htmlcode, etree.HTMLParser())
    89. 

  89.     result = str(html.xpath('//p/text()')).strip(" ['']").replace(u'\\n', '').replace("', '', '", '')
    90. 

  90.     return result
    91. 

  91. def getSeries(a):
    92. 

  92.     html = etree.fromstring(a, etree.HTMLParser())  # //table/tr[1]/td[1]/text()
    93. 

  93.     result1 = str(html.xpath('//th[contains(text(),"シリーズ")]/../td/a/text()')).strip(" ['']").strip('\\n    ').strip(
    94. 

  94.         '\\n')
    95. 

  95.     result2 = str(html.xpath('//th[contains(text(),"シリーズ")]/../td/text()')).strip(" ['']").strip('\\n    ').strip(
    96. 

  96.         '\\n')
    97. 

  97.     return str(result1 + result2).strip('+').replace("', '", '').replace('"', '')
    98. 

  98. def main(number2):
    99. 

  99.     number=number2.upper()
    100. 

  100.     htmlcode=str(get_html('https://www.mgstage.com/product/product_detail/'+str(number)+'/',cookies={'adc':'1'}))
    101. 

  101.     soup = BeautifulSoup(htmlcode, 'lxml')
    102. 

  102.     a = str(soup.find(attrs={'class': 'detail_data'})).replace('\n                                        ','').replace('                                ','').replace('\n                            ','').replace('\n                        ','')
    103. 

  103.     b = str(soup.find(attrs={'id': 'introduction'})).replace('\n                                        ','').replace('                                ','').replace('\n                            ','').replace('\n                        ','')
    104. 

  104.     #print(b)
    105. 

  105.     dic = {
    106. 

  106.         'title': getTitle(htmlcode).replace("\\n",'').replace('        ',''),
    107. 

  107.         'studio': getStudio(a),
    108. 

  108.         'outline': getOutline(b),
    109. 

  109.         'runtime': getRuntime(a),
    110. 

  110.         'director': getDirector(a),
    111. 

  111.         'actor': getActor(a),
    112. 

  112.         'release': getRelease(a),
    113. 

  113.         'number': getNum(a),
    114. 

  114.         'cover': getCover(htmlcode),
    115. 

  115.         'imagecut': 0,
    116. 

  116.         'tag': getTag(a),
    117. 

  117.         'label':getLabel(a),
    118. 

  118.         'year': getYear(getRelease(a)),  # str(re.search('\d{4}',getRelease(a)).group()),
    119. 

  119.         'actor_photo': '',
    120. 

  120.         'website':'https://www.mgstage.com/product/product_detail/'+str(number)+'/',
    121. 

  121.         'source': 'mgstage.py',
    122. 

  122.         'series': getSeries(a),
    123. 

  123.     }
    124. 

  124.     js = json.dumps(dic, ensure_ascii=False, sort_keys=True, indent=4, separators=(',', ':'), )  # .encode('UTF-8')
    125. 

  125.     return js
    126. 

  126.     #print(htmlcode)
    127. 

  127. 

  128. if __name__ == '__main__':
    129. 

  129.     print(main('SIRO-4149'))
    130.