/wowengine/util/useragents.py

import re

# thanks to django-experiments


ROBOTS_TAGS = """
    baidu gigabot googlebot yandexbot tversity yeti
    msnbot bingbot facebookexternalhit twitterbot twitmunin siteuptime twitterfeed zibb zyborg
    spider crawler bot slurp fetcher fetch proxy perl checker check checkup index indexer
    wordpress python mail.ru apple-pubsub java/1 wget httpclient bsalsa.com
    butterfly larbin w3c_validator ia_archiver bsalsa.com nutch
    screenshot-generator yandexblogs sitetruth.com yandexdirect crawler4j
    webagent.wise-guys.nl intraweb webmaster@livejournal.com linkdex.com
    linkpeek.com appengine-google hourlypress curl twitturls strawberryj.am
    vonchimpenfurlr short-url fetchd urlresolver url_builder http_request
    xmlrpc .php <?php xml-rpc PHP/5 pecl
    movabletype mechanize ruby libwww lynx w3m ostrovok lwp- metauri
    mailchimp.com openwave pagepeeker.com readability rss-harvester quicktime
    seostats scraping safarizator webcollage loadimpact.com
    publisher links webdav websitevila webcolars unrecognized voyager.exe dwnloadmnger
    .. watchmouse.com sniffer bodyground.ru anonymouse.org yandexmarket facebookplatform
    ichiro parser resolver pingadmin ping-admin nmap embedly yahooysmcm pipes heritrix
    duckduckgo.com typhoeus /etc rivva docomo enterprise_search kaz.kz ahrefsbot
    urlsearchhook search.bot google-site user-agent verification
"""

ROBOTS_RE = re.compile('(?:%s)' % ('|'.join(re.escape(tag) for tag in ROBOTS_TAGS.split())))

def is_robot(request):
    return ROBOTS_RE.search(request.user_agent.string.lower())