/twisted/twisted/plugins/bartercast_crawler_plugin.py
Python | 163 lines | 107 code | 29 blank | 27 comment | 10 complexity | 0d5891265d135323f06d6a8df3950847 MD5 | raw file
- """
- Run Dispersy in standalone bartercast crawler mode.
- Crawls BarterCommunity and collects interaction data from peers.
- Temporary approach: all peers will run this service if this works OK. See Github issue #3.
- """
- import os
- import signal
- from twisted.logger import globalLogPublisher
- from Tribler.dispersy.crypto import NoVerifyCrypto, NoCrypto
- # from dispersy.discovery.community import DiscoveryCommunity
- from Tribler.dispersy.dispersy import Dispersy
- from Tribler.dispersy.endpoint import StandaloneEndpoint
- from Tribler.dispersy.exception import CommunityNotFoundException
- from twisted.application.service import IServiceMaker, MultiService
- from twisted.conch import manhole_tap
- from twisted.internet import reactor
- from twisted.plugin import IPlugin
- from twisted.python import usage
- from twisted.python.log import msg
- from twisted.python.threadable import isInIOThread
- from zope.interface import implements
- from Tribler.community.bartercast4.community import BarterCommunityCrawler
- from Tribler.dispersy.tool.clean_observers import clean_twisted_observers
- clean_twisted_observers(globalLogPublisher)
- class BartercastCrawler(Dispersy):
- def __init__(self, endpoint, working_directory, silent=False, crypto=NoVerifyCrypto()):
- super(BartercastCrawler, self).__init__(endpoint, working_directory, u":memory:", crypto)
- # location of persistent storage
- self._persistent_storage_filename = os.path.join(working_directory, "persistent-storage.data")
- self._silent = silent
- self._my_member = None
- def start(self):
- assert isInIOThread()
- if super(BartercastCrawler, self).start():
- self._create_my_member()
- print "loading bartercc as member %s: " % self._my_member
- # self.register_task("unload inactive communities",
- # LoopingCall(self.unload_inactive_communities)).start(COMMUNITY_CLEANUP_INTERVAL)
- self.define_auto_load(BarterCommunityCrawler, self._my_member, (), load=True)
- # self.define_auto_load(TrackerHardKilledCommunity, self._my_member)
- # if not self._silent:
- # self._statistics_looping_call = LoopingCall(self._report_statistics)
- # self._statistics_looping_call.start(300)
- return True
- return False
- def _create_my_member(self):
- # generate a new my-member
- ec = self.crypto.generate_key(u"very-low")
- self._my_member = self.get_member(private_key=self.crypto.key_to_bin(ec))
- @property
- def persistent_storage_filename(self):
- return self._persistent_storage_filename
- def get_community(self, cid, load=False, auto_load=True):
- try:
- return super(BartercastCrawler, self).get_community(cid, True, True)
- except CommunityNotFoundException:
- return BarterCommunityCrawler.init_community(self, self.get_member(mid=cid), self._my_member)
- class Options(usage.Options):
- optFlags = [
- ["profiler" , "P", "use cProfile on the Dispersy thread"],
- ["memory-dump", "d", "use meliae to dump the memory periodically"],
- ["silent" , "s", "Prevent tracker printing to console"],
- ]
- optParameters = [
- ["statedir", "s", "." , "Use an alternate statedir" , str],
- ["ip" , "i", "0.0.0.0" , "Dispersy uses this ip" , str],
- ["port" , "p", 6421 , "Dispersy uses this UDL port" , int],
- ["crypto" , "c", "ECCrypto", "The Crypto object type Dispersy is going to use" , str],
- ["manhole" , "m", 0 , "Enable manhole telnet service listening at the specified port", int],
- ]
- class BartercastCrawlerServiceMaker(object):
- implements(IServiceMaker, IPlugin)
- tapname = "bartercast_crawler"
- description = "A BartercastCommunity statistics crawler"
- options = Options
- def makeService(self, options):
- """
- Construct a dispersy service.
- """
- tracker_service = MultiService()
- tracker_service.setName("Bartercast Crawler")
- # crypto
- if options["crypto"] == 'NoCrypto':
- crypto = NoCrypto()
- else:
- crypto = NoVerifyCrypto()
- container = [None]
- manhole_namespace = {}
- if options["manhole"]:
- port = options["manhole"]
- manhole = manhole_tap.makeService({
- 'namespace': manhole_namespace,
- 'telnetPort': 'tcp:%d:interface=127.0.0.1' % port,
- 'sshPort': None,
- 'passwd': os.path.join(os.path.dirname(__file__), 'passwd'),
- })
- tracker_service.addService(manhole)
- manhole.startService()
- def run():
- # setup
- dispersy = BartercastCrawler(StandaloneEndpoint(options["port"],
- options["ip"]),
- unicode(options["statedir"]),
- bool(options["silent"]),
- crypto)
- container[0] = dispersy
- manhole_namespace['dispersy'] = dispersy
- self._stopping = False
- def signal_handler(sig, frame):
- msg("Received signal '%s' in %s (shutting down)" % (sig, frame))
- if not self._stopping:
- self._stopping = True
- try:
- dispersy.stop()
- except Exception, e:
- msg("Got exception when stopping dispersy: %s" % e)
- reactor.stop()
- signal.signal(signal.SIGINT, signal_handler)
- signal.signal(signal.SIGTERM, signal_handler)
- # start
- print "starting dispersy"
- if not dispersy.start():
- raise RuntimeError("Unable to start Dispersy")
- # wait forever
- reactor.exitCode = 0
- reactor.callWhenRunning(run)
- # TODO: exit code
- return tracker_service
- # Now construct an object which *provides* the relevant interfaces
- # The name of this variable is irrelevant, as long as there is *some*
- # name bound to a provider of IPlugin and IServiceMaker.
- serviceMaker = BartercastCrawlerServiceMaker()