/src/fusre/service/node/basenode.py
Python | 394 lines | 391 code | 0 blank | 3 comment | 1 complexity | 5a938659df70072b4fab850a6ef622b6 MD5 | raw file
- '''
- Copyright 2011 Adam Pridgen <adam.pridgen@thecoverofnight.com>
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- @author: Adam Pridgen <adam.pridgen@thecoverofnight.com>
- '''
- import json
- from socket import *
- from service.networking.message import Message
- from service.networking.asynchsocket import MessageSocket
- from threading import Thread, Lock, Timer
- from time import sleep
- from service.util.ThreadSafe import List
- from service.util import ThreadSafe
- import traceback
- import sys
- import base64
- import signal
- class BaseNode(object):
- '''
- classdocs
- '''
-
- def __init__(self, server = None, port = 19955, request_spacing = .5,
- consumer_timer = .5, allowed_pending = 10):
- '''
- Constructor
-
- server: host to connect to initially if the crawler is not waiting for a client to connect to it
- port: port of the server to connect too.
- listen_port: port to listen on for new crawler clients
- request_spacing: time interval in seconds between successive requests through tor
- consumer_timer: time interval for polling for new messages.
- allowed_pending: number of requests the client is permitted to service at any given time
- torport: port to run tor on
- torhost: host interface to run tor on.
-
-
- '''
-
- self._msg_socks = {}
- if not server is None and port is None:
- try:
- peer = (server, port)
- self._msg_socks[peer] = self.init_msg_sock(server, port, sock = None)
- except:
- tb = traceback.format_exc(sys.exc_info())
- print("Exception while trying to connect to a: %s"%str(tb))
- raise
-
- self._server_sock = self.get_server_socket(port = listen_port)
-
- self._server_sock.register_handler("accept", self.accept_server)
-
-
- self._pending_jobs = ThreadSafe.List()
- self._running_jobs = {}
- self._running_lock = Lock()
-
- self._allowed_pending = allowed_pending
-
- self._send_handlers = {}
- self._recv_handlers = {}
- self._recv_types = {}
- self._send_types = {}
-
- self._running_jobs = {}
-
- self._handlers = {}
-
- self._adjust_time = Lock()
- self._consumer_timer = None
- self._consumer_time = consumer_timer
- self._continue_consume = False
- self._request_spacing = request_spacing
- # TODO create a logical clock class
- self._logical_clock = 0
- self._clock_lock = threading.RLock()
-
- def update_clock(self, clock_val):
- self._clock_lock.acquire()
- if self._logical_clock < clock_val:
- self._logical_clock = clock_val
- self._clock_lock.release()
- return self._logical_clock
-
- def increment_clock(self):
- self._clock_lock.acquire()
- self._logical_clock += 1
- v = self._logical_clock
- self._clock_lock.release()
- return v
- def get_clock(self):
- return self._logical_clock
- def start_service(self):
- '''
- Start the service
- '''
-
- self.start_consumer()
- self._server_sock.start_accept()
- for peer in self._msg_socks:
- msg_sock = self._msg_socks[peer]
- msg_sock.start_recv()
- msg_sock.start_send()
-
- signal.signal(signal.SIGALRM, self.sigterm_handler)
- def start_consumer(self):
- '''
- Start the consumer thread
- '''
- self._adjust_time.acquire()
- try:
- self._continue_consume = True
- if self._consumer_timer is None:
- self._consumer_timer = Timer(self._consumer_time, self.consume_jobs)
- self._consumer_timer.start()
- finally:
- self._adjust_time.release()
-
- def stop_consumer(self):
- '''
- Stop the consumer thread
- '''
- self._adjust_time.acquire()
- try:
- self._continue_consume = False
- if not self._consumer_timer is None:
- self._consumer_timer.cancel()
- self._consumer_timer = None
- finally:
- self._adjust_time.release()
-
- def check_reset_consumer(self):
- '''
- Reset the polling for the consumer thread
- '''
- self._adjust_time.acquire()
- try:
- if self._continue_consume:
- self._consumer_timer = Timer(self._consumer_time, self.consume_jobs)
- self._consumer_timer.start()
- else:
- self._consumer_timer = None
- finally:
- self._adjust_time.release()
-
-
-
-
- def consume_jobs(self):
- '''
- Handle all the pending jobs upto the number of allowed pending requests. Also maintain a
- strict interval of time between each request
- '''
- #TODO consume the next task
-
- self.check_reset_consumer()
-
- def init_msg_sock(self, host, port, sock = None):
- '''
- Initialize a message socket and set the respective "recv" and "send" handlers. These
- handlers will be called by the socket when a message is recieved or after a message is
- send.
-
- host: host name or ip for the endpoint
- port: port of the endpoint
- sock: if the socket has already been initialized (e.g. client connects to us)
- use this rather than starting a new socket.
- '''
- msg_sock = MessageSocket(host, port, self.certfile, self.keyfile, sock = sock)
- msg_sock.register_handler('recv', self.recv_msg)
- msg_sock.register_handler('send', self.sent_msg)
- return msg_sock
-
-
- def sent_msg(self, msg, instance):
- '''
- handler for after a message is sent by the underlying layer
- msg: Message sent by the lower layer network abstraction
- instance: lower layer network abstraction that performed the operation
- '''
- pass
-
- def get_bound_socket(self, server, port, backlog = 10):
- '''
- create a new listener socket to recieve connections on
- server: name to listen on (note this is not currently configurable by this crawler)
- port: port on which the server can connect to
- backlog: number of sockets allowed to be around while waiting
-
- '''
- s = socket()
- s.bind((server, port))
- s.listen(backlog)
- return s
-
-
- def accept_server(self, sock, peer, instance):
- '''
- handler called when the underlying network inteface accepts a connection.
- sock: client socket that is recieved
- peer: peer that connected to the server
- instance: lower layer network abstraction that performed the operation
- '''
- msg_sock = self.init_msg_sock(peer[0], peer[1], sock)
- msg_sock.start_recv()
- msg_sock.start_send()
- self._msg_socks[peer] = msg_sock
- def recv_msg(self, instance):
- '''
- handler called when the underlying network inteface has a completed message that needs to be
- processed. The handler will parse the message determine the type and pass the message off to the
- handler that handles the particular message type.
-
- instance: lower layer network abstraction that performed the operation
- '''
- while instance.has_recv_msg():
- msg = instance.next_recv_msg()
- msg_type = msg.msg_type()
- src = instance.getpeername()
- if not msg_type in self._recv_handlers:
- print("Dont know how to handle this type of message: %d from %s"%(msg_type, str(instance.getpeername())))
- handler = self._recv_handlers[msg_type]
- handler(src, msg)
- def send(self, peer, msg):
- '''
- Send the given message off to the addressed peer.
-
- peer: endpoint to get the message
- msg: message to send
- '''
- if peer in self._msg_socks:
- self._msg_socks[peer].send_msg(msg, self)
-
- def register_send_handler(self, msg_type, code, handler):
- '''
- Register send handlers for a particular type of message
- msg_type: string name for the message type
- code: byte/integer code for the message type (e.g. network/machine interpretation)
- handler: function that will handle the message.
-
- Parameters that should be expected by the send handler should be: message, instance
- where message is the message being sent and the src message source.
-
- eg. handler(src, message)
- '''
- self._send_handlers[msg_type] = handler
- self._send_handlers[code] = handler
- self._send_types[code] = msg_type
- self._send_types[msg_type] = code
-
- def register_recv_handler(self, msg_type, code, handler):
- '''
- Register recv handlers for a particular type of message
- msg_type: string name for the message type
- code: byte/integer code for the message type (e.g. network/machine interpretation)
- handler: function that will handle the message.
-
- Parameters that should be expected by the send handler should be: src, message
- where message is the message being sent and the src message source.
-
- eg. handler(src, message)
- '''
- self._recv_handlers[msg_type] = handler
- self._recv_handlers[code] = handler
- self._recv_types[code] = msg_type
- self._recv_types[msg_type] = code
-
-
- def handle_msg(self, src, msg):
- '''
- Handler that gets called when a message needs to be processed
- '''
- pass
-
-
- def handle_job_failure(self, job, src, msg, failure_str = "Job failed to get sent"):
- '''
- Handler that gets called when a message failed
- '''
- pass
-
-
- def handle_job_completion(self, job, src, msg):
- '''
- Handler that gets called when a job is completed
- '''
- pass
-
- def add_msg_to_pending(self, src, msg):
- '''
- Add a pending job to the queue without for the consumer thread to process
- '''
- self._pending_jobs.append_ts((src, msg))
-
- def add_job_to_running(self, t, job, msg, src):
- '''
- Add running/completed jobs from the running jobs dictionary in a thread safe manner
- '''
- self._running_lock.acquire()
- self._running_jobs[job] = ((t, job, msg, src))
- self._running_lock.release()
- def remove_job_from_running(self, job):
- '''
- Remove running/completed jobs from the running jobs dictionary in a thread safe manner
- '''
- self._running_lock.acquire()
- if not job in self._running_jobs:
- self._running_lock.release()
- return None, None, None, None
- t, job, msg, src = self._running_jobs[job]
- self._running_lock.release()
- return t, job, msg, src
- def get_server_socket(self, host = '', port = 19995, backlog = 10):
- '''
- Get a server/listener socket for future clients to connect too for web request
- '''
- sock = socket()
- sock.bind((host, port))
- sock.listen(backlog)
- return MessageSocket(host, listen_port, self.certfile,
- self.keyfile, sock = sock,
- is_server = True)
-
- def sigterm_handler(self, signum, frame):
- '''
- Cleanly handle the sigterm signal
- '''
- print("Handling Sigterm Event")
- self.stop_service()
-
- def stop_service(self):
- '''
- Stop all message socket polling threads, the consumer threads, and the tor interface.
- '''
-
- try:
- self.stop_consumer()
- except:
- print "failed to stop consumer thread"
-
- for peer in self._msg_socks.keys():
- try:
- self._msg_socks[peer].shutdown()
- except:
- tb = traceback.format_exc(sys.exc_info())
- print ("failed to shutdown msg socket: %s"%(str( peer)))
- print ("%s"%(tb))
-
-
-
- try:
- self._server_sock.shutdown()
- except:
- tb = traceback.format_exc(sys.exc_info())
- print ("failed to shutdown server socket: %s"%(str( peer)))
- print ("%s"%(tb))