/share/contrib/tmradio.net/hotline/hotline.py
Python | 374 lines | 339 code | 24 blank | 11 comment | 9 complexity | d62e2d4a4767f5d33134dcad96a45f66 MD5 | raw file
- #!/usr/bin/env python
- # vim: set fileencoding=utf-8 tw=0:
- import email
- import email.header
- import email.parser
- import logging
- import logging.handlers
- import os
- import re
- import rfc822
- import subprocess
- import sys
- import tempfile
- import time
- import traceback
- import urllib
- try:
- import imaplib2 as imaplib
- HAVE_IDLE = True
- except ImportError:
- import imaplib
- HAVE_IDLE = False
- import mad
- import mutagen.easyid3
- import yaml
- CONFIG_NAMES = ["~/.config/hotline.yaml", "/etc/hotline.yaml"]
- fn_filter = re.compile('wav|mp3|ogg', re.I)
- mutagen.easyid3.EasyID3.RegisterTXXXKey('ardj', 'ardj')
- def log_error(e, message):
- message += "\n" + traceback.format_exc(e)
- for line in message.strip().split("\n"):
- logging.error(line)
- def config_get(key, default=None):
- """Returns a value from the config file. The file is read on every call,
- which is OK because the traffic is unlikely that heavy, and you get instand
- updates without reloading the daemon."""
- for fn in CONFIG_NAMES:
- fn = os.path.expanduser(fn)
- if os.path.exists(fn):
- data = yaml.load(file(fn, "rb").read())
- return data.get(key, default)
- return default
- def install_syslog():
- """Makes use of the syslog."""
- logger = logging.getLogger()
- logger.setLevel(logging.DEBUG)
- syslog = logging.handlers.SysLogHandler(address="/dev/log")
- syslog.setLevel(logging.DEBUG)
- format_string = "hotline[%(process)d]: %(levelname)s %(message)s"
- formatter = logging.Formatter(format_string)
- syslog.setFormatter(formatter)
- logger.addHandler(syslog)
- def install_file_logger(filename):
- """Adds a custom formatter and a rotating file handler to the default
- logger."""
- folder = os.path.dirname(filename)
- if not os.path.exists(folder) or not os.access(folder, os.W_OK):
- raise Exception("Can't log to %s: no write permissions." % filename)
- max_size = 1000000
- max_count = 5
- logger = logging.getLogger()
- logger.setLevel(logging.DEBUG)
- h = logging.handlers.RotatingFileHandler(filename, maxBytes=max_size, backupCount=max_count)
- h.setFormatter(logging.Formatter('%(asctime)s - %(process)6d - %(levelname)s - %(message)s'))
- h.setLevel(logging.DEBUG)
- logger.addHandler(h)
- def message_has_audio(num, headers):
- """
- match = fn_filter.search(headers)
- if match is None:
- logging.debug("Message does not match: %s" % headers)
- return False
- """
- #logging.debug("Message %s has an audio file." % num)
- return True
- def run(cmd, wait=True):
- cmd.insert(0, "nice")
- cmd.insert(1, "-n15")
- logging.debug("Running a command: %s" % " ".join(cmd))
- tmp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- if wait:
- tmp.wait()
- def transcode(filename, body):
- logging.debug("Transcoding %s to MP3." % filename)
- tmp_name = wav_name = mp3_name = None
- try:
- tmp_name = tempfile.mktemp(suffix=os.path.splitext(filename)[1].lower())
- file(tmp_name, "wb").write(body)
- logging.debug("Wrote data to %s" % tmp_name)
- if tmp_name.endswith(".mp3"):
- mf = mad.MadFile(tmp_name)
- if mf.mode() != mad.MODE_SINGLE_CHANNEL and mf.samplerate() == 44100:
- logging.debug("File %s does not need transcoding." % filename)
- os.unlink(tmp_name)
- return body
- wav_name = tempfile.mktemp(suffix=".wav")
- run(["sox", "-q", tmp_name, "-r", "44100", "-c", "2", "-s", wav_name])
- logging.debug("Transcoded %s to %s" % (tmp_name, wav_name))
- mp3_name = tempfile.mktemp(suffix=".mp3")
- run(["lame", "--quiet", "--preset", "extreme", wav_name, mp3_name])
- run(["mp3gain", "-q", mp3_name])
- logging.debug("Transcoded %s to %s" % (wav_name, mp3_name))
- body = file(mp3_name, "rb").read()
- return body
- finally:
- if tmp_name and os.path.exists(tmp_name):
- os.unlink(tmp_name)
- if wav_name:
- os.unlink(wav_name)
- if mp3_name:
- os.unlink(mp3_name)
- def set_tags(filename, artist, date):
- tags = mutagen.easyid3.EasyID3()
- tags["artist"] = artist
- tags["title"] = time.strftime("%d.%m.%y %H:%M", date)
- tags["ardj"] = "ardj=1;labels=hotline"
- tags.save(filename)
- logging.debug("Wrote ID3 tags to %s" % filename)
- def mask_sender(name, addr, phone):
- if phone:
- phone_map = config_get("phone_map", {})
- return phone_map.get(phone, phone[:-7] + "XXX" + phone[-4:])
- addr_map = config_get("email_map", {})
- if addr in addr_map:
- return addr_map[addr]
- return name
- def process_file(name, body, sender_name, sender_addr, sender_phone, date):
- logging.debug("Incoming file: sender_name=%s sender_addr=%s sender_phone=%s" % (sender_name.encode("utf-8"), sender_addr.encode("utf-8"), sender_phone))
- sender = mask_sender(sender_name, sender_addr, sender_phone)
- logging.debug("%s (%s) sent a file: %s (%u bytes)" % (sender.encode("utf-8"), sender_addr.encode("utf-8"), name, len(body)))
- body = transcode(name, body)
- name = time.strftime(config_get("mp3_file_name", "/tmp/%Y-%m-%d-hotline-%H%M.mp3"), date)
- if os.path.exists(name):
- logging.warning("File %s already exists, skipping." % name)
- return False
- file(name, "wb").write(body)
- logging.info("Message from %s saved as %s" % (sender.encode("utf-8"), name))
- duration = int(mad.MadFile(name).total_time() / 1000)
- if duration < int(config_get("min_duration", 0)):
- logging.warning("Message is too short, skipped.")
- return False
- if duration > int(config_get("max_duration", 3600)):
- logging.warning("Message is too long, skipped.")
- return False
- set_tags(name, sender, date)
- page_name = time.strftime(config_get("page_name", "/tmp/%Y-%m-%d-hotline-%H%M.md"), date)
- if page_name:
- page = u"title: ??????? ?????: %(sender)s\ndate: %(date)s\nlabels: podcast, hotline\nfile: %(url)s\nfilesize: %(size)s\nduration: %(duration)u\n---\nNo description." % {
- "sender": sender,
- "date": time.strftime("%Y-%m-%d %H:%M:%S", date),
- "url": time.strftime(config_get("mp3_file_url", "http://example.com/files/%Y-%m-%d-hotline-%H%M.mp3"), date),
- "size": os.stat(name).st_size,
- "duration": duration,
- }
- page_dir = os.path.dirname(page_name)
- if not os.path.exists(page_dir):
- os.makedirs(page_dir)
- file(page_name, "wb").write(page.encode("utf-8"))
- logging.info("Wrote %s" % page_name)
- return True
- def decode_value(value):
- for part in re.findall("(=(?:\?[^?]+){3}\?=)", value):
- repl = u""
- for _t, _e in email.header.decode_header(part):
- if _e is None:
- repl += unicode(_e)
- else:
- repl += _t.decode(_e)
- value = value.replace(part, repl)
- return value
- def decode_file_name(encoded):
- """File names can contain all sorts of garbage, so we decode it, but only
- use the extension."""
- if encoded is not None:
- ext = decode_value(encoded).split(".")[-1]
- return "tmp." + ext
- def decode_sender(sender):
- """Decode UTF-8 base64 etc headers. decode_header() must be able to do
- this on its own, but due to a bug it fails to process multiline values."""
- sender = decode_value(sender)
- return rfc822.parseaddr(sender.replace("\r\n", ""))
- def get_phone_number(msg):
- """Extracts caller's phone number from the headers."""
- tmp = re.search("(\d+)", msg["X-WUM-FROM"] or "")
- if tmp:
- number = tmp.group(1)
- if number.startswith("8"):
- number = number[1:]
- if not number.startswith("+"):
- number = "+7" + number
- return number
- tmp = msg["X-Asterisk-CallerID"]
- if tmp:
- return tmp
- return None
- def download_message(mail, data):
- msg = email.message_from_string(data)
- logging.debug("Message-ID is %s" % msg["Message-ID"])
- sender_name, sender_addr = decode_sender(msg.get_all("From")[0])
- sender_phone = get_phone_number(msg)
- date = rfc822.parsedate(msg["Date"])
- status = False
- for part in msg.walk():
- if part.get_content_maintype() == "multipart":
- continue
- if part.get("Content-Disposition") is None:
- continue
- name = decode_file_name(part.get_filename())
- if name is None:
- logging.debug("Message part has no name, skipped.")
- continue
- data = part.get_payload(decode=True)
- if process_file(name, data, sender_name, sender_addr, sender_phone, date):
- status = True
- return status
- def check_one_message(mail, num):
- status = False
- logging.debug("Checking message %s." % num)
- result, data = mail.uid("fetch", num, "(BODY)")
- if result != "OK":
- return False
- if message_has_audio(num, data[0]):
- logging.debug("Fetching message %s" % num)
- result, data = mail.uid("fetch", num, "(RFC822)")
- if result == "OK":
- if download_message(mail, data[0][1]):
- status = True
- return status
- def search_messages(mail):
- have_new_messages = False
- logging.debug("Searching for new messages.")
- debug_num = config_get("imap_debug_message")
- if debug_num:
- message_ids = [str(debug_num)]
- else:
- result, data = mail.uid("search", "(UNSEEN)")
- if result != "OK":
- return False
- message_ids = data[0].split()
- for num in message_ids:
- try:
- if check_one_message(mail, num):
- have_new_messages = True
- except Exception, e:
- log_error(e, "Error checking message %s: %s" % (num, e))
- if have_new_messages:
- fn = config_get("postprocessor", "/bin/true")
- run(fn.split(" "), wait=False)
- else:
- logging.debug("No new messages were found.")
- def connect_and_wait():
- server = config_get("imap_server")
- logging.info("Connecting to %s" % server)
- mail = imaplib.IMAP4_SSL(server)
- mail.login(config_get("imap_user"), config_get("imap_password"))
- mail.select(config_get("imap_folder", "INBOX"))
- # Maybe there's something already.
- search_messages(mail)
- if config_get("imap_debug_message"):
- mail.logout()
- exit(0)
- while HAVE_IDLE:
- mail.idle()
- search_messages(mail)
- def loop():
- while True:
- try:
- connect_and_wait()
- except KeyboardInterrupt:
- logging.info("Interrupted by user.")
- return
- except Exception, e:
- log_error(e, "ERROR: %s, restarting in 5 seconds." % e)
- time.sleep(5)
- logging.debug("Sleeping for 60 seconds.")
- time.sleep(60)
- install_syslog()
- install_file_logger("/radio/logs/hotline.log")
- loop()