/robot/models.py

import logging
from datetime import datetime
from hashlib import sha256

import simplejson
from django.contrib.auth.models import User
from django.contrib.postgres.fields import ArrayField
from django.db import models, IntegrityError
from django_extensions.db.fields.json import JSONField

from common.assertlib import *
from common.enums import SubscriptionStatus
from common.helpers import now

log = logging.getLogger(__name__)


class Item(models.Model):
    hash = models.CharField(max_length=64, primary_key=True)
    service = models.CharField(max_length=30)
    title = models.TextField(null=True, blank=True, default=None)
    description = models.TextField(null=True, blank=True, default=None)
    tags = ArrayField(models.TextField(null=True, blank=True, default=None))
    item_type = models.CharField(max_length=30)
    media_type = models.CharField(max_length=30)
    author = models.TextField()
    author_image = models.TextField(null=True, blank=True, default=None)
    author_image_large = models.TextField(null=True, blank=True, default=None)
    image = models.TextField(null=True, blank=True, default=None)
    image_large = models.TextField(null=True, blank=True, default=None)
    image_small = models.TextField(null=True, blank=True, default=None)
    image_width = models.IntegerField(null=True, blank=True, default=None)
    image_height = models.IntegerField(null=True, blank=True, default=None)
    image_large_width = models.IntegerField(null=True, blank=True, default=None)
    image_large_height = models.IntegerField(null=True, blank=True, default=None)
    image_small_width = models.IntegerField(null=True, blank=True, default=None)
    image_small_height = models.IntegerField(null=True, blank=True, default=None)
    video = models.TextField(null=True, blank=True, default=None)
    video_width = models.IntegerField(null=True, blank=True, default=None)
    video_height = models.IntegerField(null=True, blank=True, default=None)
    video_large = models.TextField(null=True, blank=True, default=None)
    video_large_width = models.IntegerField(null=True, blank=True, default=None)
    video_large_height = models.IntegerField(null=True, blank=True, default=None)
    location = ArrayField(models.FloatField(null=True, blank=True, default=None), size=2)
    location_name = models.TextField(null=True, blank=True, default=None)
    url = models.TextField(null=True, blank=True, default=None)
    url_in = models.TextField(null=True, blank=True, default=None)
    url_ext = models.TextField(null=True, blank=True, default=None)
    item_hash = models.CharField(max_length=64, null=True, blank=True, default=None)
    content_id = models.CharField(max_length=100)
    content_secondary_id = models.CharField(max_length=100, null=True, blank=True, default=None)
    content_created_at = models.DateTimeField()
    content_updated_at = models.DateTimeField(null=True, blank=True, default=None)
    content_order_by_date = models.DateTimeField()
    visibility = models.CharField(max_length=10)
    points = models.IntegerField(null=True, blank=True, default=None)
    comments = models.IntegerField(null=True, blank=True, default=None)
    is_duplicate = models.BooleanField(default=False)
    is_deleted = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True)
    updated_at = models.DateTimeField(auto_now=True)

    class Meta:
        db_table = 'item'

    def __str__(self):
        return 'Model <%s> (%s, %s)' % (self.__class__.__name__, self.service, self.get_hash())

    def get_hash(self):
        content_id = self.content_id if self.content_secondary_id is None else self.content_secondary_id

        return sha256(self.service + str(content_id)).hexdigest()

    def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
        # if not self.does_hash_exist():
        #     super(self.__class__, self).save()
        #     log.debug('Updated -- %s' % str(self))
        #     return

        self.hash = self.get_hash()

        if self.tags is None:
            self.tags = []

        if self.location is None:
            self.location = []

        self.validate()

        try:
            super(self.__class__, self).save()
        except IntegrityError, e:
            log.debug('Duplicate -- %s' % str(self))
            return

        log.debug('Stored -- %s' % str(self))

    def to_json(self, encode=True):
        json = {
            'hash': self.get_hash(),
            'content_id': self.content_id,
            'content_created_at': self.content_created_at,
            'content_order_by_date': self.content_order_by_date,
            'item_type': self.item_type,
            'media_type': self.media_type,
            'service': self.service,
            'author': self.author,
            'visibility': str(self.visibility)
        }

        if self.title:
            json['title'] = self.title
        if self.description:
            json['description'] = self.description
        if self.image:
            json['image'] = self.image
            json['image_width'] = self.image_width
            json['image_height'] = self.image_height
        if self.image_large:
            json['image_large'] = self.image_large
            json['image_large_width'] = self.image_large_width
            json['image_large_height'] = self.image_large_height
        if self.image_small:
            json['image_small'] = self.image_small
            json['image_small_width'] = self.image_small_width
            json['image_small_height'] = self.image_small_height
        if self.video:
            json['video'] = self.video
            json['video_width'] = self.video_width
            json['video_height'] = self.video_height
        if self.video_large:
            json['video_large'] = self.video_large
            json['video_large_width'] = self.video_large_width
            json['video_large_height'] = self.video_large_height
        if self.author_image:
            json['author_image'] = self.author_image
        if self.author_image_large:
            json['author_image_large'] = self.author_image_large
        if self.tags and len(self.tags) > 0:
            json['tags'] = list(self.tags)
        if self.location and len(self.location) > 0:
            json['location'] = self.location
        if self.location_name:
            json['location_name'] = self.location_name
        if self.url:
            json['url'] = self.url
        if self.url_in:
            json['url_in'] = self.url_in
        if self.url_ext:
            json['url_ext'] = self.url_ext

        if encode:
            json['content_created_at'] = str(json['content_created_at'])

            return simplejson.dumps(json)
        return json

    @staticmethod
    def from_json(src):
        """

        :rtype : Item
        :param src: dict
        :return item:
        """

        m = Item()
        m.content_id = src['content_id']
        if 'content_secondary_id' in src:
            m.content_secondary_id = src['content_secondary_id']

        if 'content_order_by_date' in src:
            m.content_order_by_date = src['content_order_by_date']

        m.content_created_at = src['content_created_at']
        m.item_type = src['item_type']
        m.media_type = src['media_type']
        m.service = src['service']
        m.author = src['author']
        m.visibility = src['visibility']
        m.created_at = now()

        if 'content_updated_at' in src:
            m.content_updated_at = src['content_updated_at']
        if 'title' in src:
            m.title = src['title']
        if 'description' in src:
            m.description = src['description']
        if 'image' in src:
            m.image = src['image']
            m.image_width = src['image_width']
            m.image_height = src['image_height']
        if 'image_large' in src:
            m.image_large = src['image_large']
            m.image_large_width = src['image_large_width']
            m.image_large_height = src['image_large_height']
        if 'image_small' in src:
            m.image_small = src['image_small']
            m.image_small_width = src['image_small_width']
            m.image_small_height = src['image_small_height']
        if 'video' in src:
            m.video = src['video']
            m.video_width = src['video_width']
            m.video_height = src['video_height']
        if 'video_large' in src:
            m.video_large = src['video_large']
            m.video_large_width = src['video_large_width']
            m.video_large_height = src['video_large_height']
        if 'author_image' in src:
            m.author_image = src['author_image']
        if 'author_image_large' in src:
            m.author_image_large = src['author_image_large']
        if 'tags' in src:
            m.tags = src['tags']
        if 'url' in src:
            m.url = src['url']
        if 'url_in' in src:
            m.url_in = src['url_in']
        if 'url_ext' in src:
            m.url_ext = src['url_ext']
        if 'location' in src:
            m.location = src['location']
        if 'location_name' in src:
            m.location_name = src['location_name']
        if 'points' in src:
            m.points = src['points']
        if 'comments' in src:
            m.comments = src['comments']

        return m

    def to_es_json(self):
        """

        :rtype dict
        :return:
        """
        obj = self.to_json(False)

        if self.tags:
            obj['tags'] = list(self.tags)
        if self.points:
            obj['points'] = self.points
        if self.comments:
            obj['comments'] = self.comments
        if self.location and len(self.location) == 2:
            obj['location'] = {
                'lat': self.location[0],
                'lon': self.location[1]
            }
        if self.location_name:
            obj['location_name'] = self.location_name

        return obj

    def to_es_update_json(self):
        obj = {
            'updated_at': now()
        }

        if self.points is not None:
            obj['points'] = self.points
        if self.comments is not None:
            obj['comments'] = self.comments

        return obj

    def validate(self):
        assertIsNot(self.content_id, None)
        assertIsInstance(self.created_at, datetime)
        assertIsInstance(self.content_created_at, datetime)

        assertIsNot(self.item_type, None)
        assertIsInstance(str(self.item_type), basestring)
        assertNotEqual(len(str(self.item_type)), 0)

        assertIsNot(self.media_type, None)
        assertIsInstance(str(self.media_type), basestring)
        assertNotEqual(len(str(self.media_type)), 0)

        assertIsInstance(self.service, basestring)
        assertNotEqual(len(unicode(self.service)), 0)
        assertIsInstance(self.author, basestring)
        assertNotEqual(len(unicode(self.author)), 0)
        assertIsNot(self.visibility, None)

    def is_indexable(self):
        # return not settings.TESTING
        return True

    def does_hash_exist(self):
        try:
            Item.objects.get(pk=self.get_hash())

            return True
        except Item.DoesNotExist:
            return False


class ItemRawManager(models.Manager):
    def add_item(self, hash, json):
        item, created = self.get_or_create(hash=hash, defaults={'json': json})
        if not created:
            item.json = json
            item.updated_at = now()
            item.save()


class ItemRaw(models.Model):
    hash = models.CharField(max_length=64, primary_key=True)
    json = JSONField()
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    objects = ItemRawManager()

    class Meta:
        db_table = 'item_raw'


class UserItemManager(models.Manager):
    pass


class UserItem(models.Model):
    user = models.ForeignKey(User, db_index=True)
    item = models.ForeignKey('Item')
    is_read = models.BooleanField(default=False)
    is_glued = models.BooleanField(default=False)
    is_deleted = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    objects = UserItemManager()

    class Meta:
        db_table = 'user_item'
        unique_together = [['user', 'item']]

    @staticmethod
    def add_item(user, item):
        user_item, created = UserItem.objects.get_or_create(user=user, item=item,
                                                            defaults={'is_read': False, 'is_glued': False})
        return user_item

    def mark_as_read(self):
        if self.is_read is False:
            self.is_read = True
            self.save()


class UserItemSubscriptionManager(models.Manager):
    def get_items_to_index(self, batch=10):
        return self.filter(is_indexed=False)[:100]


class UserItemSubscription(models.Model):
    user_item = models.ForeignKey('UserItem')
    user_subscription = models.ForeignKey('UserSubscription')
    is_indexed = models.BooleanField(default=False)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    objects = UserItemSubscriptionManager()

    class Meta:
        db_table = 'user_item_subscription'

    @staticmethod
    def add_item(user_item, sub):
        try:
            user_item_sub, created = UserItemSubscription.objects.get_or_create(user_item=user_item, user_subscription=sub)
        except UserItemSubscription.MultipleObjectsReturned:
            for dup in UserItemSubscription.objects.filter(user_item=user_item, user_subscription=sub)[1:]:
                dup.delete()
            return UserItemSubscription.add_item(user_item, sub)

        return user_item_sub


class UserServiceToken(models.Model):
    profile = models.ForeignKey('UserServiceProfile', null=True, blank=True, default=None, db_index=True)
    scope = models.CharField(max_length=255)
    token = models.CharField(max_length=255)
    token_secret = models.CharField(max_length=255, null=True, blank=True, default=None)
    refresh_token = models.CharField(max_length=255, null=True, blank=True, default=None)
    expiry = models.DateTimeField(null=True, blank=True, default=None)
    is_active = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    class Meta:
        db_table = 'user_service_token'

    def __unicode__(self):
        return u'UserServiceToken (%s, %s)' % (str(self.profile), self.is_active)


class Service(models.Model):
    category = models.ForeignKey('Category')
    name = models.CharField(max_length=100)
    short_name = models.SlugField(max_length=30, unique=True)
    description = models.CharField(max_length=255, null=True, blank=True, default=None)
    is_locked = models.BooleanField(default=False)
    is_visible = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    class Meta:
        db_table = 'service'

    def __unicode__(self):
        return u'%s' % self.name


class UserServiceProfileManager(models.Manager):
    def profile_for_username(self, user, service, username):
        sp, created = self.get_or_create(user=user, service=service, identifier=username, defaults={'friendly_name': username})

        return sp


class UserServiceProfile(models.Model):
    user = models.ForeignKey(User, db_index=True)
    service = models.ForeignKey('Service')
    identifier = models.CharField(max_length=50)
    friendly_name = models.CharField(max_length=50)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    objects = UserServiceProfileManager()

    class Meta:
        db_table = 'user_service_profile'
        unique_together = [
            ['user', 'service', 'identifier']
        ]

    def __unicode__(self):
        return '%s at %s' % (self.friendly_name, str(self.service))


class Category(models.Model):
    upper_category = models.ForeignKey('Category', null=True, blank=True, default=None)
    name = models.CharField(max_length=20)
    description = models.CharField(max_length=255, null=True, blank=True, default=None)
    is_visible = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    class Meta:
        db_table = 'category'
        verbose_name_plural = 'categories'

    def __unicode__(self):
        return self.name


class Source(models.Model):
    service = models.ForeignKey('Service')
    script_name = models.CharField(max_length=100)
    name = models.CharField(max_length=30)
    description = models.CharField(max_length=100, null=True, blank=True, default=None)
    url = models.CharField(max_length=200)
    item_ordering_field = models.CharField(max_length=20, default='content_created_at')
    allow_value = models.BooleanField(default=False)
    value = models.CharField(max_length=100, null=True, blank=True, default=None)
    value_hint = models.CharField(max_length=100, null=True, blank=True, default=None)
    min_refresh = models.IntegerField(default=5 * 60)
    do_auto_subscribe = models.BooleanField(default=False)
    is_locked = models.BooleanField(default=False)
    is_per_user = models.BooleanField(default=True)
    is_active = models.BooleanField(default=True)
    is_public = models.BooleanField(default=True)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    class Meta:
        db_table = 'source'
        unique_together = [
            ['service', 'script_name', 'value']
        ]

    def __unicode__(self):
        return u'%s (%s)' % (self.name, str(self.service))


class SubscriptionManager(models.Manager):
    def find_and_update_ready_to_refresh(self):
        log.debug('Looking for tasks to set due to refresh')

        for sub in self.filter(due_refresh=False, status=str(SubscriptionStatus.READY), is_active=True):
            if sub.last_refresh_at is None:
                sub.due_refresh = True
                sub.save()
                continue

            since_refresh = now() - sub.last_refresh_at

            # check if tokens left
            if sub.no_tokens_left():
                log.debug('No tokens left')

            if since_refresh.total_seconds() > sub.source.min_refresh:
                sub.due_refresh = True
                sub.save()

                log.debug('Set %s as due to refresh' % sub.source.script_name)
                # else:
                #     log.debug('%d < %d' % (since_refresh.total_seconds(), sub.source.min_refresh))

    def next_to_run(self):
        log.debug('Looking for tasks to run')

        tasks = []
        for task in self.filter(due_refresh=True, status=str(SubscriptionStatus.READY), is_active=True):
            if task.has_subscribers():
                task.set_as_enqueued()

                tasks.append(task)

        log.debug('Found %d tasks' % len(tasks))
        return tasks


class Subscription(models.Model):
    user = models.ForeignKey(User, null=True, blank=True, default=None)
    source = models.ForeignKey('Source')
    profile = models.ForeignKey('UserServiceProfile', null=True, blank=True, default=None)
    value = models.CharField(max_length=100, null=True, blank=True, default=None)
    status = models.CharField(max_length=15, default=str(SubscriptionStatus.READY), choices=(
        (str(SubscriptionStatus.READY), 'Ready'),
        (str(SubscriptionStatus.ENQUEUED), 'Enqueued'),
        (str(SubscriptionStatus.IN_PROGRESS), 'In Progress'),
        (str(SubscriptionStatus.FAILED), 'Failed')))
    tokens_used = models.IntegerField(default=0, null=True, blank=True)
    tokens_left = models.IntegerField(default=0, null=True, blank=True)
    tokens_reset_at = models.DateTimeField(null=True, blank=True, default=None)
    overwritten_by_subscription_id = models.IntegerField(null=True, blank=True, default=None)
    due_refresh = models.BooleanField(default=True)
    is_active = models.BooleanField(default=True)
    is_hidden = models.BooleanField(default=False)
    last_refresh_at = models.DateTimeField(null=True, blank=True, default=None)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    objects = SubscriptionManager()

    class Meta:
        db_table = 'subscription'
        index_together = [
            ['due_refresh', 'status', 'is_active']
        ]
        unique_together = [
            ['source', 'value']
        ]

    def __unicode__(self):
        if self.value:
            return u'Subscription of %s (%s)' % (str(self.source), self.value)
        return u'Subscription of %s' % str(self.source)

    def has_subscribers(self):
        return self.subscribers().count() > 0

    def no_tokens_left(self):
        if self.tokens_reset_at is None:
            return False

        return self.tokens_left < 1 and self.tokens_reset_at > now()

    def subscribers(self):
        return UserSubscription.objects.filter(subscription=self.id)

    def add_subscriber(self, user, profile=None):
        if UserSubscription.objects.filter(subscription=self, user=user, profile=profile).count() > 0:
            log.debug('%s was already a subscriber' % str(profile))
            return False

        log.debug('Adding subscriber %s' % str(profile))

        us = UserSubscription()
        us.subscription = self
        us.user = user
        us.profile = profile
        us.save()

        log.debug('%s added as a subscriber' % str(profile))
        return True

    def remove_subscriber(self, user, profile):
        return UserSubscription.objects.filter(subscription=self, user=user, profile=profile).delete()

    def set_as_ready(self):
        self.status = str(SubscriptionStatus.READY)
        self.save()
        log.debug('%s is READY' % str(self))

    def set_as_enqueued(self):
        self.status = str(SubscriptionStatus.ENQUEUED)
        self.due_refresh = False
        self.save()
        log.debug('%s is ENQUEUED' % str(self))

    def set_as_in_progress(self):
        self.status = str(SubscriptionStatus.IN_PROGRESS)
        self.due_refresh = False
        self.save()
        log.debug('%s is IN PROGRESS' % str(self))

    def set_as_failed(self):
        self.status = str(SubscriptionStatus.FAILED)
        self.due_refresh = True
        self.save()
        log.debug('%s has FAILED' % str(self))

    @staticmethod
    def add_subscription(source, user, profile=None, value=None):
        if not source.is_per_user:
            current_sub = Subscription.objects.filter(source=source, value=value)
            if current_sub.count() > 0:
                log.debug('Subscription found')

                s = current_sub[0]
                s.add_subscriber(user, profile)

                return s

        s = Subscription()
        s.user = user
        s.profile = profile
        s.source = source
        s.value = value
        s.save()

        s.add_subscriber(user, profile)

        log.debug('Subscription created')
        return s


class SubscriptionLog(models.Model):
    subscription = models.ForeignKey('Subscription', db_index=True)
    url = models.TextField(null=True, blank=True, default=None)
    code = models.IntegerField()
    msg = models.TextField(null=True, blank=True, default=None)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)

    class Meta:
        db_table = 'subscription_log'
        verbose_name = 'Subscription Log'

    @staticmethod
    def add_log(subscription, code, msg=None, url=None):
        return SubscriptionLog(subscription=subscription, code=code, msg=msg, url=url).save()


class UserSubscription(models.Model):
    subscription = models.ForeignKey('Subscription', db_index=True)
    user = models.ForeignKey(User, db_index=True)
    profile = models.ForeignKey('UserServiceProfile', null=True, blank=True, default=None)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    class Meta:
        db_table = 'user_subscription'
        verbose_name = 'user\'s subscription'
        unique_together = [
            ['subscription', 'user', 'profile']
        ]

    def __unicode__(self):
        return u'User Subscription of %s by %s' % (str(self.subscription), str(self.profile))


class State(models.Model):
    subscription = models.ForeignKey('Subscription')
    value = models.CharField(max_length=255)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)
    updated_at = models.DateTimeField(auto_now=True, editable=False)

    class Meta:
        db_table = 'state'

    def __unicode__(self):
        return u'State of %s' % str(self.subscription)

    @staticmethod
    def for_subscription(subscription):
        """
        :type subscription: Subscription
        :rtype : State
        """
        try:
            state = State.objects.get(subscription=subscription)

            log.debug('Loaded state %s' % str(state))
            return state
        except State.DoesNotExist:
            log.debug('Did not found stored state')
            return State(subscription=subscription, value=None)


class Feedback(models.Model):
    user = models.ForeignKey(User, db_index=True)
    feedback = models.TextField(editable=False)
    created_at = models.DateTimeField(auto_now_add=True, editable=False)

    class Meta:
        db_table = 'feedback'