/views.py
Python | 2674 lines | 2231 code | 135 blank | 308 comment | 152 complexity | 7c74b08b2d1d2375c4fd12459027fdd9 MD5 | raw file
Large files files are truncated, but you can click here to view the full file
- """
- :copyright: Copyright 2010, by Kevin Dunn
- :license: BSD, see LICENSE file for details.
- TO ADD:
- ---------
- Add a time-out to comment compiling (5 seconds): then force a return
- Deferred processing of comments using a message queue:
- http://www.turnkeylinux.org/blog/django-celery-rabbitmq
- Why? Because we have to do a repo checkout, update the files, commit
- CAPTCHA
- Use PostgreSQL instead
- Handle the case where commit fails because a user name is not present.
- FUTURE
- ======
- Create a web-interface to approve or reject comments; allowing the comment admin
- to pick various reasons to reject comment and append extra info to the poster.
- Also provide option NOT to send an email at all (simply reject the posting).
- """
- # Standard library imports
- import os, sys, random, subprocess, pickle, re, logging.handlers, datetime
- import smtplib, time, shutil
- from collections import defaultdict, namedtuple
- from StringIO import StringIO
- # Settings for the ucomment application
- from conf import settings as conf
- # Django and Jinja import imports
- from django import forms, template
- from django.shortcuts import render_to_response
- from django.contrib import auth as django_auth
- from django.core import cache as django_cache
- from django.core import serializers
- from django.core.context_processors import csrf
- from django.core.mail import send_mail, BadHeaderError
- from django.http import HttpResponse, HttpResponseRedirect
- from django.core.urlresolvers import reverse as django_reverse
- from django.utils import simplejson # used for XHR returns
- from django.utils import html as django_html # used for clean search results
- from jinja2 import Template # Jinja2 is Sphinx dependency; should be available
- from jinja2.exceptions import TemplateSyntaxError
- # Sphinx import
- from sphinx.util.osutil import ensuredir
- from sphinx.application import Sphinx, SphinxError
- if conf.repo_DVCS_type == 'hg':
- import hgwrapper as dvcs
- dvcs.executable = conf.repo_DVCS_exec
- dvcs.local_repo_physical_dir = conf.local_repo_physical_dir
- # Import the application's models, without knowing the application name.
- models = getattr(__import__(conf.app_dirname, None, None, ['models']),'models')
- # The ucomment directive:
- COMMENT_DIRECTIVE = '.. ucomment::'
- # These words will be removed from any search
- # Taken from: http://www.ranks.nl/resources/stopwords.html
- STOP_WORDS = ['I', 'a', 'an', 'are', 'as', 'at', 'be', 'by', 'com', 'for',
- 'from', 'how', 'in', 'is', 'it', 'of', 'on', 'that', 'the',
- 'this', 'to', 'was', 'what', 'when', 'who', 'will', 'with',
- 'www']
- # Code begins from here
- # ---------------------
- log_file = logging.getLogger('ucomment')
- log_file.setLevel(logging.INFO)
- fh = logging.handlers.RotatingFileHandler(conf.log_filename,
- maxBytes=5000000,
- backupCount=10)
- formatter = logging.Formatter(('%(asctime)s - %(name)s '
- '- %(levelname)s - %(message)s'))
- fh.setFormatter(formatter)
- log_file.addHandler(fh)
- class UcommentError(Exception):
- """ A generic error inside this Django application. Will log the error,
- and email the site administrator.
- This class must be initiated an exception object:
- * UcommentError(exc_object)
- But an optional string can also be provided, to give the log and email
- message some extra information:
- * UcommentError(exc_object, err_msg)
- This will figure out where the error was raised and provide some
- source code lines in the email.
- An alternative way to call this class is to just with a string input
- * UcommentError(err_msg)
- But this will only email and log the given error message string.
- The exception will not be raised again here: it is the calling function's
- choise whether to reraise it (and possibly interrupt the user's experience),
- or whether to just let the application continue.
- """
- def __init__(self, error, extra=''):
- if isinstance(error, Exception):
- exc = sys.exc_info()
- from inspect import getframeinfo
- emsg = 'The error that was raised: ' + str(exc[1]) + '\n\n'
- if extra:
- emsg += 'This additional information was provided: "%s"' % extra
- emsg += '\n\n'
- emsg += self.format_frame(getframeinfo(exc[2], context=5))
- else:
- # Handles an older syntax
- emsg = 'The following error was raised: ' + error.__repr__()
- if isinstance(error, UcommentError) and error.raised:
- return
- self.raised = True # prevent errors with emailing from cycling again.
- log_file.error(emsg)
- alert_system_admin(emsg)
- def format_frame(self, traceback):
- """ Receives a named tuple, ``traceback``, created by the ``inspect``
- module's ``getframeinfo()`` function. Formats it to string output.
- """
- out = '*\tfilename = %s\n' % os.path.abspath(traceback.filename)
- out += '*\tline number = %s\n' % traceback.lineno
- out += '*\tfunction = %s(...)\n' % traceback.function
- out += '*\tsource code where error occurred:\n'
- for idx, line in enumerate(traceback.code_context):
- out += '\t\t%s' % line.rstrip()
- if idx == traceback.index:
- out += ' <--- error occurred here\n'
- else:
- out += '\n'
- return out + '\n'
- def create_codes_ID(num):
- """
- Creates a new comment identifier; these appear in the source code for
- each page; they must be short, not confusing, and low chance of collisions.
- Intentionally does not include "i", "I", "l" "D", "o", "O", "Z", "0" to
- avoid visual confusion with similar-looking characters. We (ab)use this
- fact to create the orphan comment reference with name = '_ORFN_'.
- 53 characters, N=4 combinations = 53^4 = many comment ID's
- """
- valid_letters = 'abcdefghjkmnpqrstuvwxyzABCEFGHJKLMNPQRSTUVWXY23456789'
- return ''.join([random.choice(valid_letters) for i in range(num)])
- def convert_web_name_to_link_name(page_name, prefix=''):
- """
- Converts the web page name over to the ``link_name`` used in the Django
- model for ``Page``.
- If a prefix is provided (e.g. the fully web address), then this is stripped
- off first, then the rest is used as the page_name.
- """
- if prefix:
- page_name = page_name.split(prefix)[1]
- if '?' in page_name:
- page_name = page_name.split('?')[0]
- if conf.url_views_prefix:
- return page_name.split('/'+conf.url_views_prefix+'/')[1].rstrip('/')
- else:
- return page_name.lstrip('/').rstrip('/')
- # TODO(KGD): remove ``conf.url_views_prefix``, and the need for this
- # function. Can we not use the reverse(..) function?
- def get_site_url(request, add_path=True, add_views_prefix=False):
- """
- Utility function: returns the URL from which this Django application is
- served. E.g. when receiving a Django ``request`` object from the user
- on comment submission, their URL might be:
- https://site.example.com/document/_submit-comment/
- >>> get_site_url(request)
- 'https://site.example.com/document/_submit-comment/'
- """
- # TODO(KGD): Consider using ``request.build_absolute_uri()`` instead
- out = 'http://'
- if request.is_secure():
- out = 'https://'
- out += request.get_host()
- if add_path:
- out += request.path
- if add_views_prefix:
- if conf.url_views_prefix:
- out += '/' + conf.url_views_prefix + '/'
- else:
- out += '/'
- return out
- def get_IP_address(request):
- """
- Returns the visitor's IP address as a string.
- """
- # Catchs the case when the user is on a proxy
- try:
- ip = request.META['HTTP_X_FORWARDED_FOR']
- except KeyError:
- ip = ''
- else:
- # HTTP_X_FORWARDED_FOR is a comma-separated list; take first IP:
- ip = ip.split(',')[0]
- if ip == '' or ip.lower() == 'unkown':
- ip = request.META['REMOTE_ADDR'] # User is not on a proxy
- return ip
- # Comment preview and submission functions
- # ----------------------------------------
- class CommentForm(forms.Form):
- """ Comment form as seen on the server """
- email = forms.EmailField(required=False)
- comment_raw = forms.CharField(min_length=conf.comment_min_length,
- max_length=conf.comment_max_length)
- def valid_form(p_email, comment_raw):
- """ Verifies if a valid form was filled out.
- Returns an empty string if the form is valid.
- Returns a string, containing an HTML error message if the form is not valid.
- """
- # Ignore empty email addresses; email field is optional anyway, but we do
- # want to alert the user if the email is invalid.
- if p_email.strip() == '':
- p_email = 'no.email@example.com'
- user_form = CommentForm({'email': p_email,
- 'comment_raw': comment_raw})
- if not user_form.is_valid():
- error_dict = user_form.errors
- errors = ['<ul class="ucomment-error">']
- if 'email' in error_dict:
- errors.append(('<li> Your email address is not in the correct '
- 'format.</li>'))
- if 'comment_raw' in error_dict:
- errors.append(('<li> Comments must have between %i and %i '
- 'characters.</li>' % (conf.comment_min_length,
- conf.comment_max_length)))
- errors.append('</ul>')
- log_file.info('Returning with these errors:' + str(errors))
- return ''.join(errors)
- else:
- return ''
- def initial_comment_check(request):
- """
- Provides a preliminary check of the comment submission.
- * Must be a POST request not at a GET request.
- * Must have a valid email address.
- * Comment length must be appropriate (see conf/settings.py file).
- """
- if request.method == 'POST':
- c_comment_RST = request.POST['comment']
- p_email = request.POST['email']
- errors = valid_form(p_email, c_comment_RST)
- if errors:
- web_response = HttpResponse(errors, status=200)
- web_response['Ucomment'] = 'Preview-Invalid input'
- return False, web_response
- else:
- return True, c_comment_RST
- elif request.method == 'GET':
- return False, HttpResponse('N/A', status=404)
- elif request.method == 'OPTIONS':
- # Handles a Firefox probe before the POST request is received.
- web_response = HttpResponse(status=404)
- web_response['Access-Control-Allow-Origin'] = '*'
- return False, web_response
- else:
- log_file.warn(request.method + ' received; not handled; return 400.')
- return False, HttpResponse(status=400)
- def preview_comment(request):
- """
- User has clicked the "Preview comment" button in browser. Using XHR, the
- comment is POSTed here, extracted, and compiled.
- """
- # ``Success``: means the submission was validated, or was processed.
- # If it failed, then ``response`` will contain an appropriate HttpResponse
- # that can be returned right away.
- success, response = initial_comment_check(request)
- if success:
- web_response = HttpResponse(status=200)
- try:
- compiled_comment_HTML = compile_comment(response)
- web_response['Ucomment'] = 'Preview-OK'
- except Exception as err:
- # Should an error occur while commenting, log it, but respond to
- # the user.
- UcommentError(err, ('An exception occurred while generating a '
- 'comment preview for the user.'))
- compiled_comment_HTML = ('<p>An error occurred while processing '
- 'your comment. The error has been '
- 'reported to the website administrator.'
- '<p>UTC time of error: %s' % \
- datetime.datetime.utcnow().ctime())
- web_response['Ucomment'] = 'Preview-Exception'
- log_file.info('COMPILE: from IP=%s; comment: "%s"' % \
- (get_IP_address(request), response))
- web_response.write(compiled_comment_HTML)
- return web_response
- else:
- return response
- def compile_comment(comment):
- """
- First scans the ``comment`` string, then compiles the RST to HTML.
- """
- # The Javascript XHR request have a timeout value (set to 5 seconds).
- # set a timer on the compile time? If more than 5 seconds to
- # compile, then log the comment, return a response back to the user.
- start_time = time.time()
- comment = compile_RST_to_HTML(comment)
- end_time = time.time()
- if (end_time-start_time) > 3:
- log_file.warning(('Comment compile time exceeded 3 seconds; server'
- 'load too high?'))
- return comment
- def call_sphinx_to_compile(working_dir):
- """
- Changes to the ``working_dir`` directory and compiles the RST files to
- pickle files, according to settings in the conf.py file.
- Returns nothing, but logs if an error occurred.
- """
- build_dir = os.path.abspath(working_dir + os.sep + '_build')
- ensuredir(working_dir)
- ensuredir(build_dir)
- status = StringIO()
- warning = StringIO()
- try:
- app = Sphinx(srcdir=working_dir, confdir=working_dir,
- outdir = build_dir + os.sep + 'pickle',
- doctreedir = build_dir + os.sep + 'doctrees',
- buildername = 'pickle',
- status = status,
- warning = warning,
- freshenv = True,
- warningiserror = False,
- tags = [])
- # Call the ``pickle`` builder
- app.build()
- except SphinxError as e:
- if warning.tell():
- warning.seek(0)
- for line in warning.readlines():
- log_file.warn('COMMENT: ' + line)
- msg = ('Sphinx error occurred when compiling comment (error type = %s): '
- '%s' % (e.category, str(e)))
- UcommentError(err, msg)
- if app.statuscode == 0:
- log_file.info("COMMENT: Successfully compiled the reader's comment.")
- else:
- log_file.error("COMMENT: Non-zero status code when compiling.")
- def convert_raw_RST(raw_RST):
- """
- Performs any sanitizing of the user's input.
- Currently performs:
- * converts '\\' to '\\\\': i.e. single slash converted to double-slash,
- because Sphinx converts is back to a single slash
- """
- out = raw_RST.replace('\\', '\\\\')
- # You can perform any other filtering here, if required.
- return out
- def compile_RST_to_HTML(raw_RST):
- """ Compiles the RST string, ``raw_RST`, to HTML. Performs no
- further checking on the RST string.
- If it is a comment, then we don't modify the HTML with extra class info.
- But we do filter comments to disable hyperlinks.
- Also copy over generated MATH media to the correct directory on the server.
- """
- ensuredir(conf.comment_compile_area)
- modified_RST = convert_raw_RST(raw_RST)
- with open(conf.comment_compile_area + os.sep + 'index.rst', 'w') as fhand:
- fhand.write(modified_RST)
- try:
- conf_file = conf.comment_compile_area + os.sep + 'conf.py'
- f = file(conf_file, 'r')
- except IOError:
- # Store a fresh copy of the "conf.py" file, found in
- # ../sphinx-extensions/ucomment-conf.py; copy it to comment destination.
- this_file = os.path.abspath(__file__).rstrip(os.sep)
- parent = this_file[0:this_file.rfind(os.sep)]
- src = os.sep.join([parent, 'sphinx-extensions', 'ucomment-conf.py'])
- shutil.copyfile(src, conf.comment_compile_area + os.sep + 'conf.py')
- else:
- f.close()
- # Compile the comment
- call_sphinx_to_compile(conf.comment_compile_area)
- pickle_f = ''.join([conf.comment_compile_area, os.sep, '_build', os.sep,
- 'pickle', os.sep, 'index.fpickle'])
- with open(pickle_f, 'r') as fhand:
- obj = pickle.load(fhand)
- html_body = obj['body'].encode('utf-8')
- # Any equations in the HTML? Transfer these images to the media directory
- # and rewrite the URL's in the HTML.
- return transfer_html_media(html_body)
- def transfer_html_media(html_body):
- """
- Any media files referred to in the HTML comment are transferred to a
- sub-directory on the webserver.
- The links are rewritten to refer to the updated location.
- """
- mathdir = ''.join([conf.comment_compile_area, os.sep, '_build', os.sep,
- 'pickle', os.sep, '_images', os.sep, 'math', os.sep])
- ensuredir(mathdir)
- dst_dir = conf.MEDIA_ROOT + 'comments' + os.sep
- ensuredir(dst_dir)
- for mathfile in os.listdir(mathdir):
- shutil.copyfile(mathdir + mathfile, dst_dir + mathfile)
- src_prefix = 'src="'
- math_prefix = '_images' + os.sep + 'math' + os.sep
- replacement_text = ''.join([src_prefix, conf.MEDIA_URL,
- 'comments', os.sep])
- html_body = re.sub(src_prefix + math_prefix, replacement_text, html_body)
- return html_body
- def create_poster(request):
- """
- Creates a new ``CommentPoster`` object from a web submission, ``request``.
- """
- p_name = request.POST['name'].strip() or 'Anonymous contributor'
- p_email = request.POST['email'].strip()
- # The default (unchecked box) is for opt-in = False
- p_opted_in = True
- try:
- # Fails if unchecked (default); succeeds if checked: caught in "else"
- p_opted_in = request.POST['updates'] == 'get_updates'
- except KeyError:
- p_opted_in = False
- # Get the poster entry, or create a new one. Always create a new poster
- # entry for anonymous posters.
- c_IP_address = get_IP_address(request)
- c_UA_string = request.META.get('HTTP_USER_AGENT', '')[0:499]
- p_default = {'name' : p_name,
- 'long_name': p_name + '__' + c_IP_address + '__' + c_UA_string,
- 'email': p_email,
- 'number_of_approved_comments': 0,
- 'avatar_link': '', # add this functionality later?
- 'auto_approve_comments': False,
- 'opted_in': False}
- if p_email:
- if p_name.lower() not in ('anonymous', 'anonymous contributor', ''):
- p_default['long_name'] = p_name
- poster, _ = models.CommentPoster.objects.get_or_create(email=p_email,
- defaults=p_default)
- poster.opted_in = poster.opted_in or p_opted_in
- poster.save()
- else:
- poster, _ = models.CommentPoster.objects.get_or_create(\
- defaults=p_default)
- # Change settings for all posters:
- if poster.number_of_approved_comments >= conf.number_before_auto_approval:
- poster.auto_approve_comments = True
- poster.number_of_approved_comments += 1
- poster.save()
- log_file.info('POSTER: Created/updated poster: ' + str(poster))
- return poster
- def submit_and_store_comment(request):
- """
- The user has typed in a comment and previewed it. Now store it and queue
- it for approval.
- ``Comment`` objects have 3 ForeignKeys which must already exist prior:
- * ``page``: the page name on which the comment appears
- * ``poster``: an object representing the person making the comment
- * ``reference``: a comment reference that facilitates making the comment
- """
- start_time = time.time()
- # Response back to user, if everything goes OK
- response = HttpResponse(status=200)
- response['Ucomment'] = 'Submission-OK'
- try:
- html_template = Template(conf.once_submitted_HTML_template)
- except TemplateSyntaxError as err:
- # Log the error, but don't disrupt the response to the user.
- html_template = Template('Thank you for your submission.')
- UcommentError(err, "Error in 'once_submitted_HTML_template'.")
- # Note about variable names: ``p_`` prefix refers to poster objects in the
- # database, while ``c_`` = refers to comment objects.
- # ForeignKey: comment object (setup only; will be created at the end)
- # --------------------------
- success, c_comment_RST = initial_comment_check(request)
- if not success:
- return c_comment_RST
- else:
- c_comment_HTML = compile_comment(c_comment_RST)
- # Only get the comment reference via its root:
- ref = models.CommentReference.objects.filter(\
- comment_root=request.POST['comment_root'])
- if len(ref):
- c_reference = ref[0]
- else:
- # One possibility to consider is to create the comment reference
- # right here. However, it is quite hard to do this properly, because
- # do not know all the field properties for a CommentReference object:
- # such as the line number, page_link_name, node_type, and others.
- # This will only occur in the exceptional case when the document
- # has been republished, and the user still has a previous version in
- # their browser. Hence the page reload request.
- response.write(('<p>A minor error occurred while processing your '
- 'comment.<p>The only way to correct it is to reload '
- 'the page you are on, and to resubmit your comment. '
- '<p>Sorry for the inconvenience.'))
- log_file.warn('COMMENT: User posting a comment from an older page.')
- return response
- # Below the comment reference appears an unique node:
- used_nodes = []
- c_node = create_codes_ID(conf.short_node_length)
- for comment in c_reference.comment_set.all():
- used_nodes.append(comment.node)
- while c_node in used_nodes:
- c_node = create_codes_ID(conf.short_node_length)
- # ForeignKey: page object; get the page on which the comment appears
- # -----------------------
- link_name = convert_web_name_to_link_name(request.POST['page_name'])
- c_page = models.Page.objects.filter(link_name=link_name)[0]
- # ForeignKey: comment poster objects
- # -----------------------------------
- poster = create_poster(request)
- # We can update the response as soon as we have created the poster object
- response.write(html_template.render(settings=conf, poster=poster))
- if poster.number_of_approved_comments >= conf.number_before_auto_approval:
- c_is_approved = True
- c_node_for_RST = c_node
- else:
- c_node_for_RST = c_node + '*' # indicates comment is not approved yet
- # Do all the work here of adding the comment to the RST sources
- revision_changeset, c_root = commit_comment_to_sources(\
- c_reference,
- c_node_for_RST,
- update_RST_with_comment)
- # NOTE: the line numbers for any comment references that might appear below
- # the current comment reference will be incorrect - they will be too
- # low. However, their line numbers will be rectified once the
- # document is republished (comment references are updated).
- # An error occurred:
- if revision_changeset == False:
- # Technically the submission is NOT OK, but the comment admin has been
- # emailed about the problem and can manually enter the comment into
- # the database and RST source files.
- return response
- # Update the ``comment_root_is_used`` field in the comment reference, since
- # this root can never be used again.
- c_reference.comment_root_is_used = True
- # Also update the changeset information. In the future we will update
- # comments for this node from this newer repository.
- c_reference.revision_changeset = revision_changeset
- c_reference.save()
- # Create the comment object
- c_datetime_submitted = c_datetime_approved = datetime.datetime.now()
- c_IP_address = get_IP_address(request)
- c_UA_string = request.META.get('HTTP_USER_AGENT', '')[0:499]
- c_approval_code = create_codes_ID(conf.approval_code_length)
- c_rejection_code = create_codes_ID(conf.approval_code_length)
- c_is_approved = c_is_rejected = False
- # For now all comments have no parent (they are descendents of the comment
- # root). Later, perhaps, we can add threading functionality, so that users
- # can respond to previous comments. Then the parent of a new comment will
- # be given by: ``c_root + ':' + parent_comment.node``
- c_parent = c_root
- the_comment, _ = models.Comment.objects.get_or_create(
- page = c_page,
- poster = poster,
- reference = c_reference,
- node = c_node,
- parent = c_parent,
- UA_string = c_UA_string,
- IP_address = c_IP_address,
- datetime_submitted = c_datetime_submitted,
- datetime_approved = c_datetime_approved,
- approval_code = c_approval_code,
- rejection_code = c_rejection_code,
- comment_HTML = c_comment_HTML,
- comment_RST = c_comment_RST,
- is_rejected = c_is_rejected,
- is_approved = c_is_approved)
- log_file.info('COMMENT: Submitted comment now saved in the database.')
- # Send emails to the poster and comment admin regarding the new comment
- # TODO(KGD): queue it
- emails_after_submission(poster, the_comment, request)
- total_time = str(round(time.time() - start_time, 1))
- log_file.info(('COMMENT: Emails to poster and admin sent successfully; '
- "returning response back to user's browser. Total time to "
- ' process comment = %s secs.') % total_time)
- return response
- def approve_reject_comment(request, code):
- """
- Either approves or rejects the comment, depending on the code received.
- Approved comments:
- - The # part after the comment node is removed in the RST file
- - The comment is marked as approved in the database and will appear on
- the next page refresh.
- - The poster is notified by email (if an email was supplied)
- Rejected comments:
- - The # part after the comment node is changed to a * in the RST file
- - The comment is marked as rejected in the database.
- - The poster is notified by email (if an email was supplied)
- """
- # Response back to user, if everything goes OK
- response = HttpResponse(status=200)
- approve = models.Comment.objects.filter(approval_code=code)
- reject = models.Comment.objects.filter(rejection_code=code)
- # Settings used to approve the comment: we remove the '*'
- if len(approve) == 1:
- verb = 'approved'
- symbol = '\*' # escaped, because it will be used in a regular expressn
- replace = ''
- comment = approve[0]
- comment.is_approved = True
- comment.is_rejected = False
- email_func = email_poster_approved
- # Settings used to reject the comment: we change the '*' to a '#'
- elif len(reject) == 1:
- verb = 'rejected'
- symbol = '\*'
- replace = '#*'
- comment = reject[0]
- comment.is_approved = False
- comment.is_rejected = True
- email_func = email_poster_rejected
- # Bad approve/reject code given: don't mention anything; just return a 404.
- else:
- return HttpResponse('', status=404)
- revision_changeset, _ = commit_comment_to_sources(comment.reference,
- comment.node,
- update_RST_comment_status,
- additional={'search': symbol,
- 'replace': replace})
- if revision_changeset == False:
- # An error occurred while committing the comment. An email has already
- # been sent. Return a message to the user:
- response.write(('An error occurred while approving/rejecting the '
- 'comment. Please check the log files and/or email for '
- 'the site administrator.'))
- return response
- comment.reference.comment_root_is_used = True
- # Also update the changeset information. In the future we will update
- # comments for this node from this newer repository.
- comment.reference.revision_changeset = revision_changeset
- comment.reference.save()
- if verb == 'approved':
- comment.poster.number_of_approved_comments += 1
- elif verb == 'rejected':
- comment.poster.number_of_approved_comments -= 1
- comment.poster.number_of_approved_comments = max(0,
- comment.poster.number_of_approved_comments)
- comment.poster.save()
- comment.datetime_approved = datetime.datetime.now()
- comment.save()
- # Remove the comment count cache for the page on which this comment appears
- cache_key = 'counts_for__' + comment.page.link_name
- django_cache.cache.delete(cache_key)
- # Send an email the comment poster: rejected or approved
- email_func(comment.poster, comment)
- approve_reject_template = Template(('<pre>'
- 'The comment was {{action}}.\n\n'
- '\t* Comment root = {{reference.comment_root}}\n'
- '\t* Comment node = {{comment.node}}\n'
- '\t* At line number = {{reference.line_number}}\n'
- '\t* In file name = {{filename}}\n'
- '\t* Committed as changeset = {{changeset}}\n\n</pre>'))
- output = approve_reject_template.render(action=verb.upper(),
- reference = comment.reference,
- comment = comment,
- filename = os.path.split(\
- comment.reference.file_name)[1],
- changeset = revision_changeset)
- response.write(output)
- return response
- # Repository manipulation functions
- # ---------------------------------
- def update_local_repo(rev='tip'):
- """
- Updates the local repository from the remote repository and must be used
- before performing any write operations on files in the repo.
- If the local repository does not exist, it will first create a full clone
- from the remote repository.
- Then it does the equivalent of:
- * hg pull (pulls in changes from the remote repository)
- * hg update (takes our repo up to tip)
- * hg merge (merges any changes that might be required)
- Then if the optional input ``rev`` is provided, it will revert the
- repository to that revision, given by a string, containing the hexadecimal
- indicator for the required revision.
- This function returns the hexadecimal changeset for the local repo as
- it has been left after all this activity.
- """
- # First check if the local repo exists; if not, create a clone from
- # the remote repo.
- try:
- try:
- ensuredir(conf.local_repo_physical_dir)
- except OSError as err:
- msg = ('The local repository location does not exist, or cannot '
- 'be created.')
- raise UcommentError(err, msg)
- hex_str = dvcs.get_revision_info()
- except dvcs.DVCSError:
- try:
- dvcs.clone_repo(conf.remote_repo_URL, conf.local_repo_URL)
- except dvcs.DVCSError as error_remote:
- msg = ('The remote repository does not exist, or is '
- 'badly specified in the settings file.')
- raise UcommentError(error_remote, msg)
- log_file.info('Created a clone of the remote repo in the local path')
- # Update the local repository to rev='tip' from the source repo first
- try:
- dvcs.pull_update_and_merge()
- except dvcs.DVCSError as err:
- raise UcommentError(err, 'Repository update and merge error')
- hex_str = dvcs.get_revision_info()
- if rev != 'tip' and isinstance(rev, basestring):
- hex_str = dvcs.check_out(rev=rev)
- return hex_str
- def commit_to_repo_and_push(commit_message):
- """
- Use this after performing any write operations on files.
- Commits to the local repository; pushes updates from the local repository
- to the remote repository.
- Optionally, it will also update the remote repository to the tip (the
- default is not to do this).
- Returns the changeset code for the local repo on completion.
- """
- hex_str = dvcs.commit_and_push_updates(commit_message)
- # Merge failed! Log it and email the site admin
- if not(hex_str):
- raise UcommentError(('Repo merging failed (conflicts?) when trying to '
- 'commit. Commit message was: %s' % commit_message))
- # Check that changeset and revision matches the remote repo numbers
- return hex_str
- def commit_comment_to_sources(reference, node, func, additional=None):
- """
- Commits or updates a comment in the RST sources.
- ``reference``: is a comment reference object from the database and tells
- us how and where to add the comment
- ``node``: is the commment noded (string) that is added to the RST source.
- ``func``: does the work of either adding or updating the RST sources.
- ``additional``: named keywords and values in a dict that will be passed
- to ``func``.
- On successful completion it will return:
- ``revision_changeset``: string identifier for the updated repository
- ``comment_root``: a string of the comment root that was added/updated
- """
- # This part is sensitive to errors occurring when writing to the
- # RST source files.
- try:
- # Get the RST file to the revision required for adding the comment:
- hex_str = update_local_repo(reference.revision_changeset)
- f_handle = file(reference.file_name, 'r')
- RST_source = f_handle.readlines()
- f_handle.close()
- # Add the comment to the RST source; send the comment reference
- # which has all the necessary input information in it.
- try:
- if additional == None:
- additional = {}
- c_root = func(comment_ref = reference,
- comment_node = node,
- RST_source = RST_source, **additional)
- except Exception as err:
- # will be caught in outer try-except
- # TODO(KGD): test that this works as expected: what happens after?
- raise UcommentError(err, ('General error while adding or updating '
- 'comment in the RST sources.'))
- # Write the update list of strings, RST_source, back to the file
- f_handle = file(reference.file_name, 'w')
- f_handle.writelines(RST_source)
- f_handle.close()
- short_filename = os.path.split(reference.file_name)[1]
- commit_message = ('COMMIT: Automatic comment [comment_root=%s, '
- 'node=%s, line=%s, file=%s]; repo_id=%s') % \
- (c_root, node, str(reference.line_number),
- short_filename, hex_str)
- hex_str = commit_to_repo_and_push(commit_message)
- log_file.info(commit_message)
- return hex_str, c_root
- except (UcommentError, dvcs.DVCSError) as err:
- UcommentError(err)
- return False, False
- def update_RST_with_comment(comment_ref, comment_node, RST_source):
- """
- Appends the ``comment_node`` string (usually a 2-character string), to the
- appropriate line in the RST_source (a list of strings).
- ``comment_ref`` provides the line number and node type which will be
- commented. We always use an existing comment root, if there is one,
- otherwise we create a new comment root according to these rules:
- Paragraphs, titles, literal_block (source code blocks), tables and figures
- will have their ucomment appended in the second blank line after the node.
- List items (bullet points and ordered lists) will have their ucomment
- appended in the very next line, always.
- In all cases, an existing ucomment directive for that node will be searched
- for and added to.
- The simplest example possible: comment_node='2s' and comment_root='sR4fa4':
- RST_source before (1 line)
- Here is a paragraph of text.
- RST_source after (3 lines):
- Here is a paragraph of text.
- .. ucomment:: sR4fa4: 2s
- Output and side effects:
- * Returns the comment_root as the only output.
- * Modifies the list of strings, ``RST_source`` in place.
- """
- # We can only handle these nodes: anything else will raise an error
- KNOWN_NODES = set(('paragraph', 'title', 'literal_block', 'table',
- 'image', 'list_item', 'displaymath'))
- # Regular expression, which when matched AT THE START OF A LINE, indicate
- # list items in the RST syntax. See the full RST specification:
- # http://docutils.sourceforge.net/docs/user/rst/quickstart.html
- RST_LIST_ITEMS_AS_RE = re.compile(r'''(\s*)
- ( # group all list item types
- (\-)|(\*)|(\+)| # any bullet list items
- (\#\.)| # auto-enumerate: "#."
- (\w*\.)| # enumerated: "12." or "A." or "i."
- (\(\w*\))| # enumerated: "(23)" or "(B)"
- (\w\)) # enumerated: "12)" or "iii)"
- ) # end of all list item types
- (?P<space>\s*) # catch trailing spaces at end.''', \
- re.X)
- # Maps ``node_type`` to RST directives
- NODE_DIRECTIVE_MAP = {'displaymath': ['math'],
- 'image': ['image', 'figure'],
- 'table': ['table', 'csv-table', 'list_table'],
- 'literal_block': ['code-block', 'literalinclude'],}
- # Nodes given by the ``keys`` in NODE_DIRECTIVE_MAP allow blank lines
- # within in their content, so determining where to place the ucomment
- # directive cannot rely purely on finding the next blank line. Example:
- #
- # |Before |After
- # |--------------------------------------------------------------
- # 1 |.. figure:: the_figure.png |.. figure:: the_figure.png
- # 2 | :scale: 100% | :scale: 100%
- # 3 | |
- # 4 | Figure caption goes here. | Figure caption goes here.
- # 5 | |
- # 6 |Next paragraph begins here. |..ucomment:: ABCDEF: 2b
- # 7 | |
- # 8 | |Next paragraph begins here.
- #
- # It would be wrong to place the ucomment directive at/around line 3
- # as this would cut of the figure's caption. What we do instead is to
- # find the end of the node and insert the comment at that point.
- def wrap_return(RST_source, insert, prefix):
- """
- Adds the new ucomment directive and return the updated RST_source,
- or, appends to the existing comment.
- """
- if prefix is None:
- comment = RST_source[insert]
- c_root = comment.strip()[dir_len+1:dir_len+1+conf.root_node_length]
- # Always add a comma after the last comment
- if not comment.rstrip().endswith(','):
- suffix = ', '
- else:
- suffix = ' '
- RST_source[insert] = comment[0:-1] + suffix + comment_node + ',\n'
- else:
- c_root = comment_ref.comment_root
- line_to_add = prefix + COMMENT_DIRECTIVE + ' ' + c_root + ': ' + \
- comment_node + ',\n'
- if comment_ref.node_type in KNOWN_NODES:
- RST_source.insert(insert, '\n')
- RST_source.insert(insert, line_to_add)
- # Pop off the last line of text that was artifically added.
- RST_source.pop()
- return c_root
- # A few corner cases are solved if we ensure the file ends with a blank line
- if RST_source[-1].strip() != '':
- RST_source.append('\n')
- # Force an unrelated line at the end of the file to avoid coding
- # specifically for end-effects.
- RST_source.extend(['___END__OF___FILE___\n'])
- # The comment reference line numbers are 1-based; we need 0-based numbers
- line_num = comment_ref.line_number - 1
- dir_len = len(COMMENT_DIRECTIVE)
- if comment_ref.node_type not in KNOWN_NODES:
- raise UcommentError('Unknown node type: "%s"' % str(comment_ref))
- # To find any spaces at the start of a line
- prefix_re = re.compile('^\s*')
- prefix_match = prefix_re.match(RST_source[line_num])
- prefix = prefix_match.group()
- # There is one exception though: source code blocks marked with '::'
- # While the node's line number refers to the first line of code, the correct
- # prefix is the amount of space at the start of the line containing the '
- # double colons. Search backwards to find them.
- # If we can't find them, then this literal_block is assumed to be a
- # 'code-block', or 'literalinclude' node (i.e. no double colons).
- if comment_ref.node_type == 'literal_block':
- double_colon = re.compile(r'(?P<space>\s*)(.*)::(\s*)')
- directive_re = r'^(\s*)\.\. '
- for directive in NODE_DIRECTIVE_MAP['literal_block']:
- directive_re += '(' + directive + ')|'
- directive_re = directive_re[0:-1] + r'::(\s*)(.*)'
- directive = re.compile(directive_re)
- double_colon_line = line_num + 1
- for line in RST_source[line_num::-1]:
- double_colon_line -= 1
- if directive.search(line):
- break # it is one of the other directives
- if double_colon.match(line):
- prefix = double_colon.match(line).group('space')
- # Do some surgery on the RST source if the author is using
- # double colons. By example
- #
- # |Below is some code, the line ends with a double colon::
- # |
- # | >>> a = 'some source code'
- # |
- # Replace it as follows.
- #
- # |Below is some code, the line ends with a double colon:
- # |
- # |::
- # |
- # | >>> a = 'some source code'
- # |
- if line.strip() != '::':
- dci = line.rindex('::')
- RST_source[double_colon_line] = line[0:dci] + line[dci+1:]
- RST_source.insert(double_colon_line+1, '\n')
- RST_source.insert(double_colon_line+2, prefix + '::\n')
- # This will get saved in ``submit_and_store_comment(...)``
- if not isinstance(comment_ref, tuple):
- # Minor technicality: we used named tuples in unit tests
- comment_ref.line_number = comment_ref.line_number + 2
- line_num += 2
- break
- # The point where the ucomment directive will be inserted
- insert = line_num + 1
- # We are *always* given the top line number of the node: so we only have to
- # search for the insertion point below that. We will start examining from
- # the first line below that.
- finished = False
- next_line = ''
- idx_next = 0
- for idx, line in enumerate(RST_source[line_num+1:]):
- insert += 1 # insert = line_num + idx + 1
- bias = idx + 2
- if line.strip() == '' or comment_ref.node_type == 'list_item':
- if comment_ref.node_type == 'list_item':
- bias -= 1
- # Keep looking further down for an existing ucomment directive
- for idx_next, next_line in enumerate(RST_source[line_num+bias:]):
- if next_line.strip() != '':
- if next_line.lstrip()[0:dir_len] == COMMENT_DIRECTIVE:
- insert = line_num + bias + idx_next
- prefix = None
- return wrap_return(RST_source, insert, prefix)
- finished = True
- break
- if finished:
- next_prefix = prefix_re.match(next_line.rstrip('\n')).group()
- # Certain nodes cannot rely purely on blank lines to mark their end
- if comment_ref.node_type in NODE_DIRECTIVE_MAP.keys():
- # Break if a non-blank line has the same, or lower indentation
- # level than the environment's level (``prefix``)
- if len(next_prefix.expandtabs()) <= len(prefix.expandtabs()):
- break
- else:
- finished = False
- # ``list_item``s generally are commented on the very next line, but
- # first ensure the next line is in fact another list_item.
- # If the next line is a continuation of the current list_item, then
- # set ``finished`` to False, and keep searching.
- # blank or not, but
- elif comment_ref.node_type == 'list_item':
- cleaned = next_line[prefix_match.end():]
- # Most list items will break on this criterion (that the next
- # line contains a list item)
- if RST_LIST_ITEMS_AS_RE.match(cleaned):
- insert = insert + (bias - 2) + idx_next
- # Subtract off extra lines to handle multiline items
- if bias > 1:
- insert -= (bias -1)
- break
- # but the final entry in a list will break on this criterion
- elif len(next_prefix.expandtabs()) <= len(prefix.expandtabs()):
- #insert = insert - 1 # commented out: passes bullet_11
- break
- # It wasn't really the end of the current item (the one being
- # commented on). It's just that this item is written over
- # multiple lines.
- else:
- finished = False
- else:
- break
- # Lastly, list items require a bit more work to handle. What we want:
- #
- # |Before |After
- # |--------------------------------------------------------------
- # |#. This is a list item |#. This is a list item
- # |<no blank line originally here> | .. ucomment:: ABCDEF: 2a,
- # |#. Next list item |#. Next list item
- if comment_ref.node_type == 'list_item':
- # list_item's need to have a different level of indentation
- # If the ``RST_source[line_num]`` is ``____#.\tTwo.\n``, then
- # (note that _ represents a space)
- # * remainder = '#.\tTwo.\n' i.e. removed upfront spaces
- # * the_list_item = '#.' i.e. what defines the list
- # * prefix = '______\t' i.e. what to put before '.. ucomment'
- # * list_item.group('space')='\t' i.e. the tab that appears after '#.'
- remainder = RST_source[line_num][prefix_match.end():]
- list_item = RST_LIST_ITEMS_AS_RE.match(remainder)
- the_list_item = list_item.group().rstrip(list_item.group('space'))
- prefix = prefix + ' ' * len(the_list_item) + list_item.group('space')
- c_root = wrap_return(RST_source, insert, prefix)
- # There was no spaced between the list_items
- if idx_next == 0 and finished:
- RST_source.insert(insert, '\n')
- return c_root
- else:
- return wrap_return(RST_source, insert, prefix)
- def update_RST_comment_status(comment_ref, comment_node, RST_source, \
- search, replace):
- """
- Searches for the existing ``comment_ref`` and ``comment_node`` in the list
- of strings given by ``RST_source``. Finds the ``search`` character and
- replaces it with the ``replace`` character (indicating whether the comment
- was approved or rejected).
- The ``RST_source`` is a list of strings; this list will be updated in place.
- """
- comment_re = re.compile(r'(\s*)\.\. ' + COMMENT_DIRECTIVE.strip(' .:') + \
- r'::(\s*)(?P<root>\w*)(\s*):(\s*)(?P<nodes>.+)')
- idx = -1
- line = ''
- for idx, line in enumerate(RST_source[comment_ref.line_number-1:]):
- rematch = comment_re.match(line)
- if rematch:
- if rematch.group('root') == comment_ref.comment_root:
- break
- if idx < 0:
- log_file.error(('Comment (%s) was to be changed, but was not found in'
- 'the RST_sources.') % str(comment_ref))
- return RST_source
- # We have the correct node. Now edit the code.
- nodes = rematch.group('nodes')
- nodes = re.sub(comment_node + search, comment_node + replace, nodes)
- to_replace = line[0:rematch.start('nodes')] + nodes
- RST_source[comment_ref.line_number - 1 + idx] = to_replace + '\n'
- return comment_ref.comment_root
- # Emailing functions
- # ------------------
- def send_email(from_address, to_addresses, subject, message):
- """
- Basic function to send email according to the four required string inputs.
- Let Django send the message; it takes care of opening and closing the
- connection, as well as locking for thread safety.
- """
- if subject and message and from_address:
- try:
- send_mail(subj…
Large files files are truncated, but you can click here to view the full file