/httparchive/views/trends.py
Python | 376 lines | 327 code | 40 blank | 9 comment | 35 complexity | d50ef8657ff62c78bbbb148378e9b0f8 MD5 | raw file
Possible License(s): GPL-2.0, BSD-3-Clause, Apache-2.0
- import datetime
- from decimal import Decimal
- from pyramid.view import view_config
- from sqlalchemy import func, distinct
- import pandas
- from gviz_data_table import Table
- from httparchive.models import DBSession, Stat, CDN
- from .utils import (
- format_size,
- json,
- dataframe_to_rows,
- )
- try:
- a = long
- except NameError:
- long = int
- @view_config(route_name='trends', renderer='templates/trends.pt')
- class View(object):
- _trends = None
- _dates = None
- _slices = None
- _slice = None
- trend_charts = ("trend_urls", "trend_load_times", "trend_requests",
- "trend_html", "trend_js", "trend_css", "trend_image",
- "trend_flash", "trend_fonts",
- "trend_speed", "trend_dom", "trend_domains", "trend_cache",
- "trend_google_libs", "trend_percent_flash",
- "trend_percent_fonts", "trend_compressed", "trend_https",
- "trend_errors", "trend_redirects", "trend_percent_cdn")
- def __init__(self, context, request):
- self.request = request
- self.session = DBSession()
- def __call__(self):
- self.start = self.request.GET.get("start") or str(self.dates[min([0, -25])])
- self.stop = self.request.GET.get("stop") or str(max(self.dates))
- if self.stop < self.start:
- self.stop = None
- slice = self.request.GET.get("slice")
- if slice not in self.slices:
- slice = self.slices[0]
- self.slice = slice
- return {'start':self.start, 'stop':self.stop, 'slice':slice}
- @property
- def dates(self):
- """
- List possible dates
- """
- if getattr(self, "_dates"):
- return self._dates
- q = self.session.query(Stat.labelDate)
- q = q.filter(Stat.slice == "All")
- q = q.filter(Stat.device == "IE8")
- q = q.filter(Stat.labelDate != None)
- q = q.order_by(Stat.labelDate)
- self._dates = [r.labelDate for r in q]
- return self._dates
- @property
- def slices(self):
- """
- List possible slices
- """
- if getattr(self, "_slices"):
- return self._slices
- q = self.session.query(Stat.slice)
- q = q.group_by(Stat.slice)
- q = q.order_by(Stat.slice)
- self._slices = [r.slice for r in q]
- return self._slices
- @property
- def trends(self):
- """
- Get trends for the specified period
- """
- if getattr(self, "_trends"):
- return self._trends
- q = self.session.query(
- Stat.labelDate,
- Stat.numurls,
- Stat.reqTotal,
- Stat.reqHTML,
- Stat.reqJS,
- Stat.reqCSS,
- Stat.reqImg,
- Stat.reqFlash,
- Stat.reqFont,
- func.round(Stat.bytesTotal / 1024).label("bytesTotal"),
- func.round(Stat.bytesHTML / 1024).label("bytesHTML"),
- func.round(Stat.bytesJS / 1024).label("bytesJS"),
- func.round(Stat.bytesCSS / 1024).label("bytesCSS"),
- func.round(Stat.bytesImg / 1024).label("bytesImg"),
- func.round(Stat.bytesFlash / 1024).label("bytesFlash"),
- func.round(Stat.bytesFont / 1024).label("bytesFont"),
- Stat.PageSpeed,
- Stat.SpeedIndex,
- func.round(Stat.numDomains).label("numDomains"),
- Stat.bytesHTMLDoc,
- Stat.numDOMElements,
- Stat.maxDomainReqs,
- Stat.maxage0,
- Stat.perFlash,
- Stat.perFonts,
- Stat.perGlibs,
- Stat.perErrors,
- Stat.perHttps,
- Stat.perRedirects,
- Stat.perCompressed,
- Stat.perCdn,
- Stat.onLoad,
- Stat.renderStart,
- )
- q = q.filter(Stat.slice == self.slice)
- q = q.filter(Stat.device == 'IE8')
- q = q.filter(Stat.labelDate != None)
- if self.start is not None:
- q = q.filter(Stat.labelDate >= func.date(self.start))
- if self.stop is not None:
- q = q.filter(Stat.labelDate <= func.date(self.stop))
- if not self.stop and not self.start:
- q = q.limit(25)
- q = q.order_by(Stat.labelDate)
- self._trends = q.all()
- return self._trends
- @property
- def chart_data(self):
- for chart in self.trend_charts:
- yield {'name': chart, 'data':getattr(self, chart)}
- @property
- def trend_urls(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("# URLs Surveyed", long)
- for r in self.trends:
- table.append([r.labelDate, r.numurls])
- return table.encode()
- @property
- def trend_load_times(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("Start Render Time in seconds", float)
- table.add_column("Load Time in seconds", float)
- for r in self.trends:
- table.append([r.labelDate,
- float(r.renderStart)/1000,
- float(r.onLoad)/1000
- ])
- return table.encode()
- @property
- def trend_requests(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("Number of Requests", int)
- table.add_column("Transfer Size (kB)", int)
- for r in self.trends:
- table.append([r.labelDate, int(r.reqTotal), int(r.bytesTotal)])
- return table.encode()
- @property
- def trend_html(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("HTML Requests", float)
- table.add_column("HTML Transfer Size (kB)", int)
- for r in self.trends:
- table.append([r.labelDate, r.reqHTML, int(r.bytesHTML)])
- return table.encode()
- @property
- def trend_js(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("JS Requests", float)
- table.add_column("JS Transfer Size (kB)", int)
- for r in self.trends:
- table.append([r.labelDate, r.reqJS, int(r.bytesJS)])
- return table.encode()
- @property
- def trend_css(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("CSS Requests", float)
- table.add_column("CSS Transfer Size (kB)", int)
- for r in self.trends:
- table.append([r.labelDate, r.reqCSS, int(r.bytesCSS)])
- return table.encode()
- @property
- def trend_image(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("Image Requests", float)
- table.add_column("Image Transfer Size (kB)", int)
- for r in self.trends:
- table.append([r.labelDate, r.reqImg, int(r.bytesImg)])
- return table.encode()
- @property
- def trend_flash(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("Flash Requests", float)
- table.add_column("Flash Transfer Size (kB)", int)
- for r in self.trends:
- table.append([r.labelDate, r.reqFlash, int(r.bytesFlash)])
- return table.encode()
- @property
- def trend_fonts(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("Font Requests", float)
- table.add_column("Font Transfer Size (kB)", int)
- for r in self.trends:
- table.append([r.labelDate, r.reqFont, int(r.bytesFont)])
- return table.encode()
- @property
- def trend_speed(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("Speed Index", int)
- for r in self.trends:
- table.append([r.labelDate, int(r.SpeedIndex)])
- return table.encode()
- @property
- def trend_domains(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("# Domains", int)
- table.add_column("Max Requests on Single Domain", int)
- for r in self.trends:
- table.append([r.labelDate, int(r.numDomains), int(r.maxDomainReqs)])
- return table.encode()
- @property
- def trend_google_libs(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites Using Google Libraries", int)
- for r in self.trends:
- table.append([r.labelDate, int(r.perGlibs)])
- return table.encode()
- @property
- def trend_percent_flash(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites Using Flash", int)
- for r in self.trends:
- table.append([r.labelDate, int(r.perFlash)])
- return table.encode()
- @property
- def trend_percent_fonts(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites Using Custom Fonts", int)
- for r in self.trends:
- table.append([r.labelDate, int(r.perFonts)])
- return table.encode()
- @property
- def trend_percent_cdn(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites Serving HTML on CDN", int)
- for r in self.trends:
- table.append([r.labelDate, int(r.perCdn)])
- return table.encode()
- @property
- def trend_cache(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites with Cacheable Resources", int)
- for r in self.trends:
- table.append([r.labelDate, r.maxage0])
- return table.encode()
- @property
- def trend_https(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites Using HTTPS", int)
- for r in self.trends:
- table.append([r.labelDate, r.perHttps])
- return table.encode()
- @property
- def trend_errors(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites with Errors", int)
- for r in self.trends:
- table.append([r.labelDate, r.perErrors])
- return table.encode()
- @property
- def trend_redirects(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("% of Sites with Redirects", int)
- for r in self.trends:
- table.append([r.labelDate, r.perRedirects])
- return table.encode()
- @property
- def trend_compressed(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("", int)
- for r in self.trends:
- table.append([r.labelDate, r.perCompressed])
- return table.encode()
- @property
- def trend_dom(self):
- table = Table()
- table.add_column("Date", datetime.date)
- table.add_column("Doc Size", int)
- table.add_column("DOM Elements", int)
- for r in self.trends:
- table.append([r.labelDate, format_size(r.bytesHTMLDoc),
- int(r.numDOMElements)])
- return table.encode()
-
- @property
- def trend_cdn(self):
- q = self.session.query(CDN.labelDate, CDN.cdn, CDN.sites)
- q = q.filter(CDN.slice==self.slice)
- q = q.filter(CDN.labelDate >= func.date(self.start))
- q = q.filter(CDN.labelDate <= func.date(self.stop))
- data = pandas.read_sql(q.statement, self.session.bind)
- pivoted = data.pivot(index="labelDate", columns="cdn", values="sites")
- pivoted.sort_index(ascending=False, inplace=True)
-
- table = Table()
- rows = dataframe_to_rows(pivoted)
- table.add_column("Date", datetime.date)
- header = next(rows)
- for col in header:
- table.add_column(col, float)
-
- for row in rows:
- row[0] = row[0].date()
- table.append(row)
-
- return table.encode()