/XBMCSkins/lib/cachedhttp-v13/cachedhttp.py
Python | 377 lines | 331 code | 33 blank | 13 comment | 62 complexity | 99a5ccbc26ccbf3cca832fbee3e1b046 MD5 | raw file
Possible License(s): BSD-2-Clause
- #ScriptName : cachedhttp (formerly cachemanager)
- Version = '1.3'
- # Author : Van der Phunck aka Aslak Grinsted. as@phunck.cmo <- not cmo but com
- # Desc : cache manager
- #
- #
-
- import sys, traceback, os.path, re
- ScriptPath = os.path.split(sys.modules['cachedhttp'].__file__)[0]+'\\' #this should always work! (note... os.getcwd() doesn't)
-
- sys.path.insert(0, ScriptPath+'ClientCookie.zip')
-
- import urllib2, re, time, string, random
- import socket,md5,urlparse
- import xbmc,xbmcgui
- import mimetypes
- import httplib
- import ClientCookie
- import base64
- import urllib
- import shutil
-
- Debug=True
- DefaultCacheTime=24*60.0 #minutes
-
-
- try: Emulating = xbmcgui.Emulating #Thanks alot to alexpoet for the xbmc.py,xmbcgui.py emulator. Very useful!
- except: Emulating = False
-
-
- def fileExists(fname):
- return os.path.exists(fname)
-
- def readCacheMeta(fname):
- if not fileExists(fname): return None
- try:
- fid=file(fname,'rb')
- info=httplib.HTTPMessage(fid)
- fid.close()
- return info
- except:
- return None
-
- def isInFolder(fname,folder):
- fname=os.path.split(fname)
- fnamefolder=fname[0].lower()+'\\'
- if not (fnamefolder[-1]=='\\'): fnamefolder=fnamefolder+'\\'
- return (fnamefolder==folder.lower())
-
- Rauth=re.compile('^(.*):(.*)@(.*)$',re.IGNORECASE)
- def parseAuthUrl(url):
- fields=urlparse.urlparse(url)
- fields=[fields[0],fields[1],fields[2],fields[3],fields[4],fields[5]]
- fauth=Rauth.findall(fields[1])
- if len(fauth)==0: return [url,'','']
- fields[1]=fauth[0][2] #host
- name=urllib.unquote(fauth[0][0]) #username
- pwd=urllib.unquote(fauth[0][1]) #password
- url=urlparse.urlunparse(fields)
- return [url,name,pwd]
-
-
- Rwhitespace=re.compile('\s\s+',re.IGNORECASE)
- Rillegalchars=re.compile('[^\.\_\w\d-]',re.IGNORECASE)
- Rfilename=re.compile('[^\\/]*$',re.IGNORECASE)
- Rdot2=re.compile('[\.\s]+',re.IGNORECASE)
- def urltoxfilename(url):
- fields=urlparse.urlparse(url)
- fname=Rfilename.findall(fields[2])
- if len(fname)==0: fname=['']
- fname=fname[0]
- if len(fname)==0:
- fname=Rfilename.findall(fields[1])
- fname=fname[0]
- fname=fname.replace('.','-')
- fname=fname.replace(',',' ')
- fname=Rillegalchars.sub(' ',fname)
- fname=Rwhitespace.sub(' ',fname)
- fname=Rdot2.sub('.',fname)
- if len(fname)>40: #i think 42 is the limit?
- ext=os.path.splitext(fname)[1]
- fname=fname[0:40-max(len(ext),7)] #7 because a different extension might be added because of the mimetype...
- if fname[-1]=='.': fname=fname[0:-1]
- fname=fname+ext
- return fname
-
-
- class CustomHandler(ClientCookie.HTTPCookieProcessor):
- def http_error_304(self, req, fp, code, message, headers):
- addinfourl = urllib2.addinfourl(fp, headers, req.get_full_url())
- addinfourl.code = code
- return addinfourl
-
-
- class CachedHTTP:
- def __init__(self):
- self.userAgent='OobaCacheMgr/'+Version
- self.urlContext=''
- self.socketTimeout=7.0
- self.cacheFolder='Z:\\~HttpCache\\'
-
- if Emulating: self.cacheFolder=ScriptPath+'Cache\\'
- try:
- os.makedirs(self.cacheFolder)
- except: pass
-
- self.cookiefile=self.cacheFolder+'~cookies.txt'
- #the tilde is there to ensure that url2xfilename doesn't create a file that might overwrite this
-
- self.defaultCachetime=24*60.0 #minutes
-
- self.cookies=ClientCookie.LWPCookieJar()
- try:
- self.cookies.revert(self.cookiefile)
- except:
- print('Could not open cookie file: '+self.cookiefile)
-
- hh=CustomHandler(self.cookies)
-
- self.opener=ClientCookie.build_opener(hh)
-
- def saveCookies(self):
- self.cookies.save(self.cookiefile)
-
- def getCookieJar(self): return self.cookies
-
- def getUserAgent(self): return self.userAgent
- def setUserAgent(self,val): self.userAgent=val
-
- def getSocketTimeout(self): return self.socketTimeout
- def setSocketTimeout(self,val): self.socketTimeout=float(val)
-
- def getCacheFolder(self): return self.cacheFolder
- # def setCacheFolder(self,val): #you shouldn't call this really
- # self.cacheFolder=str(val)
- # if not (self.cacheFolder[-1]=='\\'): self.cacheFolder=self.cacheFolder+'\\'
-
- def getUrlContext(self): return self.urlContext
- def setUrlContext(self,val): self.urlContext=str(val)
-
- def getDefaultCachetime(self): return self.defaultCachetime
- def setDefaultCachetime(self,val): self.defaultCachetime=float(val)
-
- def getFullUrl(self,url):
- if len(url)>0:
- if url[0]=='?':
- f=urlparse.urlparse(self.urlContext)
- baseurl=urlparse.urlunparse([f[0],f[1],f[2],'','',''])
- return (baseurl+url).encode('iso-8859-1')
- return urlparse.urljoin(self.urlContext,url).encode('iso-8859-1')
-
- def onDataRetrieved(self, bytesRead, totalSize, url, localfile):
- return True
- def onDownloadFinished(self,success):
- pass
-
- def url2cachemetafile(self,url): #returns the filename of the meta file associated with url ... Note it does not necessarily exist
- m = md5.new()
- m.update(url)
- digest=m.hexdigest()
- return self.cacheFolder+'~'+digest
-
-
- def cacheFilename(self,url):
- urlmetafile=self.url2cachemetafile(url)
- if fileExists(urlmetafile):
- info=readCacheMeta(urlmetafile)
- localfile=info['CM-Localfile']
- if fileExists(localfile): return localfile
- return ''
-
- def flushCache(self,url):
- urlmetafile=self.url2cachemetafile(url)
- if not fileExists(urlmetafile): return
- info=readCacheMeta(urlmetafile)
- os.remove(urlmetafile)
- if info is None: return #couldn't understand meta file...
- try:
- localfile=info['CM-localfile']
- if fileExists(localfile):
- if isInFolder(localfile,self.cacheFolder):
- os.remove(localfile)
- except: pass
-
- def getCacheMeta(self,urlmetafile,expiretime=None): #returns None if the file doesn't exist or if the data is too old.
- #expiretime is a way of overriding cachetime information in the metafile...
- if not fileExists(urlmetafile): return None
- info=readCacheMeta(urlmetafile)
- isMetaOK=not (info is None)
- try:
- if isMetaOK:
- localfile=info['CM-localfile']
- timestamp=float(info['CM-TimeStamp'])
- cachetime=float(info['CM-CacheTime'])
- if not expiretime is None: cachetime=expiretime
- if cachetime>=0: #(that is not permanent)
- isMetaOK=isMetaOK & (abs(time.time()-timestamp)<(cachetime*60))
- if fileExists(localfile):
- contentlength=int(info['Content-Length'])
- isMetaOK=isMetaOK & (os.path.getsize(localfile)==contentlength)
- if (not isMetaOK) and isInFolder(localfile,self.cacheFolder):
- os.remove(localfile)
- else:
- isMetaOK=False
- except:
- isMetaOK=False
- if not isMetaOK:
- os.remove(urlmetafile)
- return None
- return info
-
-
- def cleanCache(self,expiretime=None):
- files=os.listdir(self.cacheFolder)
- Rmetafile=re.compile('^\~[\dabcdef]{32}$',re.IGNORECASE)
- for filename in files:
- try:
- fname=filename.lower()
- if Rmetafile.match(fname):
- filename=self.cacheFolder+fname
- info=self.getCacheMeta(filename,expiretime)
- except:
- pass
-
- def urlretrieve(self,url,cachetime=None,localfile=None,ext=None,postdata=None):
- url=self.getFullUrl(url)
- urlmetafile=self.url2cachemetafile(url)
- if cachetime is None: cachetime=self.defaultCachetime
- metainfo = self.getCacheMeta(urlmetafile)
- furl=None
- fcache=None
- isDownloadCompleted=False
- try:
- self.onDataRetrieved(0, None, url, '')
- oldtimeout=socket.getdefaulttimeout()
- socket.setdefaulttimeout(self.socketTimeout)
- authurl=parseAuthUrl(url)
- if len(authurl[1])>0: #todo use HTTPBasicAuthHandler instead....
- url=authurl[0]
- base64string=base64.encodestring('%s:%s' % (authurl[1],authurl[2]))[:-1]
- authheader = "Basic %s" % base64string
- request = ClientCookie.Request(url)
- request.add_header('User-Agent',self.userAgent)
- if len(authurl[1])>0: request.add_header("Authorization", authheader)
- #if len(self.urlContext)>0: #TODO: not always
- # request.add_header('Referer',self.urlContext)
- request.add_header('Referer',url.split( '?' )[0])
- request.add_header('Cookie','clientSupportsCookies=YES') # ClientCookie doesn't set this?
- if not (metainfo is None):
- try:
- etag=metainfo['ETag']
- request.add_header('If-None-Match', etag)
- except: pass
- try:
- lastmodified = metainfo['Last-Modified']
- request.add_header('If-Modified-Since', lastmodified)
- except: pass
- furl=self.opener.open(request,postdata)
- info=furl.info()
- if not (metainfo is None):
- if hasattr(furl, 'code') and furl.code == 304:
- self.urlContext=metainfo['CM-UrlContext']
- temp=os.path.split(metainfo['CM-Localfile'])
- print('using cache: '+temp[1])
- isDownloadCompleted=True
- if not (localfile is None):
- nameext=os.path.splitext(metainfo['CM-Localfile'])
- if not (localfile.lower()==nameext[0].lower()):
- localfile=localfile+nameext[1]
- shutil.copyfile(metainfo['CM-Localfile'],localfile)
- return localfile
- return metainfo['CM-Localfile']
- else:
- self.flushCache(url)
- try:
- totalSize=int(info['Content-Length'])
- except:
- totalSize=None
-
- #------------ construct local file name ---------
- xfname=os.path.splitext(urltoxfilename(url)) #tuple: suggested (filename,ext)
- xfname=[xfname[0],xfname[1]] #otherwise you cannot write to it
- try:
- try:
- old_xfname = xfname
- filename = info['Content-Disposition'].split( '\"' )[1]
- xfname = os.path.splitext( filename )
- except:
- xfname[0] = old_xfname[0]
- mimetype=info['Content-Type'].split(';') # also understand "Content-Type: text/html; charset=utf-8"
- mimetype=mimetype[0].strip()
- mimeext=mimetypes.guess_extension(mimetype)
- if (not (mimeext is None)) and (len(mimeext)>0):
- if mimeext=='.m1v': mimeext='.mpg'
- xfname[1]=mimeext #override the one based on url alone
- except:
- pass
- if not (ext is None): xfname[1]=ext #override with manual extension
- ext=xfname[1]
- xfname=xfname[0]
- if len(ext)>0:
- if not (ext[0]=='.'): ext='.'+ext
- ext=ext[0:7] #do not allow so long extensions... Just truncate
- if localfile is None: #then autogenerate a file name for the cache
- localfile=self.cacheFolder+xfname+ext
- i=1
- while fileExists(localfile):
- i=min(i*10,100000) #add a random number to minimize fileexist checks
- localfile=self.cacheFolder+xfname[0:30]+'['+str(random.randint(0,i-1))+']'+ext
- else:
- localfile=localfile+ext
- #------------------------------------------------
- fcache=file(localfile,'wb')
- iscanceled=not self.onDataRetrieved(0, totalSize, url, localfile)
- data='...'
- blockSize=8192
- pval=0
- while len(data)>0:
- if not totalSize is None:
- if pval>totalSize: totalSize=pval*2
- data = furl.read(blockSize)
- pval=pval+len(data)
- if len(data)>0: fcache.write(data)
- if len(data)<blockSize: break
- iscanceled=not self.onDataRetrieved(pval, totalSize, url, localfile)
- if iscanceled: break
- isDownloadCompleted=not iscanceled
- self.urlContext=furl.url
- finally:
- self.onDownloadFinished(isDownloadCompleted)
- try:
- if not fcache is None: fcache.close()
- if not furl is None: furl.close()
- socket.setdefaulttimeout(oldtimeout)
- if not isDownloadCompleted: os.remove(localfile)
- except:
- pass
- if not isDownloadCompleted:
- return None
- #------------- write url meta file ------------
- #TODO: maybe do something if info['cache-control']=private?
- info['Content-Length']=str(pval)
- info['CM-Localfile']=localfile
- info['CM-urlContext']=self.urlContext
- info['CM-CacheTime']=str(cachetime)
- info['CM-TimeStamp']=str(time.time())
- info['CM-url']=url
- fuc=file(urlmetafile,'wb')
- fuc.write(str(info))
- fuc.close()
- return localfile
-
- def urlopen(self,url,data=None):
- localfile=self.urlretrieve(url,postdata=data)
- f=file(localfile, 'rb')
- data = f.read()
- f.close()
- return data
-
-
-
- class CachedHTTPWithProgress(CachedHTTP):
- def onDataRetrieved(self, bytesRead, totalSize, url, localfile):
- if (not hasattr(self,'progressbar')): self.progressbar=None
- if self.progressbar is None:
- self.progressbar=xbmcgui.DialogProgress()
- self.progressbar.create("Downloading...",url,"> "+localfile)
- if (not (bytesRead is None))&(not (totalSize is None)):
- pct=int((bytesRead % (totalSize+1))*100.0/totalSize)
- self.progressbar.update(pct,url,'> '+os.path.split(localfile)[1])
- return not self.progressbar.iscanceled()
-
- def onDownloadFinished(self,success):
- try:
- self.progressbar.close()
- except: pass
- self.progressbar=None