/support/pushsources/photonpublish.py
Python | 297 lines | 223 code | 45 blank | 29 comment | 31 complexity | 6f185b39d971e4ae2109bb6360003e5a MD5 | raw file
- #! /usr/bin/python2
- #
- # Copyright (C) 2015 VMware, Inc. All rights reserved.
- # photonpublish.py
- # Allows pushing rpms and other artifacts to
- # a bintray repository.
- # Allows queying a repository to get existing
- # file details.
- #
- # Author(s): Priyesh Padmavilasom
- #
- import sys
- import getopt
- import json
- import glob
- import os
- import hashlib
- import requests
- from requests.auth import HTTPBasicAuth
- from publishutils import publishUtils
- from publishconst import publishConst
- const = publishConst()
- class photonPublish:
- def __init__(self, context):
- self._context = context
- self._config = {}
- self.loadConfig()
- def loadConfig(self):
- confFile = self._context['config']
- if(len(confFile) == 0):
- return
- with open(confFile) as jsonFile:
- self._config = json.load(jsonFile)
- #override cmdline supplied params
- if('user' in self._context and len(self._context['user']) > 0):
- self._config['user'] = self._context['user']
- if('apikey' in self._context and len(self._context['apikey']) > 0):
- self._config['apikey'] = self._context['apikey']
- #get details of existing files in remote
- def getPackages(self):
- auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])
- #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
- url = '%s/packages/%s/%s/%s/versions/%s/files?include_unpublished=1' % \
- (self._config['baseurl'],\
- self._config['subject'],\
- self._config['repo'],\
- self._config['package'],\
- self._config['version'])
-
- req = requests.get(url, auth=auth)
- if(req.status_code >= 300):
- raise Exception(req.text)
- return req.json()
- #return list of unpublished content
- #works with details from config file
- def getUnpublished(self):
- result = []
- pkgs = self.getPackages()
- for pkg in pkgs:
- if not pkg[const.published]:
- result.append(pkg)
- return result
- #Check if the local path has any diffs with the
- #remote repo. Compare sha1 hash
- def check(self, pkgsRoot):
- result = {
- const.updates:[],
- const.new:[],
- const.obsoletes:[],
- const.verified:[]
- }
- localFiles = publishUtils.getFilesWithRelativePath(pkgsRoot)
- pkgs = self.getPackages()
- for pkg in pkgs:
- remotePath = pkg[const.path]
- if(remotePath in localFiles):
- localFiles.remove(remotePath)
- localPath = os.path.join(pkgsRoot, remotePath)
- if(os.path.isfile(localPath)):
- sha1 = publishUtils.sha1OfFile(localPath)
- if(sha1 == pkg[const.sha1]):
- result[const.verified].append(pkg)
- else:
- result[const.updates].append(pkg)
- else:
- result[const.obsoletes].append(pkg)
- for newFile in localFiles:
- result[const.new].append({const.path:newFile})
- return result
- #exec results from a check against remote
- #this will make remote and local in sync
- def syncRemote(self, root, checkResults):
- updateCount = len(checkResults[const.updates])
- newCount = len(checkResults[const.new])
- if(updateCount + newCount == 0):
- print 'Remote is up to date.'
- return
- #push updates
- print 'Updating %d files' % updateCount
- for new in checkResults[const.updates]:
- filePath = new[const.path]
- fileName = os.path.basename(filePath)
- fullPath = os.path.join(root, filePath)
- pathName = filePath.rstrip(fileName).rstrip('/')
- self.updateFile(fullPath, pathName)
- #push new files
- print 'Pushing %d new files' % newCount
- for new in checkResults[const.new]:
- filePath = new[const.path]
- fileName = os.path.basename(filePath)
- fullPath = os.path.join(root, filePath)
- pathName = filePath.rstrip(fileName).rstrip('/')
- self.pushFile(fullPath, pathName)
-
- #push a folder with rpms to a specified pathname
- #in the remote repo
- def push(self, filePath, pathName):
- results = []
- filesToPush = glob.glob(filePath)
- for fileToPush in filesToPush:
- result = self.pushFile(fileToPush, pathName)
- print result
- results.append(result)
- def pushFile(self, fileToPush, pathName):
- print 'Pushing file %s to path %s' % (fileToPush, pathName)
- result = {}
- auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])
- fileName = os.path.basename(fileToPush)
- if(len(pathName) > 0):
- pathAndFileName = '%s/%s' % (pathName, fileName)
- else:
- pathAndFileName = fileName
- #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
- url = '%s/content/%s/%s/%s/%s/%s' % \
- (self._config['baseurl'],\
- self._config['subject'],\
- self._config['repo'],\
- self._config['package'],\
- self._config['version'],\
- pathAndFileName)
- headers={'Content-Type': 'application/octet-stream'}
- with open(fileToPush, 'rb') as chunkedData:
- req = requests.put(url,
- auth=auth,
- data=chunkedData,
- headers=headers)
- if(req.status_code >= 300):
- raise Exception(req.text)
- result['destPath'] = pathAndFileName
- result['sourcePath'] = fileToPush
- result['returnCode'] = req.status_code
- result['msg'] = req.text
- return result
- def updateFile(self, fileToPush, pathName):
- print 'Updating file %s at path %s' % (fileToPush, pathName)
- result = {}
- auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])
- fileName = os.path.basename(fileToPush)
- if(len(pathName) > 0):
- pathAndFileName = '%s/%s' % (pathName, fileName)
- else:
- pathAndFileName = fileName
- #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
- url = '%s/content/%s/%s/%s/%s/%s?override=1' % \
- (self._config['baseurl'],\
- self._config['subject'],\
- self._config['repo'],\
- self._config['package'],\
- self._config['version'],\
- pathAndFileName)
- headers={'Content-Type': 'application/octet-stream'}
- with open(fileToPush, 'rb') as chunkedData:
- req = requests.put(url,
- auth=auth,
- data=chunkedData,
- headers=headers)
- if(req.status_code >= 300):
- raise Exception(req.text)
- result['destPath'] = pathAndFileName
- result['sourcePath'] = fileToPush
- result['returnCode'] = req.status_code
- result['msg'] = req.text
- return result
- #publishes pending content. Works with details from conf file.
- def publish(self):
- print 'Publishing pending files to %s/%s/%s' \
- % (self._config['repo'],
- self._config['package'],
- self._config['version'])
- result = {}
- auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])
- #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
- url = '%s/content/%s/%s/%s/%s/publish' % \
- (self._config['baseurl'],\
- self._config['subject'],\
- self._config['repo'],\
- self._config['package'],\
- self._config['version'])
- req = requests.post(url, auth=auth)
- if(req.status_code >= 300):
- raise Exception(req.text)
- return req.json()
- def showUsage():
- print 'photonpublish.py --files /rpms/*.rpm'
- print 'if you need to override config, --config /conf.conf'
- print 'if you need to override user/apikey, provide'
- print '--user username --apikey apikey'
- def validate(context):
- return len(context['config']) > 0 and len(context['files']) > 0
- #test photonPublish
- def main(argv):
- try:
- context = {
- 'config':'/etc/photonpublish.conf',
- 'user':'',
- 'apikey':'',
- 'files':'',
- 'path':'',
- 'mode':''
- }
- opts, args = getopt.getopt(
- sys.argv[1:],
- '',
- ['config=','user=','apikey=','files=','path=','mode='])
- for opt, arg in opts:
- if opt == '--config':
- context['config'] = arg
- elif opt == '--user':
- context['user'] = arg
- elif opt == '--apikey':
- context['apikey'] = arg
- elif opt == '--files':
- context['files'] = arg
- elif opt == '--path':
- context['path'] = arg
- elif opt == '--mode':
- context['mode'] = arg
- if(not validate(context)):
- showUsage()
- sys.exit(1)
- publish = photonPublish(context)
- if(context['mode'] == 'upload'):
- publish.push(context['files'], context['path'])
- elif(context['mode'] == 'check'):
- print publish.check(context['files'])
- except Exception, e:
- print "Error: %s" % e
- sys.exit(1)
- if __name__ == "__main__":
- main(sys.argv)