/support/pushsources/photonpublish.py

#! /usr/bin/python2
#
#    Copyright (C) 2015 VMware, Inc. All rights reserved.
#    photonpublish.py 
#    Allows pushing rpms and other artifacts to 
#    a bintray repository.
#    Allows queying a repository to get existing 
#    file details.
#
#    Author(s): Priyesh Padmavilasom
#

import sys
import getopt
import json
import glob
import os
import hashlib

import requests
from requests.auth import HTTPBasicAuth

from publishutils import publishUtils
from publishconst import publishConst


const = publishConst()

class photonPublish:

    def __init__(self, context):
        self._context = context
        self._config = {}
        self.loadConfig()

    def loadConfig(self):
        confFile = self._context['config']
        if(len(confFile) == 0):
            return
        with open(confFile) as jsonFile:
            self._config = json.load(jsonFile)
        #override cmdline supplied params
        if('user' in self._context and len(self._context['user']) > 0):
            self._config['user'] = self._context['user']
        if('apikey' in self._context and len(self._context['apikey']) > 0):
            self._config['apikey'] = self._context['apikey']


    #get details of existing files in remote
    def getPackages(self):
        auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])

        #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
        url = '%s/packages/%s/%s/%s/versions/%s/files?include_unpublished=1' % \
              (self._config['baseurl'],\
               self._config['subject'],\
               self._config['repo'],\
               self._config['package'],\
               self._config['version'])
        
        req = requests.get(url, auth=auth)
        if(req.status_code >= 300):
            raise Exception(req.text)

        return req.json()

    #return list of unpublished content
    #works with details from config file
    def getUnpublished(self):
        result = []
        pkgs = self.getPackages()
        for pkg in pkgs:
            if not pkg[const.published]:
                result.append(pkg)
        return result

    #Check if the local path has any diffs with the 
    #remote repo. Compare sha1 hash
    def check(self, pkgsRoot):
        result = {
                     const.updates:[],
                     const.new:[],
                     const.obsoletes:[],
                     const.verified:[]
                 }        
        localFiles = publishUtils.getFilesWithRelativePath(pkgsRoot)
        pkgs = self.getPackages()
        for pkg in pkgs:
            remotePath = pkg[const.path]

            if(remotePath in localFiles):
                localFiles.remove(remotePath)

            localPath = os.path.join(pkgsRoot, remotePath)
            if(os.path.isfile(localPath)):
                sha1 = publishUtils.sha1OfFile(localPath)
                if(sha1 == pkg[const.sha1]):
                    result[const.verified].append(pkg)
                else:
                    result[const.updates].append(pkg)
            else:
                result[const.obsoletes].append(pkg)

        for newFile in localFiles:
            result[const.new].append({const.path:newFile})

        return result

    #exec results from a check against remote
    #this will make remote and local in sync
    def syncRemote(self, root, checkResults):
        updateCount = len(checkResults[const.updates])
        newCount = len(checkResults[const.new])

        if(updateCount + newCount == 0):
            print 'Remote is up to date.' 
            return

        #push updates
        print 'Updating %d files' % updateCount 
        for new in checkResults[const.updates]:
            filePath = new[const.path]
            fileName = os.path.basename(filePath)
            fullPath = os.path.join(root, filePath)
            pathName = filePath.rstrip(fileName).rstrip('/')
            self.updateFile(fullPath, pathName)

        #push new files
        print 'Pushing %d new files' % newCount
        for new in checkResults[const.new]:
            filePath = new[const.path]
            fileName = os.path.basename(filePath)
            fullPath = os.path.join(root, filePath)
            pathName = filePath.rstrip(fileName).rstrip('/')
            self.pushFile(fullPath, pathName)
        

    #push a folder with rpms to a specified pathname 
    #in the remote repo
    def push(self, filePath, pathName):
        results = []
        filesToPush = glob.glob(filePath)
        for fileToPush in filesToPush:
            result = self.pushFile(fileToPush, pathName)
            print result
            results.append(result)

    def pushFile(self, fileToPush, pathName):
        print 'Pushing file %s to path %s' % (fileToPush, pathName)
        result = {}
        auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])

        fileName = os.path.basename(fileToPush)
        if(len(pathName) > 0):
            pathAndFileName = '%s/%s' % (pathName, fileName)
        else:
            pathAndFileName = fileName

        #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
        url = '%s/content/%s/%s/%s/%s/%s' % \
              (self._config['baseurl'],\
               self._config['subject'],\
               self._config['repo'],\
               self._config['package'],\
               self._config['version'],\
               pathAndFileName)

        headers={'Content-Type': 'application/octet-stream'}
        with open(fileToPush, 'rb') as chunkedData:
            req = requests.put(url,
                               auth=auth,
                               data=chunkedData,
                               headers=headers)

        if(req.status_code >= 300):
            raise Exception(req.text)

        result['destPath'] = pathAndFileName
        result['sourcePath'] = fileToPush
        result['returnCode'] = req.status_code
        result['msg'] = req.text
        return result 

    def updateFile(self, fileToPush, pathName):
        print 'Updating file %s at path %s' % (fileToPush, pathName)
        result = {}
        auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])

        fileName = os.path.basename(fileToPush)
        if(len(pathName) > 0):
            pathAndFileName = '%s/%s' % (pathName, fileName)
        else:
            pathAndFileName = fileName

        #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
        url = '%s/content/%s/%s/%s/%s/%s?override=1' % \
              (self._config['baseurl'],\
               self._config['subject'],\
               self._config['repo'],\
               self._config['package'],\
               self._config['version'],\
               pathAndFileName)

        headers={'Content-Type': 'application/octet-stream'}
        with open(fileToPush, 'rb') as chunkedData:
            req = requests.put(url,
                               auth=auth,
                               data=chunkedData,
                               headers=headers)

        if(req.status_code >= 300):
            raise Exception(req.text)

        result['destPath'] = pathAndFileName
        result['sourcePath'] = fileToPush
        result['returnCode'] = req.status_code
        result['msg'] = req.text
        return result 

    #publishes pending content. Works with details from conf file.
    def publish(self):
        print 'Publishing pending files to %s/%s/%s' \
               % (self._config['repo'],
                  self._config['package'],
                  self._config['version'])
        result = {}
        auth = HTTPBasicAuth(self._config['user'], self._config['apikey'])

        #form url: https://api.com/content/vmware/photon/releases/0.9 for eg.
        url = '%s/content/%s/%s/%s/%s/publish' % \
              (self._config['baseurl'],\
               self._config['subject'],\
               self._config['repo'],\
               self._config['package'],\
               self._config['version'])

        req = requests.post(url, auth=auth)

        if(req.status_code >= 300):
            raise Exception(req.text)

        return req.json()

def showUsage():
    print 'photonpublish.py --files /rpms/*.rpm'
    print 'if you need to override config, --config /conf.conf'
    print 'if you need to override user/apikey, provide'
    print '--user username --apikey apikey'

def validate(context):
    return len(context['config']) > 0 and len(context['files']) > 0

#test photonPublish
def main(argv):
    try:
        context = {
                      'config':'/etc/photonpublish.conf',
                      'user':'',
                      'apikey':'',
                      'files':'',
                      'path':'',
                      'mode':''
                  }
        opts, args = getopt.getopt(
                         sys.argv[1:],
                         '',
                         ['config=','user=','apikey=','files=','path=','mode='])
        for opt, arg in opts:
            if opt == '--config':
                context['config'] = arg
            elif opt == '--user':
                context['user'] = arg
            elif opt == '--apikey':
                context['apikey'] = arg
            elif opt == '--files':
                context['files'] = arg
            elif opt == '--path':
                context['path'] = arg
            elif opt == '--mode':
                context['mode'] = arg

        if(not validate(context)):
            showUsage()
            sys.exit(1)

        publish = photonPublish(context)
        if(context['mode'] == 'upload'):
            publish.push(context['files'], context['path'])
        elif(context['mode'] == 'check'):
            print publish.check(context['files'])

    except Exception, e:
        print "Error: %s" % e
        sys.exit(1)

if __name__ == "__main__":
    main(sys.argv)