/src/test/rgw/rgw_multi/multisite.py

https://github.com/OpenMPDK/KVCeph · Python · 387 lines · 326 code · 35 blank · 26 comment · 22 complexity · 09b8cc926bf66ca1bc40d1b6f89c535b MD5 · raw file

  1. from abc import ABCMeta, abstractmethod
  2. from six import StringIO
  3. import json
  4. from .conn import get_gateway_connection
  5. class Cluster:
  6. """ interface to run commands against a distinct ceph cluster """
  7. __metaclass__ = ABCMeta
  8. @abstractmethod
  9. def admin(self, args = None, **kwargs):
  10. """ execute a radosgw-admin command """
  11. pass
  12. class Gateway:
  13. """ interface to control a single radosgw instance """
  14. __metaclass__ = ABCMeta
  15. def __init__(self, host = None, port = None, cluster = None, zone = None, proto = 'http', connection = None):
  16. self.host = host
  17. self.port = port
  18. self.cluster = cluster
  19. self.zone = zone
  20. self.proto = proto
  21. self.connection = connection
  22. @abstractmethod
  23. def start(self, args = []):
  24. """ start the gateway with the given args """
  25. pass
  26. @abstractmethod
  27. def stop(self):
  28. """ stop the gateway """
  29. pass
  30. def endpoint(self):
  31. return '%s://%s:%d' % (self.proto, self.host, self.port)
  32. class SystemObject:
  33. """ interface for system objects, represented in json format and
  34. manipulated with radosgw-admin commands """
  35. __metaclass__ = ABCMeta
  36. def __init__(self, data = None, uuid = None):
  37. self.data = data
  38. self.id = uuid
  39. if data:
  40. self.load_from_json(data)
  41. @abstractmethod
  42. def build_command(self, command):
  43. """ return the command line for the given command, including arguments
  44. to specify this object """
  45. pass
  46. @abstractmethod
  47. def load_from_json(self, data):
  48. """ update internal state based on json data """
  49. pass
  50. def command(self, cluster, cmd, args = None, **kwargs):
  51. """ run the given command and return the output and retcode """
  52. args = self.build_command(cmd) + (args or [])
  53. return cluster.admin(args, **kwargs)
  54. def json_command(self, cluster, cmd, args = None, **kwargs):
  55. """ run the given command, parse the output and return the resulting
  56. data and retcode """
  57. s, r = self.command(cluster, cmd, args or [], **kwargs)
  58. if r == 0:
  59. output = s.decode('utf-8')
  60. output = output[output.find('{'):] # trim extra output before json
  61. data = json.loads(output)
  62. self.load_from_json(data)
  63. self.data = data
  64. return self.data, r
  65. # mixins for supported commands
  66. class Create(object):
  67. def create(self, cluster, args = None, **kwargs):
  68. """ create the object with the given arguments """
  69. return self.json_command(cluster, 'create', args, **kwargs)
  70. class Delete(object):
  71. def delete(self, cluster, args = None, **kwargs):
  72. """ delete the object """
  73. # not json_command() because delete has no output
  74. _, r = self.command(cluster, 'delete', args, **kwargs)
  75. if r == 0:
  76. self.data = None
  77. return r
  78. class Get(object):
  79. def get(self, cluster, args = None, **kwargs):
  80. """ read the object from storage """
  81. kwargs['read_only'] = True
  82. return self.json_command(cluster, 'get', args, **kwargs)
  83. class Set(object):
  84. def set(self, cluster, data, args = None, **kwargs):
  85. """ set the object by json """
  86. kwargs['stdin'] = StringIO(json.dumps(data))
  87. return self.json_command(cluster, 'set', args, **kwargs)
  88. class Modify(object):
  89. def modify(self, cluster, args = None, **kwargs):
  90. """ modify the object with the given arguments """
  91. return self.json_command(cluster, 'modify', args, **kwargs)
  92. class CreateDelete(Create, Delete): pass
  93. class GetSet(Get, Set): pass
  94. class Zone(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
  95. def __init__(self, name, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
  96. self.name = name
  97. self.zonegroup = zonegroup
  98. self.cluster = cluster
  99. self.gateways = gateways or []
  100. super(Zone, self).__init__(data, zone_id)
  101. def zone_arg(self):
  102. """ command-line argument to specify this zone """
  103. return ['--rgw-zone', self.name]
  104. def zone_args(self):
  105. """ command-line arguments to specify this zone/zonegroup/realm """
  106. args = self.zone_arg()
  107. if self.zonegroup:
  108. args += self.zonegroup.zonegroup_args()
  109. return args
  110. def build_command(self, command):
  111. """ build a command line for the given command and args """
  112. return ['zone', command] + self.zone_args()
  113. def load_from_json(self, data):
  114. """ load the zone from json """
  115. self.id = data['id']
  116. self.name = data['name']
  117. def start(self, args = None):
  118. """ start all gateways """
  119. for g in self.gateways:
  120. g.start(args)
  121. def stop(self):
  122. """ stop all gateways """
  123. for g in self.gateways:
  124. g.stop()
  125. def period(self):
  126. return self.zonegroup.period if self.zonegroup else None
  127. def realm(self):
  128. return self.zonegroup.realm() if self.zonegroup else None
  129. def is_read_only(self):
  130. return False
  131. def tier_type(self):
  132. raise NotImplementedError
  133. def has_buckets(self):
  134. return True
  135. def get_conn(self, credentials):
  136. return ZoneConn(self, credentials) # not implemented, but can be used
  137. class ZoneConn(object):
  138. def __init__(self, zone, credentials):
  139. self.zone = zone
  140. self.name = zone.name
  141. """ connect to the zone's first gateway """
  142. if isinstance(credentials, list):
  143. self.credentials = credentials[0]
  144. else:
  145. self.credentials = credentials
  146. if self.zone.gateways is not None:
  147. self.conn = get_gateway_connection(self.zone.gateways[0], self.credentials)
  148. def get_connection(self):
  149. return self.conn
  150. def get_bucket(self, bucket_name, credentials):
  151. raise NotImplementedError
  152. def check_bucket_eq(self, zone, bucket_name):
  153. raise NotImplementedError
  154. class ZoneGroup(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
  155. def __init__(self, name, period = None, data = None, zonegroup_id = None, zones = None, master_zone = None):
  156. self.name = name
  157. self.period = period
  158. self.zones = zones or []
  159. self.master_zone = master_zone
  160. super(ZoneGroup, self).__init__(data, zonegroup_id)
  161. self.rw_zones = []
  162. self.ro_zones = []
  163. self.zones_by_type = {}
  164. for z in self.zones:
  165. if z.is_read_only():
  166. self.ro_zones.append(z)
  167. else:
  168. self.rw_zones.append(z)
  169. def zonegroup_arg(self):
  170. """ command-line argument to specify this zonegroup """
  171. return ['--rgw-zonegroup', self.name]
  172. def zonegroup_args(self):
  173. """ command-line arguments to specify this zonegroup/realm """
  174. args = self.zonegroup_arg()
  175. realm = self.realm()
  176. if realm:
  177. args += realm.realm_arg()
  178. return args
  179. def build_command(self, command):
  180. """ build a command line for the given command and args """
  181. return ['zonegroup', command] + self.zonegroup_args()
  182. def zone_by_id(self, zone_id):
  183. """ return the matching zone by id """
  184. for zone in self.zones:
  185. if zone.id == zone_id:
  186. return zone
  187. return None
  188. def load_from_json(self, data):
  189. """ load the zonegroup from json """
  190. self.id = data['id']
  191. self.name = data['name']
  192. master_id = data['master_zone']
  193. if not self.master_zone or master_id != self.master_zone.id:
  194. self.master_zone = self.zone_by_id(master_id)
  195. def add(self, cluster, zone, args = None, **kwargs):
  196. """ add an existing zone to the zonegroup """
  197. args = zone.zone_arg() + (args or [])
  198. data, r = self.json_command(cluster, 'add', args, **kwargs)
  199. if r == 0:
  200. zone.zonegroup = self
  201. self.zones.append(zone)
  202. return data, r
  203. def remove(self, cluster, zone, args = None, **kwargs):
  204. """ remove an existing zone from the zonegroup """
  205. args = zone.zone_arg() + (args or [])
  206. data, r = self.json_command(cluster, 'remove', args, **kwargs)
  207. if r == 0:
  208. zone.zonegroup = None
  209. self.zones.remove(zone)
  210. return data, r
  211. def realm(self):
  212. return self.period.realm if self.period else None
  213. class Period(SystemObject, SystemObject.Get):
  214. def __init__(self, realm = None, data = None, period_id = None, zonegroups = None, master_zonegroup = None):
  215. self.realm = realm
  216. self.zonegroups = zonegroups or []
  217. self.master_zonegroup = master_zonegroup
  218. super(Period, self).__init__(data, period_id)
  219. def zonegroup_by_id(self, zonegroup_id):
  220. """ return the matching zonegroup by id """
  221. for zonegroup in self.zonegroups:
  222. if zonegroup.id == zonegroup_id:
  223. return zonegroup
  224. return None
  225. def build_command(self, command):
  226. """ build a command line for the given command and args """
  227. return ['period', command]
  228. def load_from_json(self, data):
  229. """ load the period from json """
  230. self.id = data['id']
  231. master_id = data['master_zonegroup']
  232. if not self.master_zonegroup or master_id != self.master_zonegroup.id:
  233. self.master_zonegroup = self.zonegroup_by_id(master_id)
  234. def update(self, zone, args = None, **kwargs):
  235. """ run 'radosgw-admin period update' on the given zone """
  236. assert(zone.cluster)
  237. args = zone.zone_args() + (args or [])
  238. if kwargs.pop('commit', False):
  239. args.append('--commit')
  240. return self.json_command(zone.cluster, 'update', args, **kwargs)
  241. def commit(self, zone, args = None, **kwargs):
  242. """ run 'radosgw-admin period commit' on the given zone """
  243. assert(zone.cluster)
  244. args = zone.zone_args() + (args or [])
  245. return self.json_command(zone.cluster, 'commit', args, **kwargs)
  246. class Realm(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet):
  247. def __init__(self, name, period = None, data = None, realm_id = None):
  248. self.name = name
  249. self.current_period = period
  250. super(Realm, self).__init__(data, realm_id)
  251. def realm_arg(self):
  252. """ return the command-line arguments that specify this realm """
  253. return ['--rgw-realm', self.name]
  254. def build_command(self, command):
  255. """ build a command line for the given command and args """
  256. return ['realm', command] + self.realm_arg()
  257. def load_from_json(self, data):
  258. """ load the realm from json """
  259. self.id = data['id']
  260. def pull(self, cluster, gateway, credentials, args = [], **kwargs):
  261. """ pull an existing realm from the given gateway """
  262. args += ['--url', gateway.endpoint()]
  263. args += credentials.credential_args()
  264. return self.json_command(cluster, 'pull', args, **kwargs)
  265. def master_zonegroup(self):
  266. """ return the current period's master zonegroup """
  267. if self.current_period is None:
  268. return None
  269. return self.current_period.master_zonegroup
  270. def meta_master_zone(self):
  271. """ return the current period's metadata master zone """
  272. zonegroup = self.master_zonegroup()
  273. if zonegroup is None:
  274. return None
  275. return zonegroup.master_zone
  276. class Credentials:
  277. def __init__(self, access_key, secret):
  278. self.access_key = access_key
  279. self.secret = secret
  280. def credential_args(self):
  281. return ['--access-key', self.access_key, '--secret', self.secret]
  282. class User(SystemObject):
  283. def __init__(self, uid, data = None, name = None, credentials = None, tenant = None):
  284. self.name = name
  285. self.credentials = credentials or []
  286. self.tenant = tenant
  287. super(User, self).__init__(data, uid)
  288. def user_arg(self):
  289. """ command-line argument to specify this user """
  290. args = ['--uid', self.id]
  291. if self.tenant:
  292. args += ['--tenant', self.tenant]
  293. return args
  294. def build_command(self, command):
  295. """ build a command line for the given command and args """
  296. return ['user', command] + self.user_arg()
  297. def load_from_json(self, data):
  298. """ load the user from json """
  299. self.id = data['user_id']
  300. self.name = data['display_name']
  301. self.credentials = [Credentials(k['access_key'], k['secret_key']) for k in data['keys']]
  302. def create(self, zone, args = None, **kwargs):
  303. """ create the user with the given arguments """
  304. assert(zone.cluster)
  305. args = zone.zone_args() + (args or [])
  306. return self.json_command(zone.cluster, 'create', args, **kwargs)
  307. def info(self, zone, args = None, **kwargs):
  308. """ read the user from storage """
  309. assert(zone.cluster)
  310. args = zone.zone_args() + (args or [])
  311. kwargs['read_only'] = True
  312. return self.json_command(zone.cluster, 'info', args, **kwargs)
  313. def delete(self, zone, args = None, **kwargs):
  314. """ delete the user """
  315. assert(zone.cluster)
  316. args = zone.zone_args() + (args or [])
  317. return self.command(zone.cluster, 'delete', args, **kwargs)