/ibis/inventory/export_it_inventory.py

https://github.com/Cigna/ibis · Python · 164 lines · 132 code · 11 blank · 21 comment · 7 complexity · 006fb565c9ded0b41e390f7b5a1b5c63 MD5 · raw file

  1. """Export IT inventory."""
  2. from ibis.inventory.inventory import Inventory
  3. from ibis.model.exporttable import ItTableExport
  4. class ExportITInventory(Inventory):
  5. """Class used for managing the records of source table connections.
  6. and properties in the it_table.
  7. """
  8. def __init__(self, *arg):
  9. """Init."""
  10. super(self.__class__, self).__init__(*arg)
  11. self.table = self.cfg_mgr.it_table_export
  12. def insert_export(self, it_table):
  13. """Insert a dictionary representation of a table
  14. into the it table export."""
  15. insert = False
  16. tbl_connection_factory = it_table.connection_factories
  17. where_condition = ("source_database_name='{0}' and "
  18. "source_table_name='{1}' and "
  19. "db_env='{2}'")
  20. where_condition = where_condition.format(
  21. it_table.database, it_table.table_name, it_table.db_env)
  22. if not self.get_all(where_condition):
  23. values = ("'{full_table_name}', '{source_dir}',"
  24. "{mappers}, '{jdbcurl}',"
  25. "'{connection_factories}', '{db_username}',"
  26. "'{password_file}', '{frequency}', {fetch_size},"
  27. "'{target_schema}', '{target_table}',"
  28. "'{staging_database}'")
  29. values = values.format(source_dir=it_table.dir,
  30. jdbcurl=it_table.jdbcurl,
  31. db_username=it_table.username,
  32. password_file=it_table.password_file,
  33. target_schema=it_table.target_schema,
  34. target_table=it_table.target_table,
  35. full_table_name=it_table.full_name,
  36. mappers=it_table.mappers,
  37. connection_factories=tbl_connection_factory,
  38. frequency=it_table.weight,
  39. fetch_size=it_table.fetch_size,
  40. staging_database=it_table.staging_database)
  41. query = ("insert into table {it_tbl_export} partition "
  42. "(source_database_name='{db}', source_table_name"
  43. "='{tbl}', db_env='{db_env}') values ({values})")
  44. query = query.format(it_tbl_export=self.table,
  45. db=it_table.database,
  46. tbl=it_table.table_name,
  47. db_env=it_table.db_env,
  48. values=values)
  49. self.run_query(query, self.table)
  50. insert = True
  51. msg = ('Insert new record {db} {table} {db_env} into '
  52. ' {it_table_export}')
  53. msg = msg.format(db=it_table.database, table=it_table.table_name,
  54. db_env=it_table.db_env,
  55. it_table_export=self.table)
  56. self.logger.info(msg)
  57. else:
  58. msg = ('Table {db} {tbl} {db_env} NOT INSERTED into '
  59. ' {it_table_export} Already exists!')
  60. msg = msg.format(db=it_table.database, tbl=it_table.table_name,
  61. db_env=it_table.db_env,
  62. it_table_export=self.table)
  63. self.logger.warning(msg)
  64. return insert, msg
  65. def update_export(self, it_table):
  66. """Updates/overwrites a record with a provided
  67. dictionary representation of a table
  68. """
  69. updated = False
  70. tbl_connection_factory = it_table.connection_factories
  71. where_condition = ("source_database_name='{0}' and "
  72. "source_table_name='{1}' and "
  73. "db_env='{2}'")
  74. where_condition = where_condition.format(
  75. it_table.database, it_table.table_name, it_table.db_env)
  76. if self.get_all(where_condition):
  77. values = ("'{full_table_name}', '{source_dir}',"
  78. "{mappers}, '{jdbcurl}',"
  79. "'{connection_factories}', '{db_username}',"
  80. "'{password_file}', '{frequency}', {fetch_size},"
  81. "'{target_schema}', '{target_table}',"
  82. "'{staging_database}'")
  83. values = values.format(source_dir=it_table.dir,
  84. jdbcurl=it_table.jdbcurl,
  85. db_username=it_table.username,
  86. password_file=it_table.password_file,
  87. target_schema=it_table.target_schema,
  88. target_table=it_table.target_table,
  89. full_table_name=it_table.full_name,
  90. mappers=it_table.mappers,
  91. connection_factories=tbl_connection_factory,
  92. frequency=it_table.weight,
  93. fetch_size=it_table.fetch_size,
  94. staging_database=it_table.staging_database)
  95. query = ("insert overwrite table {it_tbl_export} partition "
  96. "(source_database_name='{db}', source_table_name"
  97. "='{tbl}', db_env='{db_env}') values ({values})")
  98. query = query.format(it_tbl_export=self.table,
  99. db=it_table.database,
  100. tbl=it_table.table_name,
  101. db_env=it_table.db_env,
  102. values=values)
  103. self.run_query(query, self.table)
  104. updated = True
  105. msg = 'Updated table {db} {table} {db_env} in {it_table_export}'
  106. msg = msg.format(db=it_table.database, table=it_table.table_name,
  107. db_env=it_table.db_env,
  108. it_table_export=self.table)
  109. self.logger.info(msg)
  110. else:
  111. msg = ('Table {db} {tbl} {db_env} NOT UPDATED in {it_table_export}'
  112. ' It does not exist!')
  113. msg = msg.format(db=it_table.database, tbl=it_table.table_name,
  114. db_env=it_table.db_env,
  115. it_table_export=self.table)
  116. self.logger.warning(msg)
  117. return updated, msg
  118. def parse_requests_export(self, requests):
  119. """Return a list of dictionaries of table representation.
  120. Args:
  121. requests: List of ibis.inventory.request_inventory.Request
  122. Returns:
  123. List[{ibis.model.exporttable.ItTableExport}].
  124. """
  125. tables = []
  126. for request_table in requests:
  127. it_table = ItTableExport(request_table.get_meta_data(),
  128. self.cfg_mgr)
  129. tables.append(it_table)
  130. return tables
  131. def get_all_tables_export(self):
  132. """Return a list of all records from it_table_export."""
  133. tables = []
  134. query = 'SELECT * FROM {tbl}'.format(tbl=self.table)
  135. result = self.get_rows(query)
  136. if result:
  137. for table_row in result:
  138. tables.append(self._build_row_dict_export(table_row))
  139. return tables
  140. def _build_row_dict_export(self, row):
  141. """Builds it_table row dict
  142. Args:
  143. row: list of row values from it_table
  144. """
  145. tbl_dict = {
  146. 'full_table_name': row[0], 'source_dir': row[1], 'mappers': row[2],
  147. 'jdbcurl': row[3], 'connection_factories': row[4],
  148. 'db_username': row[5], 'password_file': row[6],
  149. 'frequency': row[7], 'fetch_size': row[8], 'target_schema': row[9],
  150. 'target_table': row[10], 'staging_database': row[11],
  151. 'source_database_name': row[-3], 'source_table_name': row[-2],
  152. 'db_env': row[-1]}
  153. return tbl_dict