/webspider/models/base.py

https://github.com/GuozhuHe/webspider · Python · 205 lines · 109 code · 32 blank · 64 comment · 14 complexity · af49cddd7ea804b106d01736afb71f2b MD5 · raw file

  1. # coding: utf-8
  2. import logging
  3. from sqlalchemy import MetaData, inspect, func, text
  4. from sqlalchemy.ext.declarative import declarative_base
  5. from tornado.util import ObjectDict
  6. from webspider.utils import sql
  7. from webspider.utils.classproperty import classproperty
  8. __all__ = ['BaseModel']
  9. logger = logging.getLogger(__name__)
  10. _Base = declarative_base()
  11. class BaseModel(_Base):
  12. __abstract__ = True
  13. __table_args__ = {
  14. 'mysql_engine': 'InnoDB',
  15. 'mysql_charset': 'utf8mb4',
  16. 'extend_existing': True,
  17. }
  18. metadata = MetaData(bind=sql.db_engine, reflect=True)
  19. @classproperty
  20. def session(cls):
  21. return sql.get_session()
  22. @classproperty
  23. def pk_name(cls):
  24. """主键名"""
  25. return inspect(cls).primary_key[0].name
  26. @classproperty
  27. def pk(cls):
  28. """表主键"""
  29. return getattr(cls, cls.pk_name)
  30. def dict(self):
  31. """sqlalchemy object -> dict"""
  32. columns = self.__table__.columns.keys()
  33. return ObjectDict((column, getattr(self, column)) for column in columns)
  34. @classmethod
  35. def count(cls, filter=None, filter_by=None):
  36. """
  37. 获取数据库中记录的数目
  38. :param filter: apply the given filtering criterion to a copy of this Query,
  39. using SQL expressions.
  40. :param filter_by: apply the given filtering criterion to a copy of this Query,
  41. using keyword expressions as a dict.
  42. :return:
  43. """
  44. query = cls.session.query(func.count(cls.pk))
  45. if filter is not None:
  46. query = query.filter(filter)
  47. if filter_by is not None:
  48. query = query.filter_by(**filter_by)
  49. return query.scalar()
  50. @classmethod
  51. def add(cls, **values):
  52. """添加记录"""
  53. obj = cls(**values)
  54. cls.session.add(obj)
  55. cls.session.flush()
  56. return getattr(obj, obj.pk_name)
  57. @classmethod
  58. def get_by_pk(cls, pk):
  59. """通过主键值获取记录"""
  60. query = cls.session.query(cls).filter(cls.pk == pk)
  61. return query.scalar()
  62. @classmethod
  63. def get_one(cls, filter=None, filter_by=None):
  64. """
  65. 获取记录
  66. :param filter: apply the given filtering criterion to a copy of this Query,
  67. using SQL expressions.
  68. :param filter_by: apply the given filtering criterion to a copy of this Query,
  69. using keyword expressions as a dict.
  70. :return:
  71. """
  72. query = cls.session.query(cls)
  73. if filter is not None:
  74. query = query.filter(filter)
  75. if filter_by is not None:
  76. query = query.filter_by(**filter_by)
  77. return query.first()
  78. @classmethod
  79. def list(cls, columns=None, filter=None, filter_by=None, order_by=None, group_by=None, offset=None, limit=None):
  80. """
  81. 批量获取记录
  82. :param columns: the columns you want to query, SQL expression, column, or mapped entity expected
  83. :param filter: apply the given filtering criterion to a copy of this Query,
  84. using SQL expressions.
  85. :param filter_by: apply the given filtering criterion to a copy of this Query,
  86. using keyword expressions as a dict.
  87. :param order_by: apply one or more ORDER BY criterion to the query and return
  88. the newly resulting ``Query``
  89. :param group_by: apply one or more GROUP BY criterion to the query and return
  90. the newly resulting :class:`.Query`
  91. :param offset: Apply an ``OFFSET`` to the query and return the newly resulting
  92. ``Query``.
  93. :param limit: Apply a ``LIMIT`` to the query and return the newly resulting
  94. ``Query``.
  95. :return:
  96. """
  97. query = cls.session.query(cls)
  98. if columns:
  99. query = cls.session.query(columns)
  100. if filter is not None:
  101. query = query.filter(filter)
  102. if filter_by is not None:
  103. query = query.filter_by(**filter_by)
  104. if group_by is not None:
  105. query = query.group_by(group_by)
  106. if order_by is not None:
  107. query = query.order_by(order_by)
  108. if offset is not None:
  109. query = query.offset(offset)
  110. if limit is not None:
  111. query = query.limit(limit)
  112. result = query.all()
  113. return result
  114. @classmethod
  115. def is_exist(cls, filter=None, filter_by=None):
  116. """
  117. 判断某个记录是否存在
  118. :param filter: apply the given filtering criterion to a copy of this Query,
  119. using SQL expressions.
  120. :param filter_by: apply the given filtering criterion to a copy of this Query,
  121. using keyword expressions as a dict.
  122. :return: boolean
  123. """
  124. return cls.count(filter=filter, filter_by=filter_by) != 0
  125. @classmethod
  126. def update(cls, filter=None, filter_by=None, values=None):
  127. """更新数据
  128. :param filter: apply the given filtering criterion to a copy of this Query,
  129. using SQL expressions.
  130. :param filter_by: apply the given filtering criterion to a copy of this Query,
  131. using keyword expressions as a dict.
  132. :param values: values to update
  133. :return: type: int, affected rows
  134. """
  135. query = cls.session.query(cls)
  136. if filter is not None:
  137. query = query.filter(filter)
  138. if filter_by is not None:
  139. query = query.filter_by(**filter_by)
  140. affect_rows = query.update(values)
  141. return affect_rows
  142. @classmethod
  143. def update_by_pk(cls, pk, values):
  144. """主键更新数据
  145. :param pk: 主键值
  146. :param values: dict 要更新的值key=value 形式
  147. :return: 返回变更的行数
  148. """
  149. return cls.update(filter=(cls.pk == pk), values=values)
  150. @classmethod
  151. def execute_sql_string(cls, sql_string, parameters_dict=None):
  152. """
  153. 直接执行 sql 语句
  154. eg:
  155. sql_string = 'select * from temp where id = :numbers' and parameters_dict = {'numbers': 1}
  156. >> select * from temp where id = 1
  157. :param sql_string: the sql string you want to execute
  158. :param parameters_dict: parameters
  159. :return: if query returns_rows return rows(List(tuple)) else return affect_rows(int)
  160. """
  161. query = cls.session.execute(text(sql_string), parameters_dict)
  162. if query.returns_rows:
  163. return query.fetchall()
  164. else:
  165. return query.rowcount
  166. @classmethod
  167. def batch_add(cls, instances):
  168. """批量添加记录"""
  169. if not all([isinstance(instance, cls) for instance in instances]):
  170. raise ValueError('all instances must be {table_name} model instance'.format(table_name=cls.__tablename__))
  171. cls.session.bulk_save_objects(instances)