/ts/sql/util.ts

https://github.com/WhisperSystems/Signal-Browser · TypeScript · 260 lines · 215 code · 36 blank · 9 comment · 46 complexity · 5a009bb43ee0bdb2c4c5a4ae9e215dc9 MD5 · raw file

  1. // Copyright 2021 Signal Messenger, LLC
  2. // SPDX-License-Identifier: AGPL-3.0-only
  3. import type { Database } from 'better-sqlite3';
  4. import { isNumber, last } from 'lodash';
  5. export type EmptyQuery = [];
  6. export type ArrayQuery = Array<Array<null | number | bigint | string>>;
  7. export type Query = { [key: string]: null | number | bigint | string | Buffer };
  8. export type JSONRows = Array<{ readonly json: string }>;
  9. export type TableType =
  10. | 'attachment_downloads'
  11. | 'conversations'
  12. | 'identityKeys'
  13. | 'items'
  14. | 'messages'
  15. | 'preKeys'
  16. | 'senderKeys'
  17. | 'sessions'
  18. | 'signedPreKeys'
  19. | 'stickers'
  20. | 'unprocessed';
  21. // This value needs to be below SQLITE_MAX_VARIABLE_NUMBER.
  22. const MAX_VARIABLE_COUNT = 100;
  23. export function objectToJSON<T>(data: T): string {
  24. return JSON.stringify(data);
  25. }
  26. export function jsonToObject<T>(json: string): T {
  27. return JSON.parse(json);
  28. }
  29. //
  30. // Database helpers
  31. //
  32. export function getSQLiteVersion(db: Database): string {
  33. const { sqlite_version: version } = db
  34. .prepare<EmptyQuery>('select sqlite_version() AS sqlite_version')
  35. .get();
  36. return version;
  37. }
  38. export function getSchemaVersion(db: Database): number {
  39. return db.pragma('schema_version', { simple: true });
  40. }
  41. export function setUserVersion(db: Database, version: number): void {
  42. if (!isNumber(version)) {
  43. throw new Error(`setUserVersion: version ${version} is not a number`);
  44. }
  45. db.pragma(`user_version = ${version}`);
  46. }
  47. export function getUserVersion(db: Database): number {
  48. return db.pragma('user_version', { simple: true });
  49. }
  50. export function getSQLCipherVersion(db: Database): string | undefined {
  51. return db.pragma('cipher_version', { simple: true });
  52. }
  53. //
  54. // Various table helpers
  55. //
  56. export function batchMultiVarQuery<ValueT>(
  57. db: Database,
  58. values: Array<ValueT>,
  59. query: (batch: Array<ValueT>) => void
  60. ): [];
  61. export function batchMultiVarQuery<ValueT, ResultT>(
  62. db: Database,
  63. values: Array<ValueT>,
  64. query: (batch: Array<ValueT>) => Array<ResultT>
  65. ): Array<ResultT>;
  66. export function batchMultiVarQuery<ValueT, ResultT>(
  67. db: Database,
  68. values: Array<ValueT>,
  69. query:
  70. | ((batch: Array<ValueT>) => void)
  71. | ((batch: Array<ValueT>) => Array<ResultT>)
  72. ): Array<ResultT> {
  73. if (values.length > MAX_VARIABLE_COUNT) {
  74. const result: Array<ResultT> = [];
  75. db.transaction(() => {
  76. for (let i = 0; i < values.length; i += MAX_VARIABLE_COUNT) {
  77. const batch = values.slice(i, i + MAX_VARIABLE_COUNT);
  78. const batchResult = query(batch);
  79. if (Array.isArray(batchResult)) {
  80. result.push(...batchResult);
  81. }
  82. }
  83. })();
  84. return result;
  85. }
  86. const result = query(values);
  87. return Array.isArray(result) ? result : [];
  88. }
  89. export function createOrUpdate<Key extends string | number>(
  90. db: Database,
  91. table: TableType,
  92. data: Record<string, unknown> & { id: Key }
  93. ): void {
  94. const { id } = data;
  95. if (!id) {
  96. throw new Error('createOrUpdate: Provided data did not have a truthy id');
  97. }
  98. db.prepare<Query>(
  99. `
  100. INSERT OR REPLACE INTO ${table} (
  101. id,
  102. json
  103. ) values (
  104. $id,
  105. $json
  106. )
  107. `
  108. ).run({
  109. id,
  110. json: objectToJSON(data),
  111. });
  112. }
  113. export function bulkAdd(
  114. db: Database,
  115. table: TableType,
  116. array: Array<Record<string, unknown> & { id: string | number }>
  117. ): void {
  118. db.transaction(() => {
  119. for (const data of array) {
  120. createOrUpdate(db, table, data);
  121. }
  122. })();
  123. }
  124. export function getById<Key extends string | number, Result = unknown>(
  125. db: Database,
  126. table: TableType,
  127. id: Key
  128. ): Result | undefined {
  129. const row = db
  130. .prepare<Query>(
  131. `
  132. SELECT *
  133. FROM ${table}
  134. WHERE id = $id;
  135. `
  136. )
  137. .get({
  138. id,
  139. });
  140. if (!row) {
  141. return undefined;
  142. }
  143. return jsonToObject(row.json);
  144. }
  145. export function removeById<Key extends string | number>(
  146. db: Database,
  147. table: TableType,
  148. id: Key | Array<Key>
  149. ): void {
  150. if (!Array.isArray(id)) {
  151. db.prepare<Query>(
  152. `
  153. DELETE FROM ${table}
  154. WHERE id = $id;
  155. `
  156. ).run({ id });
  157. return;
  158. }
  159. if (!id.length) {
  160. throw new Error('removeById: No ids to delete!');
  161. }
  162. const removeByIdsSync = (ids: Array<string | number>): void => {
  163. db.prepare<ArrayQuery>(
  164. `
  165. DELETE FROM ${table}
  166. WHERE id IN ( ${id.map(() => '?').join(', ')} );
  167. `
  168. ).run(ids);
  169. };
  170. batchMultiVarQuery(db, id, removeByIdsSync);
  171. }
  172. export function removeAllFromTable(db: Database, table: TableType): void {
  173. db.prepare<EmptyQuery>(`DELETE FROM ${table};`).run();
  174. }
  175. export function getAllFromTable<T>(db: Database, table: TableType): Array<T> {
  176. const rows: JSONRows = db
  177. .prepare<EmptyQuery>(`SELECT json FROM ${table};`)
  178. .all();
  179. return rows.map(row => jsonToObject(row.json));
  180. }
  181. export function getCountFromTable(db: Database, table: TableType): number {
  182. const result: null | number = db
  183. .prepare<EmptyQuery>(`SELECT count(*) from ${table};`)
  184. .pluck(true)
  185. .get();
  186. if (isNumber(result)) {
  187. return result;
  188. }
  189. throw new Error(`getCountFromTable: Unable to get count from table ${table}`);
  190. }
  191. export class TableIterator<ObjectType extends { id: string }> {
  192. constructor(
  193. private readonly db: Database,
  194. private readonly table: TableType,
  195. private readonly pageSize = 500
  196. ) {}
  197. *[Symbol.iterator](): Iterator<ObjectType> {
  198. const fetchObject = this.db.prepare<Query>(
  199. `
  200. SELECT json FROM ${this.table}
  201. WHERE id > $id
  202. ORDER BY id ASC
  203. LIMIT $pageSize;
  204. `
  205. );
  206. let complete = false;
  207. let id = '';
  208. while (!complete) {
  209. const rows: JSONRows = fetchObject.all({
  210. id,
  211. pageSize: this.pageSize,
  212. });
  213. const messages: Array<ObjectType> = rows.map(row =>
  214. jsonToObject(row.json)
  215. );
  216. yield* messages;
  217. const lastMessage: ObjectType | undefined = last(messages);
  218. if (lastMessage) {
  219. ({ id } = lastMessage);
  220. }
  221. complete = messages.length < this.pageSize;
  222. }
  223. }
  224. }