PageRenderTime 46ms CodeModel.GetById 23ms RepoModel.GetById 0ms app.codeStats 0ms

/packages/rocketchat-importer-slack/server.coffee

https://gitlab.com/brandnewcampaign/townsquare-chat
CoffeeScript | 320 lines | 260 code | 40 blank | 20 comment | 78 complexity | 4d2d134467d96d77a4b01b0a584a032d MD5 | raw file
  1. Importer.Slack = class Importer.Slack extends Importer.Base
  2. constructor: (name, descriptionI18N, fileTypeRegex) ->
  3. super(name, descriptionI18N, fileTypeRegex)
  4. @userTags = []
  5. @bots = {}
  6. prepare: (dataURI, sentContentType, fileName) =>
  7. super(dataURI, sentContentType, fileName)
  8. # try
  9. {image, contentType} = RocketChatFile.dataURIParse dataURI
  10. zip = new @AdmZip(new Buffer(image, 'base64'))
  11. zipEntries = zip.getEntries()
  12. tempChannels = []
  13. tempUsers = []
  14. tempMessages = {}
  15. for entry in zipEntries
  16. do (entry) =>
  17. if entry.entryName == 'channels.json'
  18. @updateProgress Importer.ProgressStep.PREPARING_CHANNELS
  19. tempChannels = JSON.parse entry.getData().toString()
  20. else if entry.entryName == 'users.json'
  21. @updateProgress Importer.ProgressStep.PREPARING_USERS
  22. tempUsers = JSON.parse entry.getData().toString()
  23. for user in tempUsers when user.is_bot
  24. @bots[user.profile.bot_id] = user
  25. else if not entry.isDirectory and entry.entryName.indexOf('/') > -1
  26. item = entry.entryName.split('/') #random/2015-10-04.json
  27. channelName = item[0] #random
  28. msgGroupData = item[1].split('.')[0] #2015-10-04
  29. if not tempMessages[channelName]
  30. tempMessages[channelName] = {}
  31. # Catch files which aren't valid JSON files, ignore them
  32. try
  33. tempMessages[channelName][msgGroupData] = JSON.parse entry.getData().toString()
  34. catch
  35. console.warn "#{entry.entryName} is not a valid JSON file! Unable to import it."
  36. # Insert the users record, eventually this might have to be split into several ones as well
  37. # if someone tries to import a several thousands users instance
  38. usersId = @collection.insert { 'import': @importRecord._id, 'importer': @name, 'type': 'users', 'users': tempUsers }
  39. @users = @collection.findOne usersId
  40. @updateRecord { 'count.users': tempUsers.length }
  41. @addCountToTotal tempUsers.length
  42. # Insert the channels records.
  43. channelsId = @collection.insert { 'import': @importRecord._id, 'importer': @name, 'type': 'channels', 'channels': tempChannels }
  44. @channels = @collection.findOne channelsId
  45. @updateRecord { 'count.channels': tempChannels.length }
  46. @addCountToTotal tempChannels.length
  47. # Insert the messages records
  48. @updateProgress Importer.ProgressStep.PREPARING_MESSAGES
  49. messagesCount = 0
  50. for channel, messagesObj of tempMessages
  51. do (channel, messagesObj) =>
  52. if not @messages[channel]
  53. @messages[channel] = {}
  54. for date, msgs of messagesObj
  55. messagesCount += msgs.length
  56. @updateRecord { 'messagesstatus': "#{channel}/#{date}" }
  57. if Importer.Base.getBSONSize(msgs) > Importer.Base.MaxBSONSize
  58. for splitMsg, i in Importer.Base.getBSONSafeArraysFromAnArray(msgs)
  59. messagesId = @collection.insert { 'import': @importRecord._id, 'importer': @name, 'type': 'messages', 'name': "#{channel}/#{date}.#{i}", 'messages': splitMsg }
  60. @messages[channel]["#{date}.#{i}"] = @collection.findOne messagesId
  61. else
  62. messagesId = @collection.insert { 'import': @importRecord._id, 'importer': @name, 'type': 'messages', 'name': "#{channel}/#{date}", 'messages': msgs }
  63. @messages[channel][date] = @collection.findOne messagesId
  64. @updateRecord { 'count.messages': messagesCount, 'messagesstatus': null }
  65. @addCountToTotal messagesCount
  66. if tempUsers.length is 0 or tempChannels.length is 0 or messagesCount is 0
  67. @updateProgress Importer.ProgressStep.ERROR
  68. return @getProgress()
  69. selectionUsers = tempUsers.map (user) ->
  70. return new Importer.SelectionUser user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot
  71. selectionChannels = tempChannels.map (channel) ->
  72. return new Importer.SelectionChannel channel.id, channel.name, channel.is_archived, true
  73. @updateProgress Importer.ProgressStep.USER_SELECTION
  74. return new Importer.Selection @name, selectionUsers, selectionChannels
  75. # catch error
  76. # @updateRecord { 'failed': true, 'error': error }
  77. # console.error Importer.ProgressStep.ERROR
  78. # throw new Error 'import-slack-error', error
  79. startImport: (importSelection) =>
  80. super(importSelection)
  81. start = Date.now()
  82. for user in importSelection.users
  83. for u in @users.users when u.id is user.user_id
  84. u.do_import = user.do_import
  85. @collection.update { _id: @users._id }, { $set: { 'users': @users.users }}
  86. for channel in importSelection.channels
  87. for c in @channels.channels when c.id is channel.channel_id
  88. c.do_import = channel.do_import
  89. @collection.update { _id: @channels._id }, { $set: { 'channels': @channels.channels }}
  90. startedByUserId = Meteor.userId()
  91. Meteor.defer =>
  92. # try
  93. @updateProgress Importer.ProgressStep.IMPORTING_USERS
  94. for user in @users.users when user.do_import
  95. do (user) =>
  96. Meteor.runAsUser startedByUserId, () =>
  97. existantUser = RocketChat.models.Users.findOneByEmailAddress user.profile.email
  98. if existantUser
  99. user.rocketId = existantUser._id
  100. @userTags.push
  101. slack: "<@#{user.id}>"
  102. slackLong: "<@#{user.id}|#{user.name}>"
  103. rocket: "@#{existantUser.username}"
  104. else
  105. userId = Accounts.createUser { email: user.profile.email, password: Date.now() + user.name + user.profile.email.toUpperCase() }
  106. Meteor.runAsUser userId, () =>
  107. Meteor.call 'setUsername', user.name
  108. Meteor.call 'joinDefaultChannels', true
  109. url = null
  110. if user.profile.image_original
  111. url = user.profile.image_original
  112. else if user.profile.image_512
  113. url = user.profile.image_512
  114. Meteor.call 'setAvatarFromService', url, null, 'url'
  115. # Slack's is -18000 which translates to Rocket.Chat's after dividing by 3600
  116. if user.tz_offset
  117. Meteor.call 'updateUserUtcOffset', user.tz_offset / 3600
  118. if user.profile.real_name
  119. RocketChat.models.Users.setName userId, user.profile.real_name
  120. #Deleted users are 'inactive' users in Rocket.Chat
  121. if user.deleted
  122. Meteor.call 'setUserActiveStatus', userId, false
  123. #TODO: Maybe send emails?
  124. user.rocketId = userId
  125. @userTags.push
  126. slack: "<@#{user.id}>"
  127. slackLong: "<@#{user.id}|#{user.name}>"
  128. rocket: "@#{user.name}"
  129. @addCountCompleted 1
  130. @collection.update { _id: @users._id }, { $set: { 'users': @users.users }}
  131. @updateProgress Importer.ProgressStep.IMPORTING_CHANNELS
  132. for channel in @channels.channels when channel.do_import
  133. do (channel) =>
  134. Meteor.runAsUser startedByUserId, () =>
  135. existantRoom = RocketChat.models.Rooms.findOneByName channel.name
  136. if existantRoom or channel.is_general
  137. if channel.is_general and channel.name isnt existantRoom?.name
  138. Meteor.call 'saveRoomSettings', 'GENERAL', 'roomName', channel.name
  139. channel.rocketId = if channel.is_general then 'GENERAL' else existantRoom._id
  140. else
  141. users = []
  142. for member in channel.members when member isnt channel.creator
  143. user = @getRocketUser member
  144. if user?
  145. users.push user.username
  146. userId = ''
  147. for user in @users.users when user.id is channel.creator
  148. userId = user.rocketId
  149. Meteor.runAsUser userId, () =>
  150. returned = Meteor.call 'createChannel', channel.name, users
  151. channel.rocketId = returned.rid
  152. # @TODO implement model specific function
  153. roomUpdate =
  154. ts: new Date(channel.created * 1000)
  155. if not _.isEmpty channel.topic?.value
  156. roomUpdate.topic = channel.topic.value
  157. lastSetTopic = channel.topic.last_set
  158. if not _.isEmpty(channel.purpose?.value) and channel.purpose.last_set > lastSetTopic
  159. roomUpdate.topic = channel.purpose.value
  160. RocketChat.models.Rooms.update { _id: channel.rocketId }, { $set: roomUpdate }
  161. @addCountCompleted 1
  162. @collection.update { _id: @channels._id }, { $set: { 'channels': @channels.channels }}
  163. missedTypes = {}
  164. ignoreTypes = { 'bot_add': true, 'file_comment': true, 'file_mention': true, 'channel_name': true }
  165. @updateProgress Importer.ProgressStep.IMPORTING_MESSAGES
  166. for channel, messagesObj of @messages
  167. do (channel, messagesObj) =>
  168. Meteor.runAsUser startedByUserId, () =>
  169. slackChannel = @getSlackChannelFromName channel
  170. if slackChannel?.do_import
  171. room = RocketChat.models.Rooms.findOneById slackChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }
  172. for date, msgs of messagesObj
  173. @updateRecord { 'messagesstatus': "#{channel}/#{date}.#{msgs.messages.length}" }
  174. for message in msgs.messages
  175. if message.type is 'message'
  176. if message.subtype?
  177. if message.subtype is 'channel_join'
  178. if @getRocketUser(message.user)?
  179. RocketChat.models.Messages.createUserJoinWithRoomIdAndUser room._id, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
  180. else if message.subtype is 'channel_leave'
  181. if @getRocketUser(message.user)?
  182. RocketChat.models.Messages.createUserLeaveWithRoomIdAndUser room._id, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
  183. else if message.subtype is 'me_message'
  184. RocketChat.sendMessage @getRocketUser(message.user), { msg: '_' + @convertSlackMessageToRocketChat(message.text) + '_', ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }, room
  185. else if message.subtype is 'bot_message'
  186. botUser = RocketChat.models.Users.findOneById 'rocket.cat', { fields: { username: 1 }}
  187. botUsername = if @bots[message.bot_id] then @bots[message.bot_id]?.name else message.username
  188. msgObj =
  189. msg: if message.text then message.text else ''
  190. ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)
  191. rid: room._id
  192. bot: true
  193. attachments: message.attachments
  194. username: if botUsername then botUsername else undefined
  195. if message.edited?
  196. msgObj.ets = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000)
  197. if message.icons?
  198. msgObj.emoji = message.icons.emoji
  199. msgObj.msg = @convertSlackMessageToRocketChat(msgObj.msg)
  200. RocketChat.sendMessage botUser, msgObj, room
  201. else if message.subtype is 'channel_purpose'
  202. RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser 'room_changed_topic', room._id, message.purpose, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
  203. else if message.subtype is 'channel_topic'
  204. RocketChat.models.Messages.createRoomSettingsChangedWithTypeRoomIdMessageAndUser 'room_changed_topic', room._id, message.topic, @getRocketUser(message.user), { ts: new Date(parseInt(message.ts.split('.')[0]) * 1000) }
  205. else if message.subtype is 'pinned_item'
  206. RocketChat.models.Messages.createWithTypeRoomIdMessageAndUser 'message_pinned', room._id, '', @getRocketUser(message.user),
  207. ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)
  208. attachments: [
  209. "text" : @convertSlackMessageToRocketChat message.attachments[0].text
  210. "author_name" : message.attachments[0].author_subname
  211. "author_icon" : getAvatarUrlFromUsername(message.attachments[0].author_subname)
  212. ]
  213. else if message.subtype is 'file_share'
  214. if message.file?.url_private_download isnt undefined
  215. details =
  216. name: message.file.name
  217. size: message.file.size
  218. type: message.file.mimetype
  219. rid: room._id
  220. @uploadFile details, message.file.url_private_download, @getRocketUser(message.user), room, new Date(parseInt(message.ts.split('.')[0]) * 1000)
  221. else
  222. if not missedTypes[message.subtype] and not ignoreTypes[message.subtype]
  223. missedTypes[message.subtype] = message
  224. else
  225. user = @getRocketUser(message.user)
  226. if user?
  227. msgObj =
  228. msg: @convertSlackMessageToRocketChat message.text
  229. ts: new Date(parseInt(message.ts.split('.')[0]) * 1000)
  230. rid: room._id
  231. u:
  232. _id: user._id
  233. username: user.username
  234. if message.edited?
  235. msgObj.ets = new Date(parseInt(message.edited.ts.split('.')[0]) * 1000)
  236. RocketChat.sendMessage @getRocketUser(message.user), msgObj, room
  237. @addCountCompleted 1
  238. console.log missedTypes
  239. @updateProgress Importer.ProgressStep.FINISHING
  240. for channel in @channels.channels when channel.do_import and channel.is_archived
  241. do (channel) =>
  242. Meteor.runAsUser startedByUserId, () =>
  243. Meteor.call 'archiveRoom', channel.rocketId
  244. @updateProgress Importer.ProgressStep.DONE
  245. timeTook = Date.now() - start
  246. console.log "Import took #{timeTook} milliseconds."
  247. # catch error
  248. # @updateRecord { 'failed': true, 'error': error }
  249. # @updateProgress Importer.ProgressStep.ERROR
  250. # console.error Importer.ProgressStep.ERROR
  251. # throw new Error 'import-slack-error', error
  252. return @getProgress()
  253. getSlackChannelFromName: (channelName) =>
  254. for channel in @channels.channels when channel.name is channelName
  255. return channel
  256. getRocketUser: (slackId) =>
  257. for user in @users.users when user.id is slackId
  258. return RocketChat.models.Users.findOneById user.rocketId, { fields: { username: 1 }}
  259. convertSlackMessageToRocketChat: (message) =>
  260. if message?
  261. message = message.replace /<!everyone>/g, '@all'
  262. message = message.replace /<!channel>/g, '@all'
  263. message = message.replace /&gt;/g, '<'
  264. message = message.replace /&lt;/g, '>'
  265. message = message.replace /&amp;/g, '&'
  266. message = message.replace /:simple_smile:/g, ':smile:'
  267. message = message.replace /:memo:/g, ':pencil:'
  268. message = message.replace /:piggy:/g, ':pig:'
  269. message = message.replace /:uk:/g, ':gb:'
  270. message = message.replace /<(http[s]?:[^>]*)>/g, '$1'
  271. for userReplace in @userTags
  272. message = message.replace userReplace.slack, userReplace.rocket
  273. message = message.replace userReplace.slackLong, userReplace.rocket
  274. return message
  275. getSelection: () =>
  276. selectionUsers = @users.users.map (user) ->
  277. return new Importer.SelectionUser user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot
  278. selectionChannels = @channels.channels.map (channel) ->
  279. return new Importer.SelectionChannel channel.id, channel.name, channel.is_archived, true
  280. return new Importer.Selection @name, selectionUsers, selectionChannels