From b4e6b8b01defdbdbbe64d11817d4070077c56c8d Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 8 Aug 2017 16:12:46 -0300 Subject: [PATCH] Merge pull request #7456 from seekingalpha/csv-import [FIX] Csv importer: work with more problematic data --- .../.npm/package/npm-shrinkwrap.json | 6 +- packages/rocketchat-importer-csv/package.js | 2 +- packages/rocketchat-importer-csv/server.js | 61 +++++++++++++++++-- .../server.js | 6 +- .../rocketchat-importer-hipchat/server.js | 6 +- packages/rocketchat-importer-slack/server.js | 6 +- .../client/admin/adminImportPrepare.html | 56 +++++++++-------- .../client/admin/adminImportPrepare.js | 6 ++ .../server/classes/ImporterSelection.js | 4 +- 9 files changed, 114 insertions(+), 39 deletions(-) diff --git a/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json b/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json index 7a0f451ebb1c..324c8efc5f33 100644 --- a/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json +++ b/packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json @@ -1,9 +1,9 @@ { "dependencies": { "csv-parse": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-1.1.7.tgz", - "from": "csv-parse@1.1.7" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-1.2.0.tgz", + "from": "csv-parse@1.2.0" } } } diff --git a/packages/rocketchat-importer-csv/package.js b/packages/rocketchat-importer-csv/package.js index 33976fbeb100..fb11d108fb55 100644 --- a/packages/rocketchat-importer-csv/package.js +++ b/packages/rocketchat-importer-csv/package.js @@ -17,5 +17,5 @@ Package.onUse(function(api) { }); Npm.depends({ - 'csv-parse': '1.1.7' + 'csv-parse': '1.2.0' }); diff --git a/packages/rocketchat-importer-csv/server.js b/packages/rocketchat-importer-csv/server.js index f701075e6a6c..3597f1df6616 100644 --- a/packages/rocketchat-importer-csv/server.js +++ b/packages/rocketchat-importer-csv/server.js @@ -131,9 +131,10 @@ Importer.CSV = class ImporterCSV extends Importer.Base { const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true)); const selectionChannels = tempChannels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate)); + const selectionMessages = this.importRecord.count.messages; super.updateProgress(Importer.ProgressStep.USER_SELECTION); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } startImport(importSelection) { @@ -232,16 +233,60 @@ Importer.CSV = class ImporterCSV extends Importer.Base { } this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }}); + //If no channels file, collect channel map from DB for message-only import + if (this.channels.channels.length === 0) { + for (const cname of this.messages.keys()) { + Meteor.runAsUser(startedByUserId, () => { + const existantRoom = RocketChat.models.Rooms.findOneByName(cname); + if (existantRoom || cname.toUpperCase() === 'GENERAL') { + this.channels.channels.push({ + id: cname.replace('.', '_'), + name: cname, + rocketId: (cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id), + do_import: true + }); + } + }); + } + } + + //If no users file, collect user map from DB for message-only import + if (this.users.users.length === 0) { + for (const [ch, messagesMap] of this.messages.entries()) { + const csvChannel = this.getChannelFromName(ch); + if (!csvChannel || !csvChannel.do_import) { + continue; + } + Meteor.runAsUser(startedByUserId, () => { + for (const msgs of messagesMap.values()) { + for (const msg of msgs.messages) { + if (!this.getUserFromUsername(msg.username)) { + const user = RocketChat.models.Users.findOneByUsername(msg.username); + if (user) { + this.users.users.push({ + rocketId: user._id, + username: user.username + }); + } + } + } + } + }); + } + } + + //Import the Messages super.updateProgress(Importer.ProgressStep.IMPORTING_MESSAGES); for (const [ch, messagesMap] of this.messages.entries()) { const csvChannel = this.getChannelFromName(ch); - if (!csvChannel.do_import) { + if (!csvChannel || !csvChannel.do_import) { continue; } const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } }); Meteor.runAsUser(startedByUserId, () => { + const timestamps = {}; for (const [msgGroupData, msgs] of messagesMap.entries()) { super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` }); for (const msg of msgs.messages) { @@ -253,8 +298,15 @@ Importer.CSV = class ImporterCSV extends Importer.Base { const creator = this.getUserFromUsername(msg.username); if (creator) { + let suffix = ''; + if (timestamps[msg.ts] === undefined) { + timestamps[msg.ts] = 1; + } else { + suffix = `-${ timestamps[msg.ts] }`; + timestamps[msg.ts] += 1; + } const msgObj = { - _id: `csv-${ csvChannel.id }-${ msg.ts }`, + _id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`, ts: new Date(parseInt(msg.ts)), msg: msg.text, rid: room._id, @@ -285,8 +337,9 @@ Importer.CSV = class ImporterCSV extends Importer.Base { getSelection() { const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true)); const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate)); + const selectionMessages = this.importRecord.count.messages; - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } getChannelFromName(channelName) { diff --git a/packages/rocketchat-importer-hipchat-enterprise/server.js b/packages/rocketchat-importer-hipchat-enterprise/server.js index 40f3762df621..961cdd26b222 100644 --- a/packages/rocketchat-importer-hipchat-enterprise/server.js +++ b/packages/rocketchat-importer-hipchat-enterprise/server.js @@ -190,10 +190,11 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, u.isDeleted, false, true)); const selectionChannels = tempRooms.map((r) => new Importer.SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate)); + const selectionMessages = this.importRecord.count.messages; super.updateProgress(Importer.ProgressStep.USER_SELECTION); - resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels)); + resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages)); })); //Wish I could make this cleaner :( @@ -431,8 +432,9 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba getSelection() { const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true)); const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate)); + const selectionMessages = this.importRecord.count.messages; - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } getChannelFromRoomIdentifier(roomIdentifier) { diff --git a/packages/rocketchat-importer-hipchat/server.js b/packages/rocketchat-importer-hipchat/server.js index 89713ce84c11..007e90efa078 100644 --- a/packages/rocketchat-importer-hipchat/server.js +++ b/packages/rocketchat-importer-hipchat/server.js @@ -131,8 +131,9 @@ Importer.HipChat = Importer.HipChat = (function() { const selectionChannels = tempRooms.map(function(room) { return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false); }); + const selectionMessages = this.importRecord.count.messages; this.updateProgress(Importer.ProgressStep.USER_SELECTION); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } startImport(importSelection) { @@ -331,7 +332,8 @@ Importer.HipChat = Importer.HipChat = (function() { const selectionChannels = this.channels.channels.map(function(room) { return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false); }); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + const selectionMessages = this.importRecord.count.messages; + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } } diff --git a/packages/rocketchat-importer-slack/server.js b/packages/rocketchat-importer-slack/server.js index 1ff1fc3eb3bc..ea12a7928bdb 100644 --- a/packages/rocketchat-importer-slack/server.js +++ b/packages/rocketchat-importer-slack/server.js @@ -92,8 +92,9 @@ Importer.Slack = class extends Importer.Base { } const selectionUsers = tempUsers.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot)); const selectionChannels = tempChannels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false)); + const selectionMessages = this.importRecord.count.messages; this.updateProgress(Importer.ProgressStep.USER_SELECTION); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } startImport(importSelection) { super.startImport(importSelection); @@ -431,6 +432,7 @@ Importer.Slack = class extends Importer.Base { getSelection() { const selectionUsers = this.users.users.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot)); const selectionChannels = this.channels.channels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false)); - return new Importer.Selection(this.name, selectionUsers, selectionChannels); + const selectionMessages = this.importRecord.count.messages; + return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages); } }; diff --git a/packages/rocketchat-importer/client/admin/adminImportPrepare.html b/packages/rocketchat-importer/client/admin/adminImportPrepare.html index cac461eb89e7..82efec25c976 100644 --- a/packages/rocketchat-importer/client/admin/adminImportPrepare.html +++ b/packages/rocketchat-importer/client/admin/adminImportPrepare.html @@ -33,35 +33,43 @@

{{_ "Actions"}}

-
-

{{_ "Users"}}

-
-
    - {{#each users}} - {{#unless is_bot}} + {{#if users.length}} +
    +

    {{_ "Users"}}

    +
    +
      + {{#each users}} + {{#unless is_bot}} +
    • + {{username}} - {{email}} + {{#if is_deleted }} ({{_ "Deleted"}}){{/if}} +
      • + {{/unless}} + {{/each}} +
    +
    +
    + {{/if}} + + {{#if channels.length}} +
    +

    {{_ "Channels"}}

    +
    +
      + {{#each channels}}
    • - {{username}} - {{email}} - {{#if is_deleted }} ({{_ "Deleted"}}){{/if}} + {{name}} + {{#if is_archived}} ({{_ "Importer_Archived"}}){{/if}} + {{#if is_private}} ({{_ "Private_Group"}}){{/if}}
      • - {{/unless}} - {{/each}} -
    + {{/each}} +
+
- + {{/if}}
-

{{_ "Channels"}}

-
-
    - {{#each channels}} -
  • - {{name}} - {{#if is_archived}} ({{_ "Importer_Archived"}}){{/if}} - {{#if is_private}} ({{_ "Private_Group"}}){{/if}} -
    • - {{/each}} -
-
+

{{_ "Messages"}}: {{message_count}}

{{else}} {{#if isPreparing}} diff --git a/packages/rocketchat-importer/client/admin/adminImportPrepare.js b/packages/rocketchat-importer/client/admin/adminImportPrepare.js index fe7e00ee045f..bdc11eb442f7 100644 --- a/packages/rocketchat-importer/client/admin/adminImportPrepare.js +++ b/packages/rocketchat-importer/client/admin/adminImportPrepare.js @@ -27,6 +27,9 @@ Template.adminImportPrepare.helpers({ }, channels() { return Template.instance().channels.get(); + }, + message_count() { + return Template.instance().message_count.get(); } }); @@ -70,6 +73,7 @@ Template.adminImportPrepare.events({ template.users.set(data.users); template.channels.set(data.channels); + template.message_count.set(data.message_count); template.loaded.set(true); template.preparing.set(false); }); @@ -131,6 +135,7 @@ Template.adminImportPrepare.onCreated(function() { this.loaded = new ReactiveVar(false); this.users = new ReactiveVar([]); this.channels = new ReactiveVar([]); + this.message_count = new ReactiveVar(0); function loadSelection(progress) { if ((progress != null ? progress.step : undefined)) { @@ -146,6 +151,7 @@ Template.adminImportPrepare.onCreated(function() { } instance.users.set(data.users); instance.channels.set(data.channels); + instance.message_count.set(data.message_count); instance.loaded.set(true); return instance.preparing.set(false); }); diff --git a/packages/rocketchat-importer/server/classes/ImporterSelection.js b/packages/rocketchat-importer/server/classes/ImporterSelection.js index 070af9a70be6..d1a45a56a972 100644 --- a/packages/rocketchat-importer/server/classes/ImporterSelection.js +++ b/packages/rocketchat-importer/server/classes/ImporterSelection.js @@ -6,10 +6,12 @@ Importer.Selection = (Importer.Selection = class Selection { // @param [String] name the name of the Importer // @param [Array] users the array of users // @param [Array] channels the array of channels + // @param [Integer] number of collected messages // - constructor(name, users, channels) { + constructor(name, users, channels, message_count) { this.name = name; this.users = users; this.channels = channels; + this.message_count = message_count; } });