Skip to content

Commit

Permalink
Merge pull request #7456 from seekingalpha/csv-import
Browse files Browse the repository at this point in the history 
[FIX] Csv importer: work with more problematic data
  • Loading branch information
@rodrigok
rodrigok committed Aug 14, 2017
1 parent 16926f8 commit b4e6b8b
Show file tree
Hide file tree
Showing 9 changed files with 114 additions and 39 deletions.
6 changes: 3 additions & 3 deletions packages/rocketchat-importer-csv/.npm/package/npm-shrinkwrap.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/rocketchat-importer-csv/package.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,5 +17,5 @@ Package.onUse(function(api) {
});

Npm.depends({
'csv-parse': '1.1.7'
'csv-parse': '1.2.0'
});
61 changes: 57 additions & 4 deletions packages/rocketchat-importer-csv/server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,10 @@ Importer.CSV = class ImporterCSV extends Importer.Base {

const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = tempChannels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;

super.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

startImport(importSelection) {
Expand Down Expand Up @@ -232,16 +233,60 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
}
this.collection.update({ _id: this.channels._id }, { $set: { 'channels': this.channels.channels }});

//If no channels file, collect channel map from DB for message-only import
if (this.channels.channels.length === 0) {
for (const cname of this.messages.keys()) {
Meteor.runAsUser(startedByUserId, () => {
const existantRoom = RocketChat.models.Rooms.findOneByName(cname);
if (existantRoom || cname.toUpperCase() === 'GENERAL') {
this.channels.channels.push({
id: cname.replace('.', '_'),
name: cname,
rocketId: (cname.toUpperCase() === 'GENERAL' ? 'GENERAL' : existantRoom._id),
do_import: true
});
}
});
}
}

//If no users file, collect user map from DB for message-only import
if (this.users.users.length === 0) {
for (const [ch, messagesMap] of this.messages.entries()) {
const csvChannel = this.getChannelFromName(ch);
if (!csvChannel || !csvChannel.do_import) {
continue;
}
Meteor.runAsUser(startedByUserId, () => {
for (const msgs of messagesMap.values()) {
for (const msg of msgs.messages) {
if (!this.getUserFromUsername(msg.username)) {
const user = RocketChat.models.Users.findOneByUsername(msg.username);
if (user) {
this.users.users.push({
rocketId: user._id,
username: user.username
});
}
}
}
}
});
}
}


//Import the Messages
super.updateProgress(Importer.ProgressStep.IMPORTING_MESSAGES);
for (const [ch, messagesMap] of this.messages.entries()) {
const csvChannel = this.getChannelFromName(ch);
if (!csvChannel.do_import) {
if (!csvChannel || !csvChannel.do_import) {
continue;
}

const room = RocketChat.models.Rooms.findOneById(csvChannel.rocketId, { fields: { usernames: 1, t: 1, name: 1 } });
Meteor.runAsUser(startedByUserId, () => {
const timestamps = {};
for (const [msgGroupData, msgs] of messagesMap.entries()) {
super.updateRecord({ 'messagesstatus': `${ ch }/${ msgGroupData }.${ msgs.messages.length }` });
for (const msg of msgs.messages) {
Expand All @@ -253,8 +298,15 @@ Importer.CSV = class ImporterCSV extends Importer.Base {

const creator = this.getUserFromUsername(msg.username);
if (creator) {
let suffix = '';
if (timestamps[msg.ts] === undefined) {
timestamps[msg.ts] = 1;
} else {
suffix = `-${ timestamps[msg.ts] }`;
timestamps[msg.ts] += 1;
}
const msgObj = {
_id: `csv-${ csvChannel.id }-${ msg.ts }`,
_id: `csv-${ csvChannel.id }-${ msg.ts }${ suffix }`,
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
rid: room._id,
Expand Down Expand Up @@ -285,8 +337,9 @@ Importer.CSV = class ImporterCSV extends Importer.Base {
getSelection() {
const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;

return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

getChannelFromName(channelName) {
Expand Down
6 changes: 4 additions & 2 deletions packages/rocketchat-importer-hipchat-enterprise/server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,10 +190,11 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba

const selectionUsers = tempUsers.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, u.isDeleted, false, true));
const selectionChannels = tempRooms.map((r) => new Importer.SelectionChannel(r.id, r.name, r.isArchived, true, r.isPrivate));
const selectionMessages = this.importRecord.count.messages;

super.updateProgress(Importer.ProgressStep.USER_SELECTION);

resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels));
resolve(new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages));
}));

//Wish I could make this cleaner :(
Expand Down Expand Up @@ -431,8 +432,9 @@ Importer.HipChatEnterprise = class ImporterHipChatEnterprise extends Importer.Ba
getSelection() {
const selectionUsers = this.users.users.map((u) => new Importer.SelectionUser(u.id, u.username, u.email, false, false, true));
const selectionChannels = this.channels.channels.map((c) => new Importer.SelectionChannel(c.id, c.name, false, true, c.isPrivate));
const selectionMessages = this.importRecord.count.messages;

return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

getChannelFromRoomIdentifier(roomIdentifier) {
Expand Down
6 changes: 4 additions & 2 deletions packages/rocketchat-importer-hipchat/server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,9 @@ Importer.HipChat = Importer.HipChat = (function() {
const selectionChannels = tempRooms.map(function(room) {
return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false);
});
const selectionMessages = this.importRecord.count.messages;
this.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

startImport(importSelection) {
Expand Down Expand Up @@ -331,7 +332,8 @@ Importer.HipChat = Importer.HipChat = (function() {
const selectionChannels = this.channels.channels.map(function(room) {
return new Importer.SelectionChannel(room.room_id, room.name, room.is_archived, true, false);
});
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}

}
Expand Down
6 changes: 4 additions & 2 deletions packages/rocketchat-importer-slack/server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,9 @@ Importer.Slack = class extends Importer.Base {
}
const selectionUsers = tempUsers.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot));
const selectionChannels = tempChannels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false));
const selectionMessages = this.importRecord.count.messages;
this.updateProgress(Importer.ProgressStep.USER_SELECTION);
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
startImport(importSelection) {
super.startImport(importSelection);
Expand Down Expand Up @@ -431,6 +432,7 @@ Importer.Slack = class extends Importer.Base {
getSelection() {
const selectionUsers = this.users.users.map(user => new Importer.SelectionUser(user.id, user.name, user.profile.email, user.deleted, user.is_bot, !user.is_bot));
const selectionChannels = this.channels.channels.map(channel => new Importer.SelectionChannel(channel.id, channel.name, channel.is_archived, true, false));
return new Importer.Selection(this.name, selectionUsers, selectionChannels);
const selectionMessages = this.importRecord.count.messages;
return new Importer.Selection(this.name, selectionUsers, selectionChannels, selectionMessages);
}
};
56 changes: 32 additions & 24 deletions packages/rocketchat-importer/client/admin/adminImportPrepare.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,35 +33,43 @@ <h1>{{_ "Actions"}}</h1>
</div>
</div>

<div class="section">
<h1>{{_ "Users"}}</h1>
<div class="section-content">
<ul>
{{#each users}}
{{#unless is_bot}}
{{#if users.length}}
<div class="section">
<h1>{{_ "Users"}}</h1>
<div class="section-content">
<ul>
{{#each users}}
{{#unless is_bot}}
<li>
<input type="checkbox" name="{{user_id}}" checked="checked" />{{username}} - {{email}}
{{#if is_deleted }} <em>({{_ "Deleted"}})</em>{{/if}}
</li>
{{/unless}}
{{/each}}
</ul>
</div>
</div>
{{/if}}

{{#if channels.length}}
<div class="section">
<h1>{{_ "Channels"}}</h1>
<div class="section-content">
<ul>
{{#each channels}}
<li>
<input type="checkbox" name="{{user_id}}" checked="checked" />{{username}} - {{email}}
{{#if is_deleted }} <em>({{_ "Deleted"}})</em>{{/if}}
<input type="checkbox" name="{{channel_id}}" checked="checked" />{{name}}
{{#if is_archived}} <em>({{_ "Importer_Archived"}})</em>{{/if}}
{{#if is_private}} <em>({{_ "Private_Group"}})</em>{{/if}}
</li>
{{/unless}}
{{/each}}
</ul>
{{/each}}
</ul>
</div>
</div>
</div>
{{/if}}

<div class="section">
<h1>{{_ "Channels"}}</h1>
<div class="section-content">
<ul>
{{#each channels}}
<li>
<input type="checkbox" name="{{channel_id}}" checked="checked" />{{name}}
{{#if is_archived}} <em>({{_ "Importer_Archived"}})</em>{{/if}}
{{#if is_private}} <em>({{_ "Private_Group"}})</em>{{/if}}
</li>
{{/each}}
</ul>
</div>
<h1>{{_ "Messages"}}: {{message_count}}</h1>
</div>
{{else}}
{{#if isPreparing}}
Expand Down
6 changes: 6 additions & 0 deletions packages/rocketchat-importer/client/admin/adminImportPrepare.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ Template.adminImportPrepare.helpers({
},
channels() {
return Template.instance().channels.get();
},
message_count() {
return Template.instance().message_count.get();
}
});

Expand Down Expand Up @@ -70,6 +73,7 @@ Template.adminImportPrepare.events({

template.users.set(data.users);
template.channels.set(data.channels);
template.message_count.set(data.message_count);
template.loaded.set(true);
template.preparing.set(false);
});
Expand Down Expand Up @@ -131,6 +135,7 @@ Template.adminImportPrepare.onCreated(function() {
this.loaded = new ReactiveVar(false);
this.users = new ReactiveVar([]);
this.channels = new ReactiveVar([]);
this.message_count = new ReactiveVar(0);

function loadSelection(progress) {
if ((progress != null ? progress.step : undefined)) {
Expand All @@ -146,6 +151,7 @@ Template.adminImportPrepare.onCreated(function() {
}
instance.users.set(data.users);
instance.channels.set(data.channels);
instance.message_count.set(data.message_count);
instance.loaded.set(true);
return instance.preparing.set(false);
});
Expand Down
4 changes: 3 additions & 1 deletion packages/rocketchat-importer/server/classes/ImporterSelection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ Importer.Selection = (Importer.Selection = class Selection {
// @param [String] name the name of the Importer
// @param [Array<Importer.User>] users the array of users
// @param [Array<Importer.Channel>] channels the array of channels
// @param [Integer] number of collected messages
//
constructor(name, users, channels) {
constructor(name, users, channels, message_count) {
this.name = name;
this.users = users;
this.channels = channels;
this.message_count = message_count;
}
});

0 comments on commit b4e6b8b

Please sign in to comment.