Skip to content

Commit

Permalink
[NEW] Integrate DEEPL translation service to RC core (RocketChat#12174)
Browse files Browse the repository at this point in the history 
* service extension for deepl

* Add translations

* Add lint-fix-command

* Adhere to new linting rules

* Hard code service enpoint URLs of providers
Having it as a setting increases complexity (user has to enter the URL by himself) without significant benefit.
If the URL changes, the code has to be adapted anyway

* Fix translation of attachment descriptions by DeepL

* Fix getSupportedLanguages by Google

* move renameSetting to Settings model

* Add migrations for rename settings

* update package.json

* Fix formatting to make consistent with the other code

* Add additional comments

* Use active provider for context menu translations

* Add invidual key for each providers

* Nothing to migrate

* Fix spelling in comment

* Replace TAPi18n library with rocketchat:tapi18n

* Use correct setting
  • Loading branch information
@vickyokrm @mrsimpson
vickyokrm authored and mrsimpson committed Aug 27, 2019
1 parent 3233371 commit 1ff545c
Show file tree
Hide file tree
Showing 11 changed files with 3,126 additions and 3,045 deletions.
36 changes: 21 additions & 15 deletions app/autotranslate/server/autotranslate.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,17 @@ import { settings } from '../../settings';
import { callbacks } from '../../callbacks';
import { Subscriptions, Messages } from '../../models';
import { Markdown } from '../../markdown/server';
import { logger } from './logger';
import { Logger } from '../../logger';

/**
* This class allows translation providers to
* register,load and also returns the active provider.
*/
export class TranslationProviderRegistry {
/**
* Registers the translation provider into the registry.
* @param {*} provider
*/
static registerProvider(provider) {
// get provider information
const metadata = provider._getProviderMetadata();
Expand All @@ -18,21 +26,27 @@ export class TranslationProviderRegistry {
TranslationProviderRegistry._providers[metadata.name] = provider;
}

/**
* Return the active Translation provider
*/
static getActiveProvider() {
return TranslationProviderRegistry._providers[TranslationProviderRegistry._activeProvider];
}

/**
* Make the activated provider by setting as the active.
*/
static loadActiveServiceProvider() {
settings.get('AutoTranslate_ServiceProvider', (key, value) => {
TranslationProviderRegistry._activeProvider = value;
// TODO RocketChat.AutoTranslate = TranslationProviderRegistry._providers[TranslationProviderRegistry._activeProvider];
});
}
}

/**
* Generic auto translate base implementation.
* Can be used as superclass for translation providers
* This class provides generic parts of implementation for
* tokenization, detokenization, call back register and unregister.
* @abstract
* @class
*/
Expand All @@ -45,14 +59,6 @@ export class AutoTranslate {
this.name = '';
this.languages = [];
this.supportedLanguages = {};
// Get the service provide API key.
settings.get('AutoTranslate_APIKey', (key, value) => {
this.apiKey = value;
});
// Get Service provider URL.
settings.get('AutoTranslate_ServiceProviderURL', (key, value) => {
this.apiEndPointUrl = value;
});
// Get Auto Translate Active flag
settings.get('AutoTranslate_Enabled', (key, value) => {
this.autoTranslateEnabled = value;
Expand Down Expand Up @@ -282,7 +288,7 @@ export class AutoTranslate {
};
*/
_getProviderMetadata() {
logger.warn('must be implemented by subclass!', '_getProviderMetadata');
Logger.warn('must be implemented by subclass!', '_getProviderMetadata');
}


Expand All @@ -294,7 +300,7 @@ export class AutoTranslate {
* @returns [{ language, name }]
*/
getSupportedLanguages(target) {
logger.warn('must be implemented by subclass!', 'getSupportedLanguages', target);
Logger.warn('must be implemented by subclass!', 'getSupportedLanguages', target);
}

/**
Expand All @@ -307,7 +313,7 @@ export class AutoTranslate {
* @return {object}
*/
_translateMessage(message, targetLanguages) {
logger.warn('must be implemented by subclass!', '_translateMessage', message, targetLanguages);
Logger.warn('must be implemented by subclass!', '_translateMessage', message, targetLanguages);
}

/**
Expand All @@ -319,7 +325,7 @@ export class AutoTranslate {
* @returns {object} translated messages for each target language
*/
_translateAttachmentDescriptions(attachment, targetLanguages) {
logger.warn('must be implemented by subclass!', '_translateAttachmentDescriptions', attachment, targetLanguages);
Logger.warn('must be implemented by subclass!', '_translateAttachmentDescriptions', attachment, targetLanguages);
}
}

Expand Down
25 changes: 16 additions & 9 deletions app/autotranslate/server/deeplTranslate.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
* @author Vigneshwaran Odayappan <vickyokrm@gmail.com>
*/


import { TAPi18n } from 'meteor/tap:i18n';
import { TAPi18n } from 'meteor/rocketchat:tap-i18n';
import { HTTP } from 'meteor/http';
import _ from 'underscore';

import { logger } from './logger';
import { TranslationProviderRegistry, AutoTranslate } from './autotranslate';
import { SystemLogger } from '../../logger/server';
import { settings } from '../../settings';

/**
* DeepL translation service provider class representation.
Expand All @@ -26,7 +26,11 @@ class DeeplAutoTranslate extends AutoTranslate {
constructor() {
super();
this.name = 'deepl-translate';
// this.apiEndPointUrl = 'https://api.deepl.com/v1/translate';
this.apiEndPointUrl = 'https://api.deepl.com/v1/translate';
// Get the service provide API key.
settings.get('AutoTranslate_DeepLAPIKey', (key, value) => {
this.apiKey = value;
});
}

/**
Expand Down Expand Up @@ -56,6 +60,8 @@ class DeeplAutoTranslate extends AutoTranslate {

/**
* Returns supported languages for translation by the active service provider.
* Deepl does not provide an endpoint yet to retrieve the supported languages.
* So each supported languages are explicitly maintained.
* @private implements super abstract method.
* @param {string} target
* @returns {object} code : value pair
Expand All @@ -65,8 +71,7 @@ class DeeplAutoTranslate extends AutoTranslate {
if (this.supportedLanguages[target]) {
return this.supportedLanguages[target];
}
// eslint-disable-next-line no-return-assign
return this.supportedLanguages[target] = [
this.supportedLanguages[target] = [
{
language: 'en',
name: TAPi18n.__('Language_English', { lng: target }),
Expand Down Expand Up @@ -104,6 +109,8 @@ class DeeplAutoTranslate extends AutoTranslate {
name: TAPi18n.__('Language_Russian', { lng: target }),
},
];

return this.supportedLanguages[target];
}
}

Expand Down Expand Up @@ -143,7 +150,7 @@ class DeeplAutoTranslate extends AutoTranslate {
translations[language] = this.deTokenize(Object.assign({}, message, { msg: translatedText }));
}
} catch (e) {
logger.deepl.error('Error translating message', e);
SystemLogger.error('Error translating message', e);
}
});
return translations;
Expand All @@ -156,7 +163,7 @@ class DeeplAutoTranslate extends AutoTranslate {
* @param {object} targetLanguages
* @returns {object} translated messages for each target language
*/
_translateAtachment(attachment, targetLanguages) {
_translateAttachmentDescriptions(attachment, targetLanguages) {
const translations = {};
const query = `text=${ encodeURIComponent(attachment.description || attachment.text) }`;
const supportedLanguages = this.getSupportedLanguages('en');
Expand All @@ -178,7 +185,7 @@ class DeeplAutoTranslate extends AutoTranslate {
}
}
} catch (e) {
logger.deepl.error('Error translating message attachment', e);
SystemLogger.error('Error translating message attachment', e);
}
});
return translations;
Expand Down
55 changes: 37 additions & 18 deletions app/autotranslate/server/googleTranslate.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@
* @author Vigneshwaran Odayappan <vickyokrm@gmail.com>
*/

import { TAPi18n } from 'meteor/tap:i18n';
import { TAPi18n } from 'meteor/rocketchat:tap-i18n';
import { HTTP } from 'meteor/http';
import _ from 'underscore';

import { AutoTranslate, TranslationProviderRegistry } from './autotranslate';
import { logger } from './logger';
import { AutoTranslate, TranslationProviderRegistry } from './autotranslate';
import { SystemLogger } from '../../logger/server';
import { settings } from '../../settings';

/**
* Represents google translate class
Expand All @@ -22,7 +23,11 @@ class GoogleAutoTranslate extends AutoTranslate {
constructor() {
super();
this.name = 'google-translate';
// this.apiEndPointUrl = 'https://translation.googleapis.com/language/translate/v2';
this.apiEndPointUrl = 'https://translation.googleapis.com/language/translate/v2';
// Get the service provide API key.
settings.get('AutoTranslate_GoogleAPIKey', (key, value) => {
this.apiKey = value;
});
}

/**
Expand Down Expand Up @@ -58,35 +63,46 @@ class GoogleAutoTranslate extends AutoTranslate {
* @returns {object} code : value pair
*/
getSupportedLanguages(target) {
let supportedLanguages = {};
if (this.autoTranslateEnabled && this.apiKey) {
if (this.supportedLanguages[target]) {
return this.supportedLanguages[target];
}

let result;
const params = { key: this.apiKey };
const params = {
key: this.apiKey,
};

if (target) {
params.target = target;
}

try {
result = HTTP.get('https://translation.googleapis.com/language/translate/v2/languages', { params });
result = HTTP.get('https://translation.googleapis.com/language/translate/v2/languages', {
params,
});
} catch (e) {
// Fallback: Get the English names of the target languages
if (e.response && e.response.statusCode === 400 && e.response.data && e.response.data.error && e.response.data.error.status === 'INVALID_ARGUMENT') {
params.target = 'en';
target = 'en';
if (!this.supportedLanguages[target]) {
result = HTTP.get('https://translation.googleapis.com/language/translate/v2/languages', { params });
result = HTTP.get('https://translation.googleapis.com/language/translate/v2/languages', {
params,
});
}
}
} finally {
if (this.supportedLanguages[target]) {
// eslint-disable-next-line no-unsafe-finally
return this.supportedLanguages[target];
}
}

if (this.supportedLanguages[target]) {
supportedLanguages = this.supportedLanguages[target];
} else {
this.supportedLanguages[target || 'en'] = result && result.data && result.data.data && result.data.data.languages;
// eslint-disable-next-line no-unsafe-finally
return this.supportedLanguages[target || 'en'];
supportedLanguages = this.supportedLanguages[target || 'en'];
}

return supportedLanguages;
}
}

Expand All @@ -102,8 +118,10 @@ class GoogleAutoTranslate extends AutoTranslate {
const translations = {};
let msgs = message.msg.split('\n');
msgs = msgs.map((msg) => encodeURIComponent(msg));

const query = `q=${ msgs.join('&q=') }`;
const supportedLanguages = this.getSupportedLanguages('en');

targetLanguages.forEach((language) => {
if (language.indexOf('-') !== -1 && !_.findWhere(supportedLanguages, { language })) {
language = language.substr(0, 2);
Expand All @@ -121,7 +139,7 @@ class GoogleAutoTranslate extends AutoTranslate {
translations[language] = this.deTokenize(Object.assign({}, message, { msg: txt }));
}
} catch (e) {
logger.google.error('Error translating message', e);
SystemLogger.error('Error translating message', e);
}
});
return translations;
Expand All @@ -132,12 +150,13 @@ class GoogleAutoTranslate extends AutoTranslate {
* @private
* @param {object} attachment
* @param {object} targetLanguages
* @returns {object} translated messages for each target language
* @returns {object} translated attachment descriptions for each target language
*/
_translateAtachment(attachment, targetLanguages) {
_translateAttachmentDescriptions(attachment, targetLanguages) {
const translations = {};
const query = `q=${ encodeURIComponent(attachment.description || attachment.text) }`;
const supportedLanguages = this.getSupportedLanguages('en');

targetLanguages.forEach((language) => {
if (language.indexOf('-') !== -1 && !_.findWhere(supportedLanguages, { language })) {
language = language.substr(0, 2);
Expand All @@ -154,7 +173,7 @@ class GoogleAutoTranslate extends AutoTranslate {
translations[language] = result.data.data.translations.map((translation) => translation.translatedText).join('\n');
}
} catch (e) {
logger.google.error('Error translating message', e);
SystemLogger.error('Error translating message', e);
}
});
return translations;
Expand Down
1 change: 0 additions & 1 deletion app/autotranslate/server/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import './methods/saveSettings';
import './methods/translateMessage';
import './googleTranslate.js';
import './deeplTranslate.js';
import './models/Settings.js';
import './methods/getProviderUiMetadata.js';

export {
Expand Down
2 changes: 2 additions & 0 deletions app/autotranslate/server/methods/getProviderUiMetadata.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,11 @@ import { TranslationProviderRegistry } from '../autotranslate';
Meteor.methods({
'autoTranslate.getProviderUiMetadata'() {
const providersMetadata = {};

if (!Meteor.userId()) {
throw new Meteor.Error('error-action-not-allowed', 'Login neccessary', { method: 'autoTranslate.getProviderUiMetadata' });
}

for (const provider in TranslationProviderRegistry._providers) {
if (TranslationProviderRegistry._providers.hasOwnProperty(provider)) {
const { name, displayName } = TranslationProviderRegistry._providers[provider]._getProviderMetadata();
Expand Down
25 changes: 16 additions & 9 deletions app/autotranslate/server/settings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,38 @@ Meteor.startup(function() {
}, {
key: 'deepl-translate',
i18nLabel: 'AutoTranslate_DeepL',
}, {
key: 'dbs-translate',
i18nLabel: 'AutoTranslate_DBS',
}],
enableQuery: [{ _id: 'AutoTranslate_Enabled', value: true }],
i18nLabel: 'AutoTranslate_ServiceProvider',
public: true,
});
settings.add('AutoTranslate_ServiceProviderURL', '', {

settings.add('AutoTranslate_GoogleAPIKey', '', {
type: 'string',
group: 'Message',
section: 'AutoTranslate',
section: 'AutoTranslate_Google',
public: true,
enableQuery: [{ _id: 'AutoTranslate_Enabled', value: true }],
i18nLabel: 'AutoTranslate_ServiceProviderURL',
i18nLabel: 'AutoTranslate_APIKey',
enableQuery: [
{
_id: 'AutoTranslate_Enabled', value: true,
},
{
_id: 'AutoTranslate_ServiceProvider', value: 'google-translate',
}],
});

settings.add('AutoTranslate_APIKey', '', {
settings.add('AutoTranslate_DeepLAPIKey', '', {
type: 'string',
group: 'Message',
section: 'AutoTranslate',
section: 'AutoTranslate_DeepL',
public: true,
i18nLabel: 'AutoTranslate_APIKey',
enableQuery: [
{
_id: 'AutoTranslate_Enabled', value: true,
}, {
_id: 'AutoTranslate_ServiceProvider', value: 'deepl-translate',
}],
});
});
Loading

0 comments on commit 1ff545c

Please sign in to comment.