diff --git a/packages/fxa-auth-server/scripts/apple-transfer-users.js b/packages/fxa-auth-server/scripts/apple-transfer-users.js new file mode 100644 index 00000000000..6d920d4fc4c --- /dev/null +++ b/packages/fxa-auth-server/scripts/apple-transfer-users.js @@ -0,0 +1,45 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +'use strict'; + +// This script is used to import users from Apple into FxA. It consumes a CSV +// containing the Apple `transfer_sub` id and exchanges it for their profile information. +// It then creates/updates the user and links the Apple account to the FxA account. +// Example input file: /tests/fixtures/users-apple.csv +// +// Usage: node scripts/apple-transfer-users.js -i -o + +const {ApplePocketFxAMigration} = require('./transfer-users/apple'); + +const program = require('commander'); + +const log = require('../lib/log')({}); +const config = require('../config').config.getProperties(); +const Token = require('../lib/tokens')(log, config); +const AuthDB = require('../lib/db')(config, log, Token); + +program + .option('-d, --delimiter [delimiter]', 'Delimiter for input file', ',') + .option('-o, --output ', 'Output filename to save results to', 'output.csv') + .option( + '-i, --input ', + 'Input filename from which to read input if not specified on the command line', + ) + .parse(process.argv); + +if (!program.input) { + console.error('input file must be specified'); + process.exit(1); +} + +async function main() { + const migration = new ApplePocketFxAMigration(program.input, config, AuthDB, program.output, program.delimiter); + await migration.load(); + await migration.transferUsers(); + await migration.close(); +} + +main(); + diff --git a/packages/fxa-auth-server/scripts/transfer-users/apple.js b/packages/fxa-auth-server/scripts/transfer-users/apple.js new file mode 100644 index 00000000000..9e9839b132d --- /dev/null +++ b/packages/fxa-auth-server/scripts/transfer-users/apple.js @@ -0,0 +1,277 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +'use strict'; + +// This contains classes and functions used to import users from Apple into FxA. +const fs = require('fs'); +const path = require('path'); +const program = require('commander'); + +const axios = require('axios'); +const random = require('../../lib/crypto/random'); +const uuid = require('uuid'); + +const GRANT_TYPE = 'client_credentials'; +const SCOPE = 'user.migration'; +const USER_MIGRATION_ENDPOINT = 'https://appleid.apple.com/auth/usermigrationinfo'; + +const APPLE_PROVIDER = 2; + +export class AppleUser { + constructor(email, transferSub, uid, alternateEmails, db, writeStream, config) { + this.email = email; + this.transferSub = transferSub; + this.uid = uid; + this.alternateEmails = alternateEmails || []; + this.db = db; + this.writeStream = writeStream; + this.config = config; + } + + // Exchanges the Apple `transfer_sub` for the user's profile information and + // moves the user to the new team. + // Ref: https://developer.apple.com/documentation/sign_in_with_apple/bringing_new_apps_and_users_into_your_team#3559300 + async exchangeIdentifiers(accessToken) { + try { + const options = { + transfer_sub: this.transferSub, + client_id: this.config.appleAuthConfig.clientId, + client_secret: this.config.appleAuthConfig.clientSecret, + }; + const res = await axios.post(USER_MIGRATION_ENDPOINT, + new URLSearchParams(options).toString(), + { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }, + ); + + // Apple user info here contains `sub` (Apple unique id), `email` and `is_private_email` + this.appleUserInfo = res.data; + return res.data; + } catch (err) { + this.setFailure(err); + if (err.response && err.response.status === 429) { + console.error(`Rate limit exceeded, try again later: ${this.transferSub}`); + } else { + console.error(`Something went wrong with transfer: ${this.transferSub} ${err}`); + } + } + } + + setSuccess(accountRecord) { + this.success = true; + this.accountRecord = accountRecord; + } + + setFailure(err) { + this.success = false; + this.err = err; + } + + async createLinkedAccount(accountRecord, sub) { + // If the user already has a linked account, delete it and create a new one. + await this.db.deleteLinkedAccount(accountRecord.uid, APPLE_PROVIDER); + await this.db.createLinkedAccount(accountRecord.uid, sub, APPLE_PROVIDER); + } + + async createUpdateFxAUser() { + const sub = this.appleUserInfo.sub; // The recipient team-scoped identifier for the user. + const appleEmail = this.appleUserInfo.email; // The private email address specific to the recipient team. + + // TODO, maybe we should mark this failure + // const isPrivateEmail = this.appleUserInfo.is_private_email; // Boolean if email is private + // if (isPrivateEmail) { + // this.setFailure({ message: 'Apple email is private' }); + // } + + // 1. Check if user exists in FxA via the uid value from Pocket. We should expect + // the uid to be valid, but if it isn't error out. + try { + if (this.uid) { + const accountRecord = await this.db.account(this.uid); + await this.createLinkedAccount(accountRecord, sub); + this.setSuccess(accountRecord); + return; + } + } catch (err) { + // We shouldn't expect Pocket to send a uid that doesn't exist in FxA, but + // if they do, error out. + const msg = `Uid not found: ${this.uid}`; + console.error(msg); + this.setFailure(err); + return; + } + + // 2. Check all emails to see if there exists a match in FxA, link Apple account + // to the FxA account. + let accountRecord; + // FxA tries to find an email match in the following order: + // 1. Primary email from Pocket + // 2. Apple email from `transfer_sub` + // 3. Alternate emails from Pocket + this.alternateEmails.unshift(appleEmail); + this.alternateEmails.unshift(this.email); + if (this.alternateEmails) { + for (const email of this.alternateEmails) { + try { + accountRecord = await this.db.accountRecord(email); + break; + } catch (err) { + // Account not found try next email + } + } + } + // There was a match! Link the Apple account to the FxA account. + if (accountRecord) { + await this.createLinkedAccount(accountRecord, sub); + this.setSuccess(accountRecord); + return; + } + + // 3. No matches mean this is a completely new FxA user, create the user and + // link the Apple account to the FxA account. + try { + const emailCode = await random.hex(16); + const authSalt = await random.hex(32); + const [kA, wrapWrapKb] = await random.hex(32, 32); + accountRecord = await this.db.createAccount({ + uid: uuid.v4({}, Buffer.alloc(16)).toString('hex'), + createdAt: Date.now(), + email: appleEmail, + emailCode, + emailVerified: true, + kA, + wrapWrapKb, + authSalt, + verifierVersion: this.config.verifierVersion, + verifyHash: Buffer.alloc(32).toString('hex'), + verifierSetAt: 0, // No password set + }); + await this.createLinkedAccount(accountRecord, sub); + this.setSuccess(accountRecord); + } catch (err) { + this.setFailure(err); + } + } + + async transferUser(accessToken) { + await this.exchangeIdentifiers(accessToken); + await this.createUpdateFxAUser(this.appleUserInfo); + this.saveResult(); + console.log(`Transfer complete: ${this.transferSub} ${this.success}`); + } + + saveResult() { + const appleEmail = this.appleUserInfo.email; + const fxaEmail = this.accountRecord.email; + const uid = this.accountRecord.uid; // Newly created uid + const transferSub = this.transferSub; + const success = this.success; + const err = (this.err && this.err.message) || ''; + const line = `${transferSub},${uid},${fxaEmail},${appleEmail},${success},${err}`; + this.writeStream.write(line + '\n'); + } +} + +export class ApplePocketFxAMigration { + constructor(filename, config, db, outputFilename, delimiter) { + this.users = []; + this.db = db; + this.filename = filename; + this.config = config; + this.delimiter = delimiter; + + this.writeStream = fs.createWriteStream(outputFilename); + this.writeStream.on('finish', () => { + console.log(`Results saved successfully ${outputFilename}`); + }); + this.writeStream.on('error', (err) => { + console.error(`There was an error writing the file: ${err}`); + }); + } + + parseCSV() { + try { + const input = fs + .readFileSync(path.resolve(this.filename)) + .toString('utf8'); + + if (!input.length) { + return []; + } + + // Parse the input file CSV style + return input.split(/\n/).map((s, index) => { + // First index is the row headers + if (index === 0) return; + + const delimiter = program.delimiter || ','; + const tokens = s.split(delimiter); + const transferSub = tokens[0]; + const uid = tokens[1]; + const email = tokens[2]; + let alternateEmails = []; + + if (tokens[3]) { + // Splits on `:` since they are not allowed in emails + alternateEmails = tokens[3].replaceAll('"', '').split(':'); + } + return new AppleUser(email, transferSub, uid, alternateEmails, this.db, this.config); + }).filter((user) => user); + } catch (err) { + console.error('No such file or directory'); + process.exit(1); + } + } + + writeStreamFileHeader() { + this.writeStream.write('transferSub,uid,fxaEmail,appleEmail,success,err\n'); + } + + writeStreamClose(){ + this.writeStream.end(); + } + + async transferUsers() { + this.writeStreamFileHeader(); + + const accessToken = await this.generateAccessToken(); + for (const user of this.users) { + await user.transferUser(accessToken); + } + + this.writeStreamClose(); + } + + async load() { + this.db = await this.db.connect(this.config); + this.users = this.parseCSV(); + console.info( + '%s accounts loaded from %s', + this.users.length, + this.filename, + ); + } + + async close() { + await this.db.close(); + } + + async generateAccessToken() { + const tokenOptions = { + grant_type: GRANT_TYPE, + scope: SCOPE, + client_id: this.config.appleAuthConfig.clientId, + client_secret: this.config.appleAuthConfig.clientSecret, + }; + const tokenRes = await axios.post(this.config.appleAuthConfig.tokenEndpoint, + new URLSearchParams(tokenOptions).toString(), + ); + console.log('Obtained access token'); + return tokenRes.data['access_token']; + } +} diff --git a/packages/fxa-auth-server/test/scripts/apple-transfer-users.js b/packages/fxa-auth-server/test/scripts/apple-transfer-users.js new file mode 100644 index 00000000000..e6071813431 --- /dev/null +++ b/packages/fxa-auth-server/test/scripts/apple-transfer-users.js @@ -0,0 +1,298 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +'use strict'; + +const ROOT_DIR = '../..'; + +const cp = require('child_process'); +const util = require('util'); +const path = require('path'); +const TestServer = require('../test_server'); + +const execAsync = util.promisify(cp.exec); +const config = require('../../config').config.getProperties(); +const fs = require('fs'); + +const mocks = require(`${ROOT_DIR}/test/mocks`); +const sinon = require('sinon'); +const assert = { ...sinon.assert, ...require('chai').assert }; + +const log = mocks.mockLog(); +const Token = require('../../lib/tokens')(log, config); +const UnblockCode = require('../../lib/crypto/random').base32( + config.signinUnblock.codeLength +); + +const DB = require('../../lib/db')(config, log, Token, UnblockCode); + +const cwd = path.resolve(__dirname, ROOT_DIR); +const execOptions = { + cwd, + env: { + ...process.env, + PATH: process.env.PATH || '', + NODE_ENV: 'dev', + LOG_LEVEL: 'error', + AUTH_FIRESTORE_EMULATOR_HOST: 'localhost:9090', + }, +}; +const axios = require('axios'); +const { ApplePocketFxAMigration, AppleUser } = require('../../scripts/transfer-users/apple'); + +describe('#integration - scripts/apple-transfer-users:', async function () { + this.timeout(30000); + let server, db, sandbox; + before(async () => { + server = await TestServer.start(config); + db = await DB.connect(config); + }); + + beforeEach(async () => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + after(async () => { + await TestServer.stop(server); + await db.close(); + }); + + it('fails if no input file', async () => { + try { + await execAsync( + 'node -r esbuild-register scripts/apple-transfer-users', + execOptions + ); + assert.fail('script should have failed'); + } catch (err) { + assert.include(err.message, 'Command failed'); + } + }); +}); + +describe('ApplePocketFxAMigration', function() { + let sandbox, migration; + beforeEach(function() { + sandbox = sinon.createSandbox(); + + sandbox.stub(fs, 'readFileSync').returns(`transferSub,uid,email\n1,1,test1@example.com\n2,,test2@example.com\n3,3,test3@example.com\n4,,test4@example.com,"test5@example.com:test6@example.com"`); + sandbox.stub(axios, 'post').resolves({ data: { access_token: 'valid_access_token' } }); + sandbox.stub(path, 'resolve').returns('valid.csv'); + sandbox.stub(DB, 'connect').resolves({}); + sandbox.stub(fs, 'createWriteStream').returns({ + write: sandbox.stub(), + end: sandbox.stub(), + on: sandbox.stub() + }); + + migration = new ApplePocketFxAMigration('valid.csv', config, DB, 'output.csv', ','); + }); + + afterEach(function() { + sandbox.restore(); + }); + + it('should load users correctly from a CSV file', async function() { + await migration.load(); + + assert.equal(migration.users.length, 4); + + function assertUser(user, user1) { + assert.equal(user.email, user1.email); + assert.equal(user.uid, user1.uid); + assert.equal(user.transferSub, user1.transferSub); + assert.deepEqual(user.alternateEmails, user1.alternateEmails); + } + + assertUser(migration.users[0], { + email: 'test1@example.com', + uid: '1', + transferSub: '1', + alternateEmails: [], + }); + + assertUser(migration.users[1], { + email: 'test2@example.com', + uid: "", + transferSub: '2', + alternateEmails: [], + }); + + assertUser(migration.users[2], { + email: 'test3@example.com', + uid: '3', + transferSub: '3', + alternateEmails: [], + }); + + assertUser(migration.users[3], { + email: 'test4@example.com', + uid: '', + transferSub: '4', + alternateEmails: ['test5@example.com', 'test6@example.com'], + }); + }); + + it('should generate access token correctly', async function() { + const token = await migration.generateAccessToken(); + assert.calledOnce(axios.post); + assert.equal(token,'valid_access_token'); + }); + + it('should call transferUser on each user correctly when transferUsers is called', async function() { + await migration.load(); + migration.users.forEach(user => { + sandbox.stub(user, 'transferUser').resolves(true); + }); + + await migration.transferUsers(); + + migration.users.forEach(user => { + assert.calledOnce(user.transferUser); + }); + }); + + it('should close db connection correctly when close is called', async function() { + migration.db = { close: sandbox.stub().resolves() }; + await migration.close(); + assert.calledOnce(migration.db.close); + }); +}); + +describe('AppleUser', function() { + let sandbox, dbStub, user, writeStreamStub; + beforeEach(function() { + sandbox = sinon.createSandbox(); + dbStub = { + account: sandbox.stub(), + deleteLinkedAccount: sandbox.stub().resolves(), + createLinkedAccount: sandbox.stub().resolves(), + createAccount: sandbox.stub().resolves(), + accountRecord: sandbox.stub().resolves(), + }; + writeStreamStub = { + write: sandbox.stub() + }; + user = new AppleUser('pocket@example.com', 'transferSub', 'uid', ['altEmail@example.com'], dbStub, writeStreamStub, config); + }); + + afterEach(function() { + sandbox.restore(); + }); + + it('should exchange identifiers correctly and update user', async function() { + const stub = sandbox.stub(axios, 'post').resolves({ data: { sub: 'sub', email: 'email@example.com', is_private_email: true } }); + const data = await user.exchangeIdentifiers('accessToken'); + + assert.calledOnce(stub); + assert.deepEqual(data, { sub: 'sub', email: 'email@example.com', is_private_email: true }); + assert.deepEqual(user.appleUserInfo, { sub: 'sub', email: 'email@example.com', is_private_email: true }); + }); + + it('should link user from FxA uid', async function() { + const accountRecord = { uid: 'uid', email: 'email@example.com' }; + dbStub.account.resolves(accountRecord); + user.appleUserInfo = { + sub: 'sub', + email: 'email@email.com', + is_private_email: false + }; + await user.createUpdateFxAUser(); + + assert.calledOnceWithExactly(dbStub.account, user.uid); + assert.calledOnceWithExactly(dbStub.deleteLinkedAccount, 'uid', 2); + assert.calledOnceWithExactly(dbStub.createLinkedAccount, 'uid', 'sub', 2); + assert.isTrue(user.success); + assert.equal(user.accountRecord,accountRecord); + }); + + it('should link user from Pocket email that has FxA account', async function() { + dbStub.account.rejects({ + errno: 102, + }); + + const accountRecord = { uid: 'uid1', email: 'pocket@example.com' }; + dbStub.accountRecord.resolves(accountRecord); + + user.uid = ''; // user does not have an account in FxA + + user.appleUserInfo = { + sub: 'sub', + email: 'apple@example.com', + is_private_email: false + }; + + await user.createUpdateFxAUser(); + + assert.calledOnceWithExactly(dbStub.accountRecord, 'pocket@example.com'); + assert.calledOnceWithExactly(dbStub.deleteLinkedAccount, 'uid1', 2); + assert.calledOnceWithExactly(dbStub.createLinkedAccount, 'uid1', 'sub', 2); + assert.isTrue(user.success); + assert.equal(user.accountRecord,accountRecord); + }); + + it('should create user from Apple email without FxA account', async function() { + dbStub.account.rejects({ + errno: 102, + }); + dbStub.accountRecord.rejects({ + errno: 102, + }); + user.uid = ''; // user does not have an account in FxA + + const accountRecord = { + email: 'apple@example.com', + uid: 'uid2' + } + dbStub.createAccount.resolves(accountRecord); + + user.appleUserInfo = { + sub: 'sub', + email: 'apple@example.com', + is_private_email: false + }; + + await user.createUpdateFxAUser(); + + assert.calledOnceWithMatch(dbStub.createAccount, { + email: 'apple@example.com' + }); + + assert.calledOnceWithExactly(dbStub.deleteLinkedAccount, 'uid2', 2); + assert.calledOnceWithExactly(dbStub.createLinkedAccount, 'uid2', 'sub', 2); + assert.isTrue(user.success); + assert.equal(user.accountRecord, accountRecord); + }); + + it('should transfer user correctly', async function() { + sandbox.stub(user, 'exchangeIdentifiers').resolves(); + sandbox.stub(user, 'createUpdateFxAUser').resolves(); + sandbox.stub(user, 'saveResult').resolves(); + await user.transferUser('accessToken'); + + sinon.assert.calledOnce(user.exchangeIdentifiers); + sinon.assert.calledOnce(user.createUpdateFxAUser); + sinon.assert.calledOnce(user.saveResult); + }); + + it('should save results correctly', async () => { + const accountRecord = { uid: 'uid', email: 'fxa@example.com' }; + dbStub.account.resolves(accountRecord); + user.appleUserInfo = { + sub: 'transferSub', + email: 'apple@example.com', + is_private_email: false + }; + user.err = undefined; + await user.createUpdateFxAUser(); + + user.saveResult(); + const expectedOutput = 'transferSub,uid,fxa@example.com,apple@example.com,true,\n'; + assert.calledWith(user.writeStream.write, expectedOutput); + }); +}); \ No newline at end of file diff --git a/packages/fxa-auth-server/test/scripts/fixtures/apple_users.csv b/packages/fxa-auth-server/test/scripts/fixtures/apple_users.csv new file mode 100644 index 00000000000..2867043bd0f --- /dev/null +++ b/packages/fxa-auth-server/test/scripts/fixtures/apple_users.csv @@ -0,0 +1,4 @@ +transfer_sub,uid,email,alternate_emails +transfersub1,uid1,existing_fxa_user@test.com,"" +transfersub2,,new_fxa_user@test.com,"new_fxa_user1@test.com:new_fxa_user2@test.com" +transfersub3,,private_relay_user@test.com,"" \ No newline at end of file