Skip to content

Commit

Permalink
Merge branch 'loadWithSOAP'
Browse files Browse the repository at this point in the history
  • Loading branch information
@ElToroAP
ElToroAP committed Oct 21, 2020
2 parents 44227ef + 93ca9e0 commit 75f0093
Show file tree
Hide file tree
Showing 5 changed files with 60 additions and 34 deletions.
10 changes: 5 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ OPTIONS
this command invocation
```

_See code: [src/commands/ETCopyData/Compare.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.12-b/src/commands/ETCopyData/Compare.ts)_
_See code: [src/commands/ETCopyData/Compare.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.13-b/src/commands/ETCopyData/Compare.ts)_

## `sfdx ETCopyData:delete [-c <string>] [-d <string>] [-s <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand All @@ -265,7 +265,7 @@ OPTIONS
this command invocation
```

_See code: [src/commands/ETCopyData/delete.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.12-b/src/commands/ETCopyData/delete.ts)_
_See code: [src/commands/ETCopyData/delete.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.13-b/src/commands/ETCopyData/delete.ts)_

## `sfdx ETCopyData:export [-c <string>] [-d <string>] [-s <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand All @@ -292,7 +292,7 @@ OPTIONS
this command invocation
```

_See code: [src/commands/ETCopyData/export.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.12-b/src/commands/ETCopyData/export.ts)_
_See code: [src/commands/ETCopyData/export.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.13-b/src/commands/ETCopyData/export.ts)_

## `sfdx ETCopyData:full [-c <string>] [-d <string>] [-s <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand All @@ -319,7 +319,7 @@ OPTIONS
this command invocation
```

_See code: [src/commands/ETCopyData/full.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.12-b/src/commands/ETCopyData/full.ts)_
_See code: [src/commands/ETCopyData/full.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.13-b/src/commands/ETCopyData/full.ts)_

## `sfdx ETCopyData:import [-c <string>] [-d <string>] [-s <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`

Expand All @@ -346,6 +346,6 @@ OPTIONS
this command invocation
```

_See code: [src/commands/ETCopyData/import.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.12-b/src/commands/ETCopyData/import.ts)_
_See code: [src/commands/ETCopyData/import.ts](https://github.com/eltoroit/ETCopyData/blob/v0.5.13-b/src/commands/ETCopyData/import.ts)_
<!-- commandsstop -->
<!-- ET-AUTO-STOP: This section is auto-updated... -->
18 changes: 9 additions & 9 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,31 @@
{
"name": "etcopydata",
"description": "SFDX Plugin to populate your scratch org and/or developer sandbox with data for multiple related sObjects.",
"version": "0.5.12-b",
"version": "0.5.13-b",
"author": "Andres Perez @ElToroIT",
"bugs": "https://github.com/eltoroit/ETCopyData/issues",
"dependencies": {
"@oclif/config": "1.17.0",
"@oclif/errors": "1.3.3",
"@oclif/plugin-help": "3.2.0",
"@salesforce/command": "3.0.3",
"npm-check-updates": "8.1.0",
"tslib": "2.0.1"
"npm-check-updates": "9.1.2",
"tslib": "2.0.3"
},
"devDependencies": {
"@oclif/dev-cli": "1.22.2",
"@oclif/test": "1.2.7",
"@salesforce/dev-config": "2.0.0",
"@types/chai": "4.2.12",
"@types/jsforce": "1.9.20",
"@types/chai": "4.2.14",
"@types/jsforce": "1.9.23",
"@types/mocha": "8.0.3",
"@types/node": "14.6.4",
"@types/node": "14.14.0",
"globby": "11.0.1",
"mocha": "8.1.3",
"mocha": "8.2.0",
"nyc": "15.1.0",
"sinon": "9.0.3",
"sinon": "9.2.0",
"ts-node": "9.0.0",
"typescript": "4.0.2"
"typescript": "4.0.3"
},
"engines": {
"node": ">=8.0.0"
Expand Down
2 changes: 1 addition & 1 deletion src/@ELTOROIT/Exporter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ export class Exporter {
org.settings
.writeToFile(org.alias + folderCode, sObjName + ".json", data)
.then(() => {
// NOTE: Clean memory, and avoid heap dumps.
// NOTE: Clean memory, and avoid heap overflow.
data.records = [];
// Now, resolve it.
resolve();
Expand Down
51 changes: 33 additions & 18 deletions src/@ELTOROIT/Importer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { BulkOptions, Date, RecordResult } from "jsforce";
import { BulkOptions, Date, RecordResult, RestApiOptions } from "jsforce";
import { ISchemaDataParent } from "./Interfaces";
import { OrgManager } from "./OrgManager";
import { LogLevel, ResultOperation, Util } from "./Util";
Expand Down Expand Up @@ -90,7 +90,7 @@ export class Importer {
this.countImportErrorsRecords = 0;
this.countImportErrorsSObjects = 0;
const sObjectsToLoad: string[] = orgDestination.order.findImportOrder();
Util.writeLog("sObjects should be processed in this order: " + sObjectsToLoad, LogLevel.TRACE);
Util.writeLog("sObjects should be processed in this order: " + sObjectsToLoad.join(", "), LogLevel.TRACE);
return this.loadAllSObjectData(orgSource, orgDestination, sObjectsToLoad, 0);
})
.then(() => {
Expand Down Expand Up @@ -298,15 +298,7 @@ export class Importer {
});

if (records.length > 0) {
// LOADING
this.matchingIds.set(sObjName, new Map<string, string>());
orgDestination.conn.bulk.pollTimeout = orgDestination.settings.pollingTimeout;
// WARNING: Salesforce Bulk has a weird behavior that if the options are not given,
// WARNING: then the rest of the parameters are shifted to the left rather than taking null as a placeholder.
const bulkOptions: BulkOptions = { concurrencyMode: "Parallel", extIdField: null };
// LEARNING: Inserting sObject records in bulk
// ELTOROIT: Bulk or SOAP?
orgDestination.conn.bulk.load(sObjName, "insert", bulkOptions, records, (error, results: any[]) => {
const processResults = (error, results: any[]) => {
let badCount: number = 0;
let goodCount: number = 0;

Expand Down Expand Up @@ -340,7 +332,22 @@ export class Importer {
Util.logResultsAdd(orgDestination, ResultOperation.IMPORT, sObjName, goodCount, badCount);

resolve(badCount);
});
};

// LOADING
this.matchingIds.set(sObjName, new Map<string, string>());
// ELTOROIT: Bulk or SOAP?
if (orgDestination.settings.useBulkAPI) {
// WARNING: Salesforce Bulk has a weird behavior that if the options are not given,
// WARNING: then the rest of the parameters are shifted to the left rather than taking null as a placeholder.
orgDestination.conn.bulk.pollTimeout = orgDestination.settings.pollingTimeout;
const options: BulkOptions = { concurrencyMode: "Parallel", extIdField: null };
// LEARNING: Inserting sObject records in bulk
orgDestination.conn.bulk.load(sObjName, "insert", options, records, processResults);
} else {
const options: RestApiOptions = { allOrNone: true, allowRecursive: true };
orgDestination.conn.sobject(sObjName).create(records, options, processResults);
}
} else {
resolve(0);
}
Expand All @@ -355,7 +362,6 @@ export class Importer {
return new Promise((resolve, reject) => {
// WARNING: Salesforce Bulk has a weird behavior that if the options are not given,
// WARNING: then the rest of the parameters are shifted to the left rather than taking null as a placeholder.
const bulkOptions: BulkOptions = { concurrencyMode: "Parallel", extIdField: null };
const keys: Array<string> = Array.from(this.twoPassReferenceFieldData.keys());

Util.serialize(
Expand Down Expand Up @@ -399,9 +405,8 @@ export class Importer {
records.push(record);
});

// Update the records using the bulk api
// ELTOROIT: Bulk or SOAP?
orgDestination.conn.bulk.load(sObjectName, "update", bulkOptions, records, (error, results: any[]) => {
// UPDATING the records
const processResults = (error, results: any[]) => {
let badCount: number = 0;
let goodCount: number = 0;

Expand All @@ -426,7 +431,16 @@ export class Importer {
Util.logResultsAdd(orgDestination, ResultOperation.IMPORT, sObjectName, goodCount, badCount);

resolveEach();
});
};

// ELTOROIT: Bulk or SOAP?
if (orgDestination.settings.useBulkAPI) {
const bulkOptions: BulkOptions = { concurrencyMode: "Parallel", extIdField: null };
orgDestination.conn.bulk.load(sObjectName, "update", bulkOptions, records, processResults);
} else {
const options: RestApiOptions = { allOrNone: true, allowRecursive: true };
orgDestination.conn.sobject(sObjectName).update(records, options, processResults);
}
});
}
)
Expand All @@ -444,14 +458,15 @@ export class Importer {

return new Promise((resolve, reject) => {
// DELETING
const options: any = { allowBulk: org.settings.useBulkAPI };
Util.writeLog(`[${org.alias}] Deleting records from [${sObjName}]`, LogLevel.TRACE);
org.conn.bulk.pollTimeout = org.settings.pollingTimeout;
// LEARNING: Deleting sObject records in bulk
// ELTOROIT: Bulk or SOAP?
org.conn
.sobject(sObjName)
.find({ CreatedDate: { $lte: Date.TOMORROW } })
.destroy(sObjName)
.destroy(sObjName, options)
.then((results: RecordResult[]) => {
let totalSuccess: number = 0;
let totalFailures: number = 0;
Expand Down
13 changes: 12 additions & 1 deletion src/@ELTOROIT/Settings.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ConfigContents, ConfigFile, fs } from "@salesforce/core";
import { AnyJson, Dictionary, toAnyJson } from "@salesforce/ts-types";
import { AnyJson, Dictionary } from "@salesforce/ts-types";
import { WhichOrg } from "./OrgManager";
import { LogLevel, Util } from "./Util";

Expand Down Expand Up @@ -41,6 +41,7 @@ export interface ISettingsValues {
includeAllCustom: boolean;
stopOnErrors: boolean;
copyToProduction: boolean;
useBulkAPI: boolean;
ignoreFieldsRaw: string;
twoPassReferenceFieldsRaw: string;
maxRecordsEachRaw: number;
Expand Down Expand Up @@ -80,6 +81,7 @@ export class Settings implements ISettingsValues {
public includeAllCustom: boolean;
public stopOnErrors: boolean;
public copyToProduction: boolean;
public useBulkAPI: boolean;
public maxRecordsEachRaw: number;
public deleteDestination: boolean;
public pollingTimeout: number;
Expand Down Expand Up @@ -365,6 +367,13 @@ export class Settings implements ISettingsValues {
})
);

// useBulkAPI
promises.push(
this.processStringValues(resValues, "useBulkAPI", false).then((value: string) => {
this.useBulkAPI = value === "true";
})
);

// rootFolder
promises.push(
this.processStringValues(resValues, "rootFolder", false)
Expand Down Expand Up @@ -557,6 +566,7 @@ export class Settings implements ISettingsValues {
output.stopOnErrors = this.stopOnErrors;
output.ignoreFields = this.ignoreFieldsRaw;
output.copyToProduction = this.copyToProduction;
output.useBulkAPI = this.useBulkAPI;
output.twoPassReferenceFields = this.twoPassReferenceFieldsRaw;
output.maxRecordsEach = this.maxRecordsEachRaw;
output.deleteDestination = this.deleteDestination;
Expand Down Expand Up @@ -629,6 +639,7 @@ export class Settings implements ISettingsValues {
this.includeAllCustom = false;
this.stopOnErrors = true;
this.copyToProduction = false;
this.useBulkAPI = false;
this.ignoreFieldsRaw = null;
this.twoPassReferenceFieldsRaw = null;
this.maxRecordsEachRaw = -1;
Expand Down

0 comments on commit 75f0093

Please sign in to comment.