From d981bba2c5b324092f3d4c598163572c2778c191 Mon Sep 17 00:00:00 2001 From: Alex Zhao Date: Wed, 14 Aug 2019 18:23:03 -0400 Subject: [PATCH 1/2] Add SOAP login support --- tap_salesforce/__init__.py | 38 ++++-- tap_salesforce/salesforce/__init__.py | 169 +++++++++++++++++++++++++- tap_salesforce/salesforce/utils.py | 37 ++++++ 3 files changed, 231 insertions(+), 13 deletions(-) create mode 100644 tap_salesforce/salesforce/utils.py diff --git a/tap_salesforce/__init__.py b/tap_salesforce/__init__.py index 13199d04..7c493faa 100644 --- a/tap_salesforce/__init__.py +++ b/tap_salesforce/__init__.py @@ -13,18 +13,31 @@ LOGGER = singer.get_logger() -REQUIRED_CONFIG_KEYS = ['refresh_token', - 'client_id', - 'client_secret', - 'start_date', - 'api_type', - 'select_fields_by_default'] +OAUTH_REQUIRED_CONFIG_KEYS = ['refresh_token', + 'client_id', + 'client_secret', + 'start_date', + 'api_type', + 'auth_type', + 'select_fields_by_default'] + +SOAP_REQUIRED_CONFIG_KEYS = ['username', + 'password', + 'start_date', + 'api_type', + 'auth_type', + 'select_fields_by_default'] CONFIG = { 'refresh_token': None, 'client_id': None, 'client_secret': None, - 'start_date': None + 'start_date': None, + 'username': None, + 'password': None, + 'security_token': None, + 'organization_id': None, + 'auth_type': None } FORCED_FULL_TABLE = { @@ -358,7 +371,11 @@ def do_sync(sf, catalog, state): LOGGER.info("Finished sync") def main_impl(): - args = singer_utils.parse_args(REQUIRED_CONFIG_KEYS) + try: + args = singer_utils.parse_args(OAUTH_REQUIRED_CONFIG_KEYS) + except: + args = singer_utils.parse_args(SOAP_REQUIRED_CONFIG_KEYS) + CONFIG.update(args.config) sf = None @@ -367,6 +384,11 @@ def main_impl(): refresh_token=CONFIG['refresh_token'], sf_client_id=CONFIG['client_id'], sf_client_secret=CONFIG['client_secret'], + sf_username=CONFIG['username'], + sf_password=CONFIG['password'], + sf_security_token=CONFIG['security_token'], + sf_organization_id=CONFIG['organization_id'], + auth_type=CONFIG['auth_type'], quota_percent_total=CONFIG.get('quota_percent_total'), quota_percent_per_run=CONFIG.get('quota_percent_per_run'), is_sandbox=CONFIG.get('is_sandbox'), diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index 71aa3040..b64a0acc 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -7,12 +7,14 @@ import singer import singer.utils as singer_utils from singer import metadata, metrics +from html import escape from tap_salesforce.salesforce.bulk import Bulk from tap_salesforce.salesforce.rest import Rest from tap_salesforce.salesforce.exceptions import ( TapSalesforceException, TapSalesforceQuotaExceededException) +from tap_salesforce.salesforce.utils import getUniqueElementValueFromXmlString LOGGER = singer.get_logger() @@ -22,6 +24,11 @@ BULK_API_TYPE = "BULK" REST_API_TYPE = "REST" +OAUTH_AUTH_TYPE = "OAUTH" +SOAP_AUTH_TYPE = "SOAP" +DEFAULT_CLIENT_ID_PREFIX = 'Tap-Salesforce' +DEFAULT_API_VERSION = '41.0' + STRING_TYPES = set([ 'id', 'string', @@ -175,8 +182,6 @@ def field_to_property_schema(field, mdata): "longitude": {"type": ["null", "number"]}, "latitude": {"type": ["null", "number"]} } - elif sf_type == 'json': - property_schema['type'] = "string" else: raise TapSalesforceException("Found unsupported type: {}".format(sf_type)) @@ -193,12 +198,18 @@ def __init__(self, token=None, sf_client_id=None, sf_client_secret=None, + sf_username=None, + sf_password=None, + sf_organization_id=None, + sf_security_token=None, quota_percent_per_run=None, quota_percent_total=None, is_sandbox=None, select_fields_by_default=None, default_start_date=None, - api_type=None): + api_type=None, + api_version=DEFAULT_API_VERSION, + auth_type=None): self.api_type = api_type.upper() if api_type else None self.refresh_token = refresh_token self.token = token @@ -207,6 +218,13 @@ def __init__(self, self.session = requests.Session() self.access_token = None self.instance_url = None + self.sf_username = sf_username + self.sf_password = sf_password + self.sf_organization_id = sf_organization_id + self.sf_security_token = sf_security_token + self.auth_type = auth_type + self.api_version = api_version + if isinstance(quota_percent_per_run, str) and quota_percent_per_run.strip() == '': quota_percent_per_run = None if isinstance(quota_percent_total, str) and quota_percent_total.strip() == '': @@ -289,13 +307,21 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False, return resp def login(self): + if self.auth_type == OAUTH_AUTH_TYPE: + self._login_oauth() + elif self.auth_type == SOAP_AUTH_TYPE: + session_id, instance = self._login_soap() + self.access_token = session_id + self.instance_url = instance + + def _login_oauth(self): if self.is_sandbox: login_url = 'https://test.salesforce.com/services/oauth2/token' else: login_url = 'https://login.salesforce.com/services/oauth2/token' login_body = {'grant_type': 'refresh_token', 'client_id': self.sf_client_id, - 'client_secret': self.sf_client_secret, 'refresh_token': self.refresh_token} + 'client_secret': self.sf_client_secret, 'refresh_token': self.refresh_token} LOGGER.info("Attempting login via OAuth2") @@ -319,6 +345,139 @@ def login(self): self.login_timer = threading.Timer(REFRESH_TOKEN_EXPIRATION_PERIOD, self.login) self.login_timer.start() + def _login_soap(self): + if self.is_sandbox is not None: + domain = 'test' if self.is_sandbox else 'login' + + if domain is None: + domain = 'login' + + soap_url = 'https://{domain}.salesforce.com/services/Soap/u/{sf_version}' + + if self.sf_client_id: + client_id = "{prefix}/{app_name}".format( + prefix=DEFAULT_CLIENT_ID_PREFIX, + app_name=self.sf_client_id) + else: + client_id = DEFAULT_CLIENT_ID_PREFIX + + soap_url = soap_url.format(domain=domain, + sf_version=self.api_version) + + # pylint: disable=E0012,deprecated-method + username = escape(self.sf_username) + password = escape(self.sf_password) + + # Check if token authentication is used + if self.sf_security_token is not None: + # Security Token Soap request body + login_soap_request_body = """ + + + + {client_id} + sf + + + + + {username} + {password}{token} + + + """.format( + username=username, password=password, token=self.sf_security_token, + client_id=client_id) + + # Check if IP Filtering is used in conjunction with organizationId + elif self.sf_organization_id is not None: + # IP Filtering Login Soap request body + login_soap_request_body = """ + + + + {client_id} + sf + + + {organizationId} + + + + + {username} + {password} + + + """.format( + username=username, password=password, organizationId=self.sf_organization_id, + client_id=self.sf_client_id) + elif username is not None and password is not None: + # IP Filtering for non self-service users + login_soap_request_body = """ + + + + {client_id} + sf + + + + + {username} + {password} + + + """.format( + username=username, password=password, client_id=self.sf_client_id) + else: + except_code = 'INVALID AUTH' + except_msg = ( + 'You must submit either a security token or organizationId for ' + 'authentication' + ) + raise Exception(except_code, except_msg) + + login_soap_request_headers = { + 'content-type': 'text/xml', + 'charset': 'UTF-8', + 'SOAPAction': 'login' + } + LOGGER.info("Attempting login via SOAP {}".format(soap_url)) + response = (self.session or requests).post( + soap_url, login_soap_request_body, headers=login_soap_request_headers) + + if response.status_code != 200: + except_code = getUniqueElementValueFromXmlString( + response.content, 'sf:exceptionCode') + except_msg = getUniqueElementValueFromXmlString( + response.content, 'sf:exceptionMessage') + raise Exception(except_code, except_msg) + + LOGGER.info("SOAP login successful") + + session_id = getUniqueElementValueFromXmlString( + response.content, 'sessionId') + server_url = getUniqueElementValueFromXmlString( + response.content, 'serverUrl') + + protocol = server_url.split("://")[0] + sf_instance = (server_url + .replace('http://', '') + .replace('https://', '') + .split('/')[0] + .replace('-api', '')) + + return session_id, "{}://{}".format(protocol, sf_instance) + def describe(self, sobject=None): """Describes all objects or a specific object""" headers = self._get_standard_headers() @@ -413,4 +572,4 @@ def get_blacklisted_fields(self): else: raise TapSalesforceException( "api_type should be REST or BULK was: {}".format( - self.api_type)) + self.api_type)) \ No newline at end of file diff --git a/tap_salesforce/salesforce/utils.py b/tap_salesforce/salesforce/utils.py new file mode 100644 index 00000000..e7970cef --- /dev/null +++ b/tap_salesforce/salesforce/utils.py @@ -0,0 +1,37 @@ +import xml.dom.minidom +import os +# Copied from https://github.com/simple-salesforce/simple-salesforce/blob/master/simple_salesforce/util.py +# pylint: disable=invalid-name +def getUniqueElementValueFromXmlString(xmlString, elementName): + """ + Extracts an element value from an XML string. + + For example, invoking + getUniqueElementValueFromXmlString( + 'bar', 'foo') + should return the value 'bar'. + """ + xmlStringAsDom = xml.dom.minidom.parseString(xmlString) + elementsByName = xmlStringAsDom.getElementsByTagName(elementName) + elementValue = None + if len(elementsByName) > 0: + elementValue = elementsByName[0].toxml().replace( + '<' + elementName + '>', '').replace('', '') + return elementValue + +# pylint: disable=invalid-name +def getUniqueElementValueFromXmlElement(xmlStringAsDom, elementName): + """ + Extracts an element value from an XML string. + + For example, invoking + getUniqueElementValueFromXmlString( + 'bar', 'foo') + should return the value 'bar'. + """ + elementsByName = xmlStringAsDom.getElementsByTagName(elementName) + elementValue = None + if len(elementsByName) > 0: + elementValue = elementsByName[0].toxml().replace( + '<' + elementName + '>', '').replace('', '') + return elementValue \ No newline at end of file From 6cefc092418bf55806e37d32fd6b329a77f070bc Mon Sep 17 00:00:00 2001 From: Alex Zhao Date: Wed, 14 Aug 2019 18:53:09 -0400 Subject: [PATCH 2/2] Apply the latest change from master --- tap_salesforce/salesforce/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index b64a0acc..a1f52a8c 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -182,6 +182,8 @@ def field_to_property_schema(field, mdata): "longitude": {"type": ["null", "number"]}, "latitude": {"type": ["null", "number"]} } + elif sf_type == 'json': + property_schema['type'] = "string" else: raise TapSalesforceException("Found unsupported type: {}".format(sf_type)) @@ -321,7 +323,7 @@ def _login_oauth(self): login_url = 'https://login.salesforce.com/services/oauth2/token' login_body = {'grant_type': 'refresh_token', 'client_id': self.sf_client_id, - 'client_secret': self.sf_client_secret, 'refresh_token': self.refresh_token} + 'client_secret': self.sf_client_secret, 'refresh_token': self.refresh_token} LOGGER.info("Attempting login via OAuth2")