From 80ecfa13f88034d57877d859f059f7af663921ed Mon Sep 17 00:00:00 2001 From: davidteather Date: Wed, 22 Sep 2021 18:02:17 -0500 Subject: [PATCH] V4.0.3 --- TikTokApi/browser_utilities/__init__.py | 1 + TikTokApi/{ => browser_utilities}/browser.py | 9 +- .../browser_utilities/browser_interface.py | 18 + .../browser_selenium.py | 84 ++-- .../{ => browser_utilities}/get_acrawler.py | 0 TikTokApi/{ => browser_utilities}/stealth.py | 0 TikTokApi/tiktok.py | 4 +- docs/TikTokApi.html | 9 +- docs/TikTokApi/browser_utilities.html | 253 ++++++++++++ .../{ => browser_utilities}/browser.html | 154 +++++-- .../browser_utilities/browser_interface.html | 382 ++++++++++++++++++ .../browser_selenium.html | 252 +++++++----- .../{ => browser_utilities}/get_acrawler.html | 40 +- .../{ => browser_utilities}/stealth.html | 12 +- docs/TikTokApi/exceptions.html | 68 ++++ docs/TikTokApi/tiktok.html | 284 ++++++++++--- docs/index.html | 10 +- docs/search.json | 2 +- 18 files changed, 1333 insertions(+), 249 deletions(-) create mode 100644 TikTokApi/browser_utilities/__init__.py rename TikTokApi/{ => browser_utilities}/browser.py (98%) create mode 100644 TikTokApi/browser_utilities/browser_interface.py rename TikTokApi/{ => browser_utilities}/browser_selenium.py (69%) rename TikTokApi/{ => browser_utilities}/get_acrawler.py (100%) rename TikTokApi/{ => browser_utilities}/stealth.py (100%) create mode 100644 docs/TikTokApi/browser_utilities.html rename docs/TikTokApi/{ => browser_utilities}/browser.html (92%) create mode 100644 docs/TikTokApi/browser_utilities/browser_interface.html rename docs/TikTokApi/{ => browser_utilities}/browser_selenium.html (83%) rename docs/TikTokApi/{ => browser_utilities}/get_acrawler.html (70%) rename docs/TikTokApi/{ => browser_utilities}/stealth.html (99%) diff --git a/TikTokApi/browser_utilities/__init__.py b/TikTokApi/browser_utilities/__init__.py new file mode 100644 index 00000000..9e900d6c --- /dev/null +++ b/TikTokApi/browser_utilities/__init__.py @@ -0,0 +1 @@ +from .browser_interface import BrowserInterface \ No newline at end of file diff --git a/TikTokApi/browser.py b/TikTokApi/browser_utilities/browser.py similarity index 98% rename from TikTokApi/browser.py rename to TikTokApi/browser_utilities/browser.py index 5a70dddd..1260bfbf 100644 --- a/TikTokApi/browser.py +++ b/TikTokApi/browser_utilities/browser.py @@ -3,17 +3,15 @@ import string import requests import logging -from threading import Thread import time -import datetime import random import json import re +from .browser_interface import BrowserInterface from urllib.parse import splitquery, parse_qs, parse_qsl # Import Detection From Stealth -from .stealth import stealth from .get_acrawler import get_acrawler, get_tt_params_script from playwright.sync_api import sync_playwright @@ -31,7 +29,7 @@ def get_playwright(): return playwright -class browser: +class browser(BrowserInterface): def __init__( self, **kwargs, @@ -44,7 +42,6 @@ def __init__( self.language = kwargs.get("language", "en") self.executablePath = kwargs.get("executablePath", None) self.device_id = kwargs.get("custom_device_id", None) - find_redirect = kwargs.get("find_redirect", False) args = kwargs.get("browser_args", []) options = kwargs.get("browser_options", {}) @@ -83,8 +80,8 @@ def __init__( args=self.args, **self.options ) except Exception as e: - raise e logging.critical(e) + raise e context = self.create_context(set_useragent=True) page = context.new_page() diff --git a/TikTokApi/browser_utilities/browser_interface.py b/TikTokApi/browser_utilities/browser_interface.py new file mode 100644 index 00000000..dfbb62f8 --- /dev/null +++ b/TikTokApi/browser_utilities/browser_interface.py @@ -0,0 +1,18 @@ +import abc +class BrowserInterface(abc.ABC): + @abc.abstractmethod + def __init__(self, **kwargs): + pass + + @abc.abstractmethod + def get_params(self, page) -> None: + pass + + # Returns verify_fp, device_id, signature, tt_params + @abc.abstractmethod + def sign_url(self, calc_tt_params=False, **kwargs) -> tuple[str, str, str, str]: + pass + + @abc.abstractmethod + def clean_up(self) -> None: + pass \ No newline at end of file diff --git a/TikTokApi/browser_selenium.py b/TikTokApi/browser_utilities/browser_selenium.py similarity index 69% rename from TikTokApi/browser_selenium.py rename to TikTokApi/browser_utilities/browser_selenium.py index bb7872c9..19fe8c49 100644 --- a/TikTokApi/browser_selenium.py +++ b/TikTokApi/browser_utilities/browser_selenium.py @@ -1,22 +1,24 @@ import random import time -import string import requests import logging from threading import Thread import time -import datetime +import re import random +import json +from .browser_interface import BrowserInterface from selenium_stealth import stealth from selenium import webdriver -from .get_acrawler import get_acrawler +from .get_acrawler import get_acrawler, get_tt_params_script +from urllib.parse import splitquery, parse_qs, parse_qsl - -class browser: +class browser(BrowserInterface): def __init__( self, **kwargs, ): + self.kwargs = kwargs self.debug = kwargs.get("debug", False) self.proxy = kwargs.get("proxy", None) self.api_url = kwargs.get("api_url", None) @@ -81,26 +83,32 @@ def setup_browser(self): renderer="Intel Iris OpenGL Engine", fix_hairline=True, ) + self.get_params(self.browser) + # NOTE: Slower than playwright at loading this because playwright can ignore unneeded files. + self.browser.get("https://www.tiktok.com/@redbull") self.browser.execute_script(get_acrawler()) + self.browser.execute_script(get_tt_params_script()) + def get_params(self, page) -> None: - # self.browser_language = await self.page.evaluate("""() => { return - # navigator.language || navigator.userLanguage; }""") - self.browser_language = "" - # self.timezone_name = await self.page.evaluate("""() => { return - # Intl.DateTimeFormat().resolvedOptions().timeZone; }""") - self.timezone_name = "" - # self.browser_platform = await self.page.evaluate("""() => { return window.navigator.platform; }""") - self.browser_platform = "" - # self.browser_name = await self.page.evaluate("""() => { return window.navigator.appCodeName; }""") - self.browser_name = "" - # self.browser_version = await self.page.evaluate("""() => { return window.navigator.appVersion; }""") - self.browser_version = "" - - self.width = page.execute_script("""return screen.width""") - self.height = page.execute_script("""return screen.height""") self.userAgent = page.execute_script("""return navigator.userAgent""") + self.browser_language = self.kwargs.get("browser_language", ("""return navigator.language""")) + self.browser_version = ("""return window.navigator.appVersion""") + + if len(self.browser_language.split("-")) == 0: + self.region = self.kwargs.get("region", "US") + self.language = self.kwargs.get("language", "en") + elif len(self.browser_language.split("-")) == 1: + self.region = self.kwargs.get("region", "US") + self.language = self.browser_language.split("-")[0] + else: + self.region = self.kwargs.get("region", self.browser_language.split("-")[1]) + self.language = self.kwargs.get("language", self.browser_language.split("-")[0]) + + self.timezone_name = self.kwargs.get("timezone_name", ("""return Intl.DateTimeFormat().resolvedOptions().timeZone""")) + self.width = ("""return screen.width""") + self.height = ("""return screen.height""") def base36encode(self, number, alphabet="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"): """Converts an integer to a base36 string.""" @@ -139,11 +147,12 @@ def gen_verifyFp(self): return f'verify_{scenario_title.lower()}_{"".join(uuid)}' - def sign_url(self, **kwargs): + def sign_url(self, calc_tt_params=False, **kwargs): url = kwargs.get("url", None) if url is None: raise Exception("sign_url required a url parameter") + tt_params = None if kwargs.get("gen_new_verifyFp", False): verifyFp = self.gen_verifyFp() else: @@ -159,24 +168,37 @@ def sign_url(self, **kwargs): else: device_id = self.device_id - return ( - verifyFp, - device_id, + url = '{}&verifyFp={}&device_id={}'.format(url, verifyFp, device_id) + # self.browser.execute_script(content=get_acrawler()) + # Should be covered by an earlier addition of get_acrawler. + evaluatedPage = ( self.browser.execute_script( ''' - var url = "''' - + url - + "&verifyFp=" - + verifyFp - + """&device_id=""" - + device_id - + """" + var url = "'''+ url + """" var token = window.byted_acrawler.sign({url: url}); return token; """ ), ) + url = '{}&_signature={}'.format(url, evaluatedPage) + # self.browser.execute_script(content=get_tt_params_script()) + # Should be covered by an earlier addition of get_acrawler. + + tt_params = self.browser.execute_script( + '''() => { + return window.genXTTParams(''' + json.dumps(dict(parse_qsl(splitquery(url)[1]))) + '''); + + }''' + ) + + return ( + verifyFp, + device_id, + evaluatedPage, + tt_params + ) + def clean_up(self): try: self.browser.close() diff --git a/TikTokApi/get_acrawler.py b/TikTokApi/browser_utilities/get_acrawler.py similarity index 100% rename from TikTokApi/get_acrawler.py rename to TikTokApi/browser_utilities/get_acrawler.py diff --git a/TikTokApi/stealth.py b/TikTokApi/browser_utilities/stealth.py similarity index 100% rename from TikTokApi/stealth.py rename to TikTokApi/browser_utilities/stealth.py diff --git a/TikTokApi/tiktok.py b/TikTokApi/tiktok.py index 56b44de6..915dd451 100644 --- a/TikTokApi/tiktok.py +++ b/TikTokApi/tiktok.py @@ -52,9 +52,9 @@ def __init__(self, **kwargs): global BASE_URL BASE_URL = "https://t.tiktok.com/" if kwargs.get("use_selenium", False): - from .browser_selenium import browser + from .browser_utilities.browser_selenium import browser else: - from .browser import browser + from .browser_utilities.browser import browser if kwargs.get("generate_static_device_id", False): self.custom_device_id = "".join(random.choice(string.digits) for num in range(19)) diff --git a/docs/TikTokApi.html b/docs/TikTokApi.html index a5415fce..ea80cf1d 100644 --- a/docs/TikTokApi.html +++ b/docs/TikTokApi.html @@ -32,7 +32,7 @@

Contents