From 1892b316811b0baff34d8a75355b1c9769a625ab Mon Sep 17 00:00:00 2001 From: Mohammad Al-Fetyani Date: Fri, 13 Aug 2021 03:15:02 +0300 Subject: [PATCH 1/3] Fix list tracks --- src/datacamp_downloader/datacamp_utils.py | 32 +++++++++++------------ src/datacamp_downloader/session.py | 5 +++- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/datacamp_downloader/datacamp_utils.py b/src/datacamp_downloader/datacamp_utils.py index cba11ee..5dc42f0 100644 --- a/src/datacamp_downloader/datacamp_utils.py +++ b/src/datacamp_downloader/datacamp_utils.py @@ -78,6 +78,7 @@ def init(self): self.has_active_subscription = False self.loggedin = False self.login_data = None + self.profile_data = None self.courses = [] self.tracks = [] @@ -145,6 +146,16 @@ def set_token(self, token): self.session.add_token(token) self._set_profile() + def get_profile_data(self): + if not self.profile_data: + profile = self.session.get(PROFILE_URL.format(slug=self.login_data["slug"])) + self.session.driver.minimize_window() + soup = BeautifulSoup(profile, "html.parser") + self.profile_data = self.session.to_json( + soup.find(id="__NEXT_DATA__").string + ) + return self.profile_data + @login_required @animate_wait def list_completed_tracks(self, refresh): @@ -366,23 +377,10 @@ def get_completed_tracks(self, refresh=False): self.tracks = [] - profile = self.session.get(PROFILE_URL.format(slug=self.login_data["slug"])) - self.session.driver.minimize_window() - - soup = BeautifulSoup(profile, "html.parser") - tracks_title = soup.findAll("div", {"class": "track-block__main"}) - tracks_link = soup.findAll( - "a", {"href": re.compile("^/tracks"), "class": "shim"} - ) - for i in range(len(tracks_link)): - link = "https://www.datacamp.com" + tracks_link[i]["href"] - self.tracks.append( - Track( - f"t{i + 1}", - tracks_title[i].getText().replace("\n", " ").strip(), - link, - ) - ) + data = self.get_profile_data() + completed_tracks = data["props"]["pageProps"]["completed_tracks"] + for i, track in enumerate(completed_tracks, 1): + self.tracks.append(Track(f"t{i}", track["title"].strip(), track["url"])) all_courses = set() # add courses for track in self.tracks: diff --git a/src/datacamp_downloader/session.py b/src/datacamp_downloader/session.py index 32ed316..964684d 100644 --- a/src/datacamp_downloader/session.py +++ b/src/datacamp_downloader/session.py @@ -83,9 +83,12 @@ def get(self, url): def get_json(self, url): page = self.get(url) page = self.driver.find_element(By.TAG_NAME, "pre").text - parsed_json = json.loads(page) + parsed_json = self.to_json(page) return parsed_json + def to_json(self, page: str): + return json.loads(page) + def get_element_by_id(self, id: str) -> WebElement: return self.driver.find_element(By.ID, id) From dd7129a6443214d3783488fae3ce1a433e0c2955 Mon Sep 17 00:00:00 2001 From: Mohammad Al-Fetyani Date: Fri, 13 Aug 2021 03:28:24 +0300 Subject: [PATCH 2/3] Fix list courses --- src/datacamp_downloader/datacamp_utils.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/datacamp_downloader/datacamp_utils.py b/src/datacamp_downloader/datacamp_utils.py index 5dc42f0..108d14d 100644 --- a/src/datacamp_downloader/datacamp_utils.py +++ b/src/datacamp_downloader/datacamp_utils.py @@ -178,12 +178,13 @@ def list_completed_courses(self, refresh): table.add_row(["ID", "Title", "Datasets", "Exercises", "Videos"]) table_so_far = table.draw() Logger.clear_and_print(table_so_far) - for course in self.get_completed_courses(refresh): + for i, course in enumerate(self.get_completed_courses(refresh), 1): all_exercises_count = sum([c.nb_exercises for c in course.chapters]) videos_count = sum([c.number_of_videos for c in course.chapters]) + course.order = i table.add_row( [ - course.order, + i, course.title, len(course.datasets), all_exercises_count - videos_count, @@ -405,11 +406,15 @@ def get_completed_courses(self, refresh=False): self.courses = [] - for course in self._get_courses_from_link( - PROFILE_URL.format(slug=self.login_data["slug"]) - ): - self.courses.append(course) - yield course + data = self.get_profile_data() + completed_courses = data["props"]["pageProps"]["completed_courses"] + for course in completed_courses: + fetched_course = self.get_course(course["id"]) + if not fetched_course: + continue + self.session.driver.minimize_window() + self.courses.append(fetched_course) + yield fetched_course if not self.courses: return [] @@ -459,7 +464,6 @@ def _get_courses_from_link(self, link: str): continue course = self.get_course(int(id)) if course: - course.order = i yield course def _get_chapter_name(self, chapter: Chapter): From c2324968ec1686beed71c644000745292b390aed Mon Sep 17 00:00:00 2001 From: Mohammad Al-Fetyani Date: Fri, 13 Aug 2021 03:30:45 +0300 Subject: [PATCH 3/3] Bump version to 3.1 --- setup.py | 2 +- src/datacamp_downloader/constants.py | 2 +- src/datacamp_downloader/downloader.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0a9eff6..5c37ede 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( name="datacamp-downloader", - version="3.0", + version="3.1", author="Mohammad Al-Fetyani", author_email="m4bh@hotmail.com", description="Download your completed courses on Datacamp easily!", diff --git a/src/datacamp_downloader/constants.py b/src/datacamp_downloader/constants.py index 3872661..0f895e0 100644 --- a/src/datacamp_downloader/constants.py +++ b/src/datacamp_downloader/constants.py @@ -4,7 +4,7 @@ LOGIN_URL = "https://www.datacamp.com/users/sign_in" LOGIN_DETAILS_URL = "https://www.datacamp.com/api/users/signed_in" -SESSION_FILE = tempfile.gettempdir() + "/.datacamp.v2" +SESSION_FILE = tempfile.gettempdir() + "/.datacamp.v3" PROFILE_URL = "https://www.datacamp.com/profile/{slug}" COURSE_DETAILS_API = "https://campus-api.datacamp.com/api/courses/{id}/" diff --git a/src/datacamp_downloader/downloader.py b/src/datacamp_downloader/downloader.py index 2b4b1f5..18c8504 100644 --- a/src/datacamp_downloader/downloader.py +++ b/src/datacamp_downloader/downloader.py @@ -8,7 +8,7 @@ from .helper import Logger from .templates.lang import Language -__version__ = "3.0.0" +__version__ = "3.1.0" def version_callback(value: bool):