-
Notifications
You must be signed in to change notification settings - Fork 291
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from iamshaynez/dev-todoist
add todoist loader
- Loading branch information
Showing
5 changed files
with
184 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -46,4 +46,5 @@ | |
| "bbdc": "BBDC", | ||
| "weread": "WeRead", | ||
| "covid": "COVID-19", | ||
| "todoist": "Todoist", | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| import datetime | ||
| import json | ||
|
|
||
| import pandas as pd | ||
| import requests | ||
|
|
||
| from github_poster.loader.base_loader import BaseLoader | ||
|
|
||
|
|
||
| class TodoistLoader(BaseLoader): | ||
| track_color = "#FFE411" | ||
| unit = "tasks" | ||
|
|
||
| def __init__(self, from_year, to_year, _type, **kwargs): | ||
| super().__init__(from_year, to_year, _type) | ||
| self.from_year = from_year | ||
| self.to_year = to_year | ||
| self.todoist_token = kwargs.get("todoist_token", "") | ||
| # another magic number, try 3 times for calling api | ||
| self.MAXIMAL_RETRY = 3 | ||
|
|
||
| @classmethod | ||
| def add_loader_arguments(cls, parser, optional): | ||
| # add argument for loader | ||
| parser.add_argument( | ||
| "--todoist_token", | ||
| dest="todoist_token", | ||
| type=str, | ||
| required=optional, | ||
| help="dev token", | ||
| ) | ||
|
|
||
| # call with token | ||
| def response(self, url, postdata): | ||
| # headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11'} | ||
| headers = {"Authorization": "Bearer {0}".format(self.todoist_token)} | ||
| res = requests.post(url=url, data=postdata, headers=headers) | ||
| resposn = res.json() | ||
| return resposn | ||
|
|
||
| # call with re-try since todoist api some time get 502 | ||
| def response_with_retry(self, url, postdata, times): | ||
| # time.sleep(1) | ||
| try: | ||
| return self.response(url, postdata) | ||
| except Exception as e: | ||
| if times >= self.MAXIMAL_RETRY: | ||
| print( | ||
| f">> Exceed maximal retry {self.MAXIMAL_RETRY}, Raise exception..." | ||
| ) | ||
| raise (e) # will stop the program without further handling | ||
| else: | ||
| times += 1 | ||
| print(f">> Exception, Retry {times} begins...") | ||
| return self.response_with_retry(url, postdata, times) | ||
|
|
||
| # call todoist dev api to get activity of completed tasks | ||
| # refer https://developer.todoist.com/sync/v9/#activity | ||
| def todoist_completed_activity(self, page, limit, offset): | ||
| data = { | ||
| "event_type": "completed", | ||
| "page": page, | ||
| "limit": limit, | ||
| "offset": offset, | ||
| } | ||
| url = "https://api.todoist.com/sync/v9/activity/get" | ||
| re = self.response_with_retry(url, data, 1) | ||
| return re | ||
|
|
||
| # json expect to be list of events format | ||
| # with event_date, event_type, id | ||
| def normalize_df(self, jsondata): | ||
| if jsondata["count"] == 0: | ||
| return pd.DataFrame(columns=["event_date", "event_type", "id"]) | ||
| df = pd.json_normalize(jsondata["events"]) | ||
| df = df[["event_date", "event_type", "id"]] | ||
| df["event_date"] = df["event_date"].str.slice(0, 10) | ||
| return df | ||
|
|
||
| def count_to_dict(self, df): | ||
| return df.groupby(["event_date"])["event_date"].count().to_dict() | ||
|
|
||
| # refer https://developer.todoist.com/sync/v9/#activity | ||
| # todoist api only allows you to get activity data by page from current day | ||
| # we will have to calculate the pages based on from and to year then manipulate the dict data | ||
| def get_api_data(self): | ||
| # init critical dates | ||
| today = datetime.datetime.today().strftime("%Y-%m-%d") | ||
| current_year = datetime.datetime.now().year | ||
| # 52.14 weeks a year, add 53 pages per full year | ||
| number_of_days = datetime.date.today().timetuple().tm_yday + 365 * ( | ||
| current_year - self.from_year | ||
| ) | ||
| # current year | ||
| page_from = ( | ||
| 0 | ||
| if current_year == self.to_year | ||
| else datetime.date.today().timetuple().tm_yday // 7 | ||
| ) | ||
| page_to = number_of_days // 7 + 1 | ||
| print("Todoist API Page range ({0},{1})".format(page_from, page_to)) | ||
| last_day_of_to_year = datetime.datetime(self.to_year, 12, 31).strftime( | ||
| "%Y-%m-%d" | ||
| ) | ||
| first_day_of_from_year = datetime.datetime(self.from_year, 1, 1).strftime( | ||
| "%Y-%m-%d" | ||
| ) | ||
|
|
||
| df = pd.DataFrame(columns=["event_date", "event_type", "id"]) | ||
| # magic number 3 is to cover the 0.14 extra week of every year when counting number of pages | ||
| for page in range(page_from, page_to + 3): | ||
| offset = 0 | ||
| limit = 100 | ||
| while True: | ||
| res = self.todoist_completed_activity(page, limit, offset) | ||
| if res["count"] >= offset: | ||
| offset = offset + limit | ||
| df_res = self.normalize_df(res) | ||
| df = pd.concat([df, df_res]) | ||
| else: | ||
| break | ||
|
|
||
| df = df[df["event_date"] >= first_day_of_from_year] | ||
| df = df[df["event_date"] <= last_day_of_to_year] | ||
|
|
||
| return df | ||
|
|
||
| def make_track_dict(self): | ||
| # generate statistics data | ||
| df = self.get_api_data() | ||
| df_dict = self.count_to_dict(df) | ||
| self.number_by_date_dict = df_dict | ||
| # print(df_dict) | ||
| for _, v in self.number_by_date_dict.items(): | ||
| self.number_list.append(v) | ||
|
|
||
| def get_all_track_data(self): | ||
| self.make_track_dict() | ||
| self.make_special_number() | ||
| # print(self.year_list) | ||
| print("不积跬步，无以至千里。Todoist欢迎你。") | ||
| return self.number_by_date_dict, self.year_list |