Small updates (#85)

* fix timestamp issues

* Updating some links to pytr-org

* Fixed --last_days option

* Small clean up

README.md

@@ -1,5 +1,5 @@
-[![GitHub tag (with filter)](https://img.shields.io/github/v/tag/marzzzello/pytr?style=for-the-badge&link=https%3A%2F%2Fgit.luolix.top%2Fmarzzzello%2Fpytr%2Ftags)](https://github.com/marzzzello/pytr/tags)
-[![PyPI build and publish](https://img.shields.io/github/actions/workflow/status/marzzzello/pytr/publish-pypi.yml?link=https%3A%2F%2Fgit.luolix.top%2Fmarzzzello%2Fpytr%2Factions%2Fworkflows%2Fpublish-pypi.yml&style=for-the-badge)](https://github.com/marzzzello/pytr/actions/workflows/publish-pypi.yml)
+[![GitHub tag (with filter)](https://img.shields.io/github/v/tag/pytr-org/pytr?style=for-the-badge&link=https%3A%2F%2Fgit.luolix.top%2Fmarzzzello%2Fpytr%2Ftags)](https://github.com/pytr-org/pytr/tags)
+[![PyPI build and publish](https://img.shields.io/github/actions/workflow/status/pytr-org/pytr/publish-pypi.yml?link=https%3A%2F%2Fgit.luolix.top%2Fmarzzzello%2Fpytr%2Factions%2Fworkflows%2Fpublish-pypi.yml&style=for-the-badge)](https://github.com/pytr-org/pytr/actions/workflows/publish-pypi.yml)
 [![PyPI - Version](https://img.shields.io/pypi/v/pytr?link=https%3A%2F%2Fpypi.org%2Fproject%2Fpytr%2F&style=for-the-badge)](https://pypi.org/project/pytr/)
 
 # pytr: Use TradeRepublic in terminal
@@ -15,7 +15,7 @@ Install release from PyPI with `pipx install pytr`
 Or install from git repo like so:
 
 ```sh
-pipx install git+https://github.com/marzzzello/pytr
+pipx install git+https://github.com/pytr-org/pytr.git
 ```
 
 ### Update

pytr/dl.py

@@ -48,7 +48,7 @@ def __init__(
         self.filepaths = []
         self.doc_urls = []
         self.doc_urls_history = []
-        self.tl = Timeline(self.tr)
+        self.tl = Timeline(self.tr, self.since_timestamp)
         self.log = get_logger(__name__)
         self.load_history()
 
@@ -66,7 +66,7 @@ def load_history(self):
             self.log.info('Created history file')
 
     async def dl_loop(self):
-        await self.tl.get_next_timeline_transactions(max_age_timestamp=self.since_timestamp)
+        await self.tl.get_next_timeline_transactions()
 
         while True:
             try:
@@ -75,11 +75,11 @@ async def dl_loop(self):
                 self.log.fatal(str(e))
 
             if subscription['type'] == 'timelineTransactions':
-                await self.tl.get_next_timeline_transactions(response, max_age_timestamp=self.since_timestamp)
+                await self.tl.get_next_timeline_transactions(response)
             elif subscription['type'] == 'timelineActivityLog':
-                await self.tl.get_next_timeline_activity_log(response, max_age_timestamp=self.since_timestamp)
+                await self.tl.get_next_timeline_activity_log(response)
             elif subscription['type'] == 'timelineDetailV2':
-                await self.tl.timelineDetail(response, self, max_age_timestamp=self.since_timestamp)
+                await self.tl.timelineDetail(response, self)
             else:
                 self.log.warning(f"unmatched subscription of type '{subscription['type']}':\n{preview(response)}")
 

pytr/main.py

@@ -9,6 +9,7 @@
 
 from importlib.metadata import version
 from pathlib import Path
+from datetime import datetime
 
 from pytr.utils import get_logger, check_version, export_transactions
 from pytr.dl import DL
@@ -198,8 +199,7 @@ def main():
         if args.last_days == 0:
             since_timestamp = 0
         else:
-            since_timestamp = (time.time() - (24 * 3600 * args.last_days)) * 1000
-
+            since_timestamp = datetime.now().timestamp() - (24 * 3600 * args.last_days)
         dl = DL(
             login(phone_no=args.phone_no, pin=args.pin, web=not args.applogin),
             args.output,

pytr/utils.py

@@ -79,7 +79,7 @@ def preview(response, num_lines=5):
 def check_version(installed_version):
     log = get_logger(__name__)
     try:
-        r = requests.get('https://api.github.com/repos/marzzzello/pytr/tags', timeout=1)
+        r = requests.get('https://api.github.com/repos/pytr-org/pytr/tags', timeout=1)
     except Exception as e:
         log.error('Could not check for a newer version')
         log.debug(str(e))
@@ -290,7 +290,7 @@ def export_transactions(input_path, output_path, lang='auto'):
 
 
 class Timeline:
-    def __init__(self, tr):
+    def __init__(self, tr, max_age_timestamp):
         self.tr = tr
         self.log = get_logger(__name__)
         self.received_detail = 0
@@ -301,21 +301,20 @@ def __init__(self, tr):
         self.num_timelines = 0
         self.timeline_events = {}
         self.timeline_events_iter = None
+        self.max_age_timestamp = max_age_timestamp
 
-    async def get_next_timeline_transactions(self, response=None, max_age_timestamp=0):
+    async def get_next_timeline_transactions(self, response=None):
         '''
         Get timelines transactions and save time in list timelines.
         Extract timeline transactions events and save them in list timeline_events
 
         '''
-
         if response is None:
             # empty response / first timeline
-            self.log.info('Awaiting #1  timeline transactions')
+            self.log.info('Subscribing to #1 timeline transactions')
             self.num_timelines = 0
             await self.tr.timeline_transactions()
         else:
-            timestamp = response['items'][-1]['timestamp']
             self.num_timelines += 1
             # print(json.dumps(response))
             self.num_timeline_details += len(response['items'])
@@ -324,30 +323,37 @@ async def get_next_timeline_transactions(self, response=None, max_age_timestamp=
                 self.timeline_events[event['id']] = event
 
             after = response['cursors'].get('after')
-            if after is None:
-                # last timeline is reached
-                await self.get_next_timeline_activity_log()
-            else:
+
+            last_transaction_time = response['items'][-1]['timestamp'][:19]
+            timestamp = datetime.fromisoformat(last_transaction_time).timestamp()
+            self.log.info(
+                f'Received #{self.num_timelines:<2} timeline transactions until {last_transaction_time}'
+            )
+            if (after is not None) and ((self.max_age_timestamp == 0) or (timestamp >= self.max_age_timestamp)):
                 self.log.info(
-                    f'Received #{self.num_timelines:<2} timeline transactions, awaiting #{self.num_timelines+1:<2} timeline transactions'
+                f'Subscribing #{self.num_timelines+1:<2} timeline transactions'
                 )
-                await self.tr.timeline_transactions(after)
+                await self.tr.timeline_transactions(after)                
+            else:
+                # last timeline is reached
+                self.log.info('Received last relevant timeline transaction')
+                await self.get_next_timeline_activity_log()
 
 
-    async def get_next_timeline_activity_log(self, response=None, max_age_timestamp=0):
+    async def get_next_timeline_activity_log(self, response=None):
         '''
         Get timelines acvtivity log and save time in list timelines.
         Extract timeline acvtivity log events and save them in list timeline_events
 
         '''
-
         if response is None:
             # empty response / first timeline
             self.log.info('Awaiting #1  timeline activity log')
             self.num_timelines = 0
             await self.tr.timeline_activity_log()
         else:
-            timestamp = datetime.fromisoformat(response['items'][-1]['timestamp'][:19]).timestamp()
+            last_transaction_time = response['items'][-1]['timestamp'][:19]
+            timestamp = datetime.fromisoformat(last_transaction_time).timestamp()
             self.num_timelines += 1
             # print(json.dumps(response))
             self.num_timeline_details += len(response['items'])
@@ -357,49 +363,26 @@ async def get_next_timeline_activity_log(self, response=None, max_age_timestamp=
                     self.timeline_events[event['id']] = event
 
             after = response['cursors'].get('after')
-            if after is None:
-                # last timeline is reached
-                self.log.info(f'Received #{self.num_timelines:<2} (last) timeline activity log')
-                self.timeline_events_iter = iter(self.timeline_events.values())
-                await self._get_timeline_details(5)
-            elif max_age_timestamp != 0 and timestamp < max_age_timestamp:
-                self.log.info(f'Received #{self.num_timelines+1:<2} timeline activity log')
-                self.log.info('Reached last relevant timeline activity log')
-                self.timeline_events_iter = iter(self.timeline_events.values())
-                await self._get_timeline_details(5, max_age_timestamp=max_age_timestamp)
-            else:
+            self.log.info(f'Received #{self.num_timelines:<2} timeline activity log unitl {last_transaction_time}')
+            if (after is not None) and ((self.max_age_timestamp == 0) or (timestamp >= self.max_age_timestamp)):
                 self.log.info(
-                    f'Received #{self.num_timelines:<2} timeline activity log, awaiting #{self.num_timelines+1:<2} timeline activity log'
+                    f'Subscribing #{self.num_timelines+1:<2} timeline activity log'
                 )
                 await self.tr.timeline_activity_log(after)
+            else:
+                self.log.info('Received last relevant timeline activity log')
+                await self._get_timeline_details()
 
-    async def _get_timeline_details(self, num_torequest, max_age_timestamp=0):
+    async def _get_timeline_details(self):
         '''
         request timeline details
         '''
-        while num_torequest > 0:
-
-            try:
-                event = next(self.timeline_events_iter)
-            except StopIteration:
-                self.log.info('All timeline details requested')
-                return False
-
+        for event in self.timeline_events.values():
             action = event.get('action')
-            # icon = event.get('icon')
             msg = ''
             timestamp_field = datetime.fromisoformat(event['timestamp'][:19]).timestamp() 
-            if max_age_timestamp != 0 and timestamp_field > max_age_timestamp:
+            if self.max_age_timestamp != 0 and (timestamp_field < self.max_age_timestamp):
                 msg += 'Skip: too old'
-            # elif icon is None:
-            #     pass
-            # elif icon.endswith('/human.png'):
-            #     msg += 'Skip: human'
-            # elif icon.endswith('/CashIn.png'):
-            #     msg += 'Skip: CashIn'
-            # elif icon.endswith('/ExemptionOrderChanged.png'):
-            #     msg += 'Skip: ExemptionOrderChanged'
-
             elif action is None:
                 if event.get('actionLabel') is None:
                     msg += 'Skip: no action'
@@ -416,11 +399,12 @@ async def _get_timeline_details(self, num_torequest, max_age_timestamp=0):
                 self.num_timeline_details -= 1
                 continue
 
-            num_torequest -= 1
             self.requested_detail += 1
             await self.tr.timeline_detail_v2(event['id'])
+        self.log.info('All timeline details requested')
+        return False
 
-    async def timelineDetail(self, response, dl, max_age_timestamp=0):
+    async def timelineDetail(self, response, dl):
         '''
         process timeline response and request timelines
         '''
@@ -429,26 +413,15 @@ async def timelineDetail(self, response, dl, max_age_timestamp=0):
         event = self.timeline_events[response['id']]
         event['details'] = response
 
-        # when all requested timeline events are received request 5 new
-        if self.received_detail == self.requested_detail:
-            remaining = len(self.timeline_events)
-            if remaining < 5:
-                await self._get_timeline_details(remaining)
-            else:
-                await self._get_timeline_details(5)
-
-        isSavingsPlan = (event["eventType"] == "SAVINGS_PLAN_EXECUTED")
-
-        if not isSavingsPlan and event['subtitle'] is not None:
-            isSavingsPlan = 'Sparplan' in event['subtitle']
-
+        is_savings_plan = (event["eventType"] == "SAVINGS_PLAN_EXECUTED")
+
         max_details_digits = len(str(self.num_timeline_details))
         self.log.info(
             f"{self.received_detail:>{max_details_digits}}/{self.num_timeline_details}: "
-            + f"{event['title']} -- {event['subtitle']} [{event['eventType']}]"
+            + f"{event['title']} -- {event['subtitle']}"
         )
 
-        if isSavingsPlan:
+        if is_savings_plan:
             subfolder = 'Sparplan'
         else:
             subfolder = {
@@ -462,14 +435,12 @@ async def timelineDetail(self, response, dl, max_age_timestamp=0):
         for section in response['sections']:
             if section['type'] != 'documents':
                 continue
-
             for doc in section['data']:
                 try:
-                    timestamp = datetime.strptime(doc['detail'], '%d.%m.%Y').timestamp() * 1000
+                    timestamp = datetime.strptime(doc['detail'], '%d.%m.%Y').timestamp()
                 except (ValueError, KeyError):
-                    timestamp = datetime.now().timestamp() * 1000
-                if max_age_timestamp == 0 or max_age_timestamp < timestamp:
-                    # save all savingsplan documents in a subdirectory
+                    timestamp = datetime.now().timestamp()
+                if self.max_age_timestamp == 0 or self.max_age_timestamp < timestamp:
                     title = f"{doc['title']} - {event['title']}"
                     if event['eventType'] in ["ACCOUNT_TRANSFER_INCOMING", "ACCOUNT_TRANSFER_OUTGOING", "CREDIT"]:
                         title += f" - {event['subtitle']}"

setup.py

@@ -10,11 +10,11 @@ def readme():
 
 setup(
     name='pytr',
-    version='0.1.9',
+    version='0.2.0',
     description='Use TradeRepublic in terminal',
     long_description=readme(),
     long_description_content_type='text/markdown',
-    url='https://gitlab.com/marzzzello/pytr/',
+    url='https://gitlab.com/pytr-org/pytr/',
     author='marzzzello',
     author_email='853485-marzzzello@users.noreply.gitlab.com',
     license='MIT',