Merge pull request #6 from JoshuaOliphant/updates_after_initial_load

Updates after initial load

avocet/app.py

@@ -3,8 +3,8 @@
 import webbrowser
 from textual import on, work, log
 from textual.app import App, ComposeResult
-from textual.containers import Center, VerticalScroll
-from textual.widgets import OptionList, ProgressBar, Label, MarkdownViewer, Footer, Header
+from textual.containers import Center, VerticalScroll, Vertical
+from textual.widgets import OptionList, ProgressBar, Label, MarkdownViewer, Header, Footer
 from textual.widgets.option_list import Option
 from sqlalchemy import create_engine
 from database_manager import DatabaseManager
@@ -20,9 +20,11 @@ class Avocet(App):
 
     def compose(self) -> ComposeResult:
         yield Header()
-        with Center():
-            yield Label("Loading...")
-            yield ProgressBar(total=100)
+        with Vertical(id="progress_bars"):
+            yield Label("Loading database...", id="database_label")
+            yield ProgressBar(id="database", total=100)
+            yield Label("Getting summaries from OpenAI", id="summary_label")
+            yield ProgressBar(id="summaries", total=100)
         with Center():
             yield OptionList(id="collection_option_list")
             yield OptionList(id="raindrop_option_list")
@@ -43,38 +45,71 @@ async def on_mount(self) -> None:
 
     @work
     async def startup(self, is_initialized):
-        # Check if the database exists
         start_time = time.time()
         raindrop_api = RaindropAPI()
 
         if not is_initialized:
+            log("Initializing...")
             await self.initialize_db(raindrop_api)
             await self.add_text()
-        self.query_one(ProgressBar).remove()
-        self.query_one(Label).remove()
+        else:
+            log("Updating...")
+            await self.update_db(raindrop_api)
+            await self.update_text()
+
+        self.database_manager.update_last_update()
+        self.query_one("#progress_bars").remove()
         await self.initialize_view()
         end_time = time.time()
         log(f"Startup time: {start_time-end_time}")
 
     async def initialize_db(self, raindrop_api):
         self.database_manager.create_tables()
         collection_data_list = await raindrop_api.get_collections()
-        self.query_one(ProgressBar).update(total=len(collection_data_list))
+        self.query_one("#database").update(total=len(collection_data_list))
         for collection_data in collection_data_list:
             self.database_manager.add_collection(collection_data)
             raindrop_data_list = await raindrop_api.get_raindrops_by_collection_id(collection_data["_id"])
             self.database_manager.add_raindrops(raindrop_data_list, collection_data["_id"])
-            self.query_one(ProgressBar).advance(1)
+            self.query_one("#database").advance(1)
+
+    async def update_db(self, raindrop_api):
+        last_update = self.database_manager.get_last_update().last_update
+        formatted_date = last_update.strftime("%Y-%m-%d")
+        collections = self.database_manager.get_collections()
+        self.query_one("#database").update(total=len(collections))
+        for collection in collections:
+            raindrops = await raindrop_api.get_raindrops_by_collection_id(
+                collection.id,
+                search=f"lastUpdate:{formatted_date}")
+            log(f"Raindrops to update: {raindrops}")
+            self.database_manager.add_raindrops(raindrops, collection.id)
+            self.query_one("#database").advance(1)
 
     async def add_text(self):
         raindrops = self.database_manager.get_all_raindrops()
+        self.query_one("#summaries").update(total=len(raindrops))
         for raindrop in raindrops:
             markdown = await self.ai.html_to_markdown(raindrop.link)
             log(f"Markdown: {markdown}")
             if markdown:
                 raindrop.summary = markdown[0]['article_summary']
+            self.query_one("#summaries").advance(1)
         self.database_manager.update_raindrops(raindrops)
 
+    async def update_text(self):
+        last_update = self.database_manager.get_last_update()
+        updated_raindrops = self.database_manager.get_updated_raindrops(last_update.last_update)
+        log(f"Number of raindrops to update: {len(updated_raindrops)}")
+        self.query_one("#summaries").update(total=len(updated_raindrops))
+        for raindrop in updated_raindrops:
+            markdown = await self.ai.html_to_markdown(raindrop.link)
+            log(f"Markdown: {markdown}")
+            if markdown:
+                raindrop.summary = markdown[0]['article_summary']
+            self.query_one("#summaries").advance(1)
+        self.database_manager.update_raindrops(updated_raindrops)
+
     async def initialize_view(self):
         collections = self.database_manager.get_collections()
         for collection in collections:

avocet/database_manager.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from sqlalchemy.orm import sessionmaker
-from models import Collection, Base, Raindrop
+from models import Collection, Base, Raindrop, Update
+from textual import log
 
 class DatabaseManager:
     def __init__(self, engine):
@@ -65,7 +66,30 @@ def get_all_raindrop_urls(self):
         session = self.Session()
         return session.query(Raindrop).with_entities(Raindrop.link).all()
 
+    def get_updated_raindrops(self, updated_after):
+        session = self.Session()
+        raindrops = session.query(Raindrop).filter(Raindrop.last_update > updated_after).all()
+        return raindrops
+
     def update_raindrops(self, raindrops):
         session = self.Session()
         session.add_all(raindrops)
         session.commit()
+
+    def update_last_update(self):
+        session = self.Session()
+        update = session.query(Update).first()
+        if not update:
+            log(f"Setting update first time")
+            update = Update(last_update=datetime.now())
+            session.add(update)
+        else:
+            log(f"Updating last_update")
+            update.last_update = datetime.now()
+            session.add(update)
+        log(f"Committing update {update}")
+        session.commit()
+
+    def get_last_update(self):
+        session = self.Session()
+        return session.query(Update).first()

avocet/models.py

@@ -27,3 +27,8 @@ class Raindrop(Base):
     tags = Column(JSON)
     # non-raindrop columns
     summary = Column(String)
+
+class Update(Base):
+    __tablename__ = "update"
+    id = Column(Integer, primary_key=True)
+    last_update = Column(DateTime)

avocet/raindrop_api.py

@@ -1,6 +1,7 @@
 from typing import Dict
 import httpx
 import os
+import datetime
 from textual import log
 
 class RaindropAPI: