Add "API mode" and more integration tests

.dockerignore

@@ -0,0 +1,2 @@
+**/.venv
+**/node_modules

Makefile

@@ -37,5 +37,4 @@ clean:
 
 .PHONY: test
 test:
-	go test tests/integration/web_static_test.go -v
-	go test tests/integration/web_dynamic_test.go -v
+	go test tests/integration/*.go -v

README.md

@@ -43,12 +43,20 @@ Feel free to edit the [docker-compose.yml](docker-compose.yml) as needed to fit
 - Node 20 or higher (use nvm)
 
 ## Development Instructions
-To install, create a fresh venv and then:
+### Application
+To install the RSS application, create a fresh venv and then:
 ```bash
 make dev
 ```
 Then to develop, in one terminal start tailwind by doing `make tw`. Then, in other start the main app by doing `make run`.
 
+### Integration Tests
+Precis has integration tests that are written in Go. They are automated to run during the pull request pipeline, but they also be run locally.
+
+First, install the version of Go specified in `go.mod`. I recommend to use a Golang version manager such as `gvm` or `g`.
+
+Then, start the application using `make run`. Finally, run the integration tests with `make test`.
+
 # Features
 ## OPML Import/Export
 Precis supports exporting your current set of feeds as OPML, as well as importing feeds from other OPML files. You can find options on the `/feeds/` page, or use the CLI as described below.

app/app.py

@@ -6,7 +6,7 @@
 
 from fastapi import FastAPI, Form, UploadFile, status
 from fastapi.requests import Request
-from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from fastapi_utils.tasks import repeat_every
@@ -18,6 +18,8 @@
 from app.rss import PrecisRSS
 from app.storage.engine import load_storage_config
 
+JSON = "application/json"
+
 logger = getLogger("uvicorn.error")
 base_path = Path(__file__).parent
 
@@ -89,21 +91,23 @@ async def root(request: Request):
         return RedirectResponse("/onboarding/")
 
 
-@app.get("/about", response_class=HTMLResponse)
+@app.get("/about")
 async def about(
     request: Request, update_status: bool = False, update_exception: str = None
 ):
-
-    return templates.TemplateResponse(
-        "about.html",
-        {
-            "request": request,
-            "settings": await bk.get_settings(),
-            "update_status": update_status,
-            "update_exception": update_exception,
-            **bk.about(),
-        },
-    )
+    response = {
+        "settings": await bk.get_settings(),
+        "update_status": update_status,
+        "update_exception": update_exception,
+        **bk.about(),
+    }
+
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "about.html", {"request": request, **response}
+        )
 
 
 @app.get("/health", response_model=HealthCheck)
@@ -112,84 +116,96 @@ async def health_check(request: Request) -> HealthCheck:
     return bk.health_check()
 
 
-@app.get("/onboarding/", response_class=HTMLResponse)
+@app.get("/onboarding/")
 async def onboarding(request: Request):
+    response = {"settings": await bk.get_settings()}
 
-    return templates.TemplateResponse(
-        "onboarding.html",
-        {
-            "request": request,
-            "settings": await bk.get_settings(),
-        },
-    )
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "onboarding.html",
+            {"request": request, **response},
+        )
 
 
 @app.get("/list-entries/{feed_id}", response_class=HTMLResponse)
 async def list_entries_by_feed(
     feed_id: str, request: Request, refresh_requested: bool = False
 ):
-
-    return templates.TemplateResponse(
-        "entries.html",
-        {
-            "request": request,
-            "settings": await bk.get_settings(),
-            "entries": list(bk.list_entries(feed_id=feed_id)),
-            "feed": await bk.get_feed_config(id=feed_id),
-            "refresh_requested": refresh_requested,
-        },
-    )
+    response = {
+        "settings": await bk.get_settings(),
+        "entries": list(bk.list_entries(feed_id=feed_id)),
+        "feed": await bk.get_feed_config(id=feed_id),
+        "refresh_requested": refresh_requested,
+    }
+
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "entries.html",
+            {"request": request, **response},
+        )
 
 
 @app.get("/recent/", response_class=HTMLResponse)
 async def list_recent_feed_entries(request: Request, refresh_requested: bool = False):
-
-    return templates.TemplateResponse(
-        "entries.html",
-        {
-            "request": request,
-            "settings": await bk.get_settings(),
-            "entries": list(bk.list_entries(feed_id=None, recent=True)),
-            "refresh_requested": refresh_requested,
-            "recent": True,
-        },
-    )
+    response = {
+        "settings": await bk.get_settings(),
+        "entries": list(bk.list_entries(feed_id=None, recent=True)),
+        "refresh_requested": refresh_requested,
+        "recent": True,
+    }
+
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "entries.html",
+            {"request": request, **response},
+        )
 
 
 @app.get("/read/{feed_entry_id}", response_class=HTMLResponse)
 async def read(request: Request, feed_entry_id: str, redrive: bool = False):
+    response = {
+        "content": await bk.get_entry_content(
+            feed_entry_id=feed_entry_id, redrive=redrive
+        ),
+        "settings": await bk.get_settings(),
+    }
 
-    return templates.TemplateResponse(
-        "read.html",
-        {
-            "request": request,
-            "content": await bk.get_entry_content(
-                feed_entry_id=feed_entry_id, redrive=redrive
-            ),
-            "settings": await bk.get_settings(),
-        },
-    )
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "read.html",
+            {"request": request, **response},
+        )
 
 
 @app.get("/settings/", response_class=HTMLResponse)
 async def settings(
     request: Request, update_status: bool = False, update_exception: str = None
 ):
-
-    return templates.TemplateResponse(
-        "settings.html",
-        {
-            "request": request,
-            "themes": Themes._member_names_,
-            "content_handler_choices": await bk.list_content_handler_choices(),
-            "summarization_handler_choices": await bk.list_summarization_handler_choices(),
-            "notification_handler_choices": await bk.list_notification_handler_choices(),
-            "settings": await bk.get_settings(),
-            "notification": bk.get_handlers(),
-            "update_status": update_status,
-            "update_exception": update_exception,
-        },
-    )
+    response = {
+        "themes": Themes._member_names_,
+        "content_handler_choices": await bk.list_content_handler_choices(),
+        "summarization_handler_choices": await bk.list_summarization_handler_choices(),
+        "notification_handler_choices": await bk.list_notification_handler_choices(),
+        "settings": await bk.get_settings(),
+        "notification": bk.get_handlers(),
+        "update_status": update_status,
+        "update_exception": update_exception,
+    }
+
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "settings.html", {"request": request, **response}
+        )
 
 
 @app.get("/settings/{handler}", response_class=HTMLResponse)
@@ -199,18 +215,20 @@ async def handler_settings(
     update_status: bool = False,
     update_exception: str = None,
 ):
-
-    return templates.TemplateResponse(
-        "handler_config.html",
-        {
-            "request": request,
-            "handler": bk.get_handler_config(handler=handler),
-            "schema": bk.get_handler_schema(handler=handler),
-            "settings": await bk.get_settings(),
-            "update_status": update_status,
-            "update_exception": update_exception,
-        },
-    )
+    response = {
+        "handler": bk.get_handler_config(handler=handler),
+        "schema": bk.get_handler_schema(handler=handler),
+        "settings": await bk.get_settings(),
+        "update_status": update_status,
+        "update_exception": update_exception,
+    }
+
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "handler_config.html", {"request": request, **response}
+        )
 
 
 @app.post("/api/update_handler/", status_code=status.HTTP_200_OK)
@@ -410,31 +428,35 @@ async def feeds(
 async def feed_settings(
     request: Request, id: str, update_status: bool = False, update_exception: str = None
 ):
-
-    return templates.TemplateResponse(
-        "feed_config.html",
-        {
-            "request": request,
-            "settings": await bk.get_settings(),
-            "feed": await bk.get_feed_config(id=id),
-            "update_status": update_status,
-            "update_exception": update_exception,
-        },
-    )
+    response = {
+        "settings": await bk.get_settings(),
+        "feed": await bk.get_feed_config(id=id),
+        "update_status": update_status,
+        "update_exception": update_exception,
+    }
+
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "feed_config.html", {"request": request, **response}
+        )
 
 
 @app.get("/feeds/new/", response_class=HTMLResponse)
 async def new_feed(request: Request, update_exception: str = None):
-
-    return templates.TemplateResponse(
-        "feed_config.html",
-        {
-            "request": request,
-            "settings": await bk.get_settings(),
-            "feed": {},
-            "update_exception": update_exception,
-        },
-    )
+    response = {
+        "settings": await bk.get_settings(),
+        "feed": {},
+        "update_exception": update_exception,
+    }
+
+    if request.headers.get("accept") == JSON:
+        return JSONResponse(content=response, media_type=JSON)
+    else:
+        return templates.TemplateResponse(
+            "feed_config.html", {"request": request, **response}
+        )
 
 
 # Utility APIs - mostly used to orchestrate tests, but available for whatever

app/context.py

@@ -6,7 +6,7 @@
 from typing import Any, List, Mapping, Optional, Type
 
 from markdown2 import markdown
-from pydantic import BaseModel, validator
+from pydantic import BaseModel, Field, validator
 from readabilipy import simple_json_from_html_string
 
 from app.content import content_retrieval_handlers
@@ -51,7 +51,7 @@ class GlobalSettings(BaseModel):
 
     finished_onboarding: bool = False
 
-    db: Any
+    db: Any = Field(exclude=True)
 
     @validator("db")
     def validate_db(cls, val):

go.mod

@@ -1,3 +1,10 @@
 module leozqin/precis_tests
 
 go 1.21.11
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

go.sum

@@ -0,0 +1,9 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=