diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..633d21e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +**/.venv +**/node_modules diff --git a/Makefile b/Makefile index b11598b..b4ec149 100644 --- a/Makefile +++ b/Makefile @@ -37,5 +37,4 @@ clean: .PHONY: test test: - go test tests/integration/web_static_test.go -v - go test tests/integration/web_dynamic_test.go -v + go test tests/integration/*.go -v diff --git a/README.md b/README.md index 1df2ec8..7069332 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,20 @@ Feel free to edit the [docker-compose.yml](docker-compose.yml) as needed to fit - Node 20 or higher (use nvm) ## Development Instructions -To install, create a fresh venv and then: +### Application +To install the RSS application, create a fresh venv and then: ```bash make dev ``` Then to develop, in one terminal start tailwind by doing `make tw`. Then, in other start the main app by doing `make run`. +### Integration Tests +Precis has integration tests that are written in Go. They are automated to run during the pull request pipeline, but they also be run locally. + +First, install the version of Go specified in `go.mod`. I recommend to use a Golang version manager such as `gvm` or `g`. + +Then, start the application using `make run`. Finally, run the integration tests with `make test`. + # Features ## OPML Import/Export Precis supports exporting your current set of feeds as OPML, as well as importing feeds from other OPML files. You can find options on the `/feeds/` page, or use the CLI as described below. diff --git a/app/app.py b/app/app.py index 15723c9..c737871 100644 --- a/app/app.py +++ b/app/app.py @@ -6,7 +6,7 @@ from fastapi import FastAPI, Form, UploadFile, status from fastapi.requests import Request -from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates from fastapi_utils.tasks import repeat_every @@ -18,6 +18,8 @@ from app.rss import PrecisRSS from app.storage.engine import load_storage_config +JSON = "application/json" + logger = getLogger("uvicorn.error") base_path = Path(__file__).parent @@ -89,21 +91,23 @@ async def root(request: Request): return RedirectResponse("/onboarding/") -@app.get("/about", response_class=HTMLResponse) +@app.get("/about") async def about( request: Request, update_status: bool = False, update_exception: str = None ): - - return templates.TemplateResponse( - "about.html", - { - "request": request, - "settings": await bk.get_settings(), - "update_status": update_status, - "update_exception": update_exception, - **bk.about(), - }, - ) + response = { + "settings": await bk.get_settings(), + "update_status": update_status, + "update_exception": update_exception, + **bk.about(), + } + + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "about.html", {"request": request, **response} + ) @app.get("/health", response_model=HealthCheck) @@ -112,84 +116,96 @@ async def health_check(request: Request) -> HealthCheck: return bk.health_check() -@app.get("/onboarding/", response_class=HTMLResponse) +@app.get("/onboarding/") async def onboarding(request: Request): + response = {"settings": await bk.get_settings()} - return templates.TemplateResponse( - "onboarding.html", - { - "request": request, - "settings": await bk.get_settings(), - }, - ) + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "onboarding.html", + {"request": request, **response}, + ) @app.get("/list-entries/{feed_id}", response_class=HTMLResponse) async def list_entries_by_feed( feed_id: str, request: Request, refresh_requested: bool = False ): - - return templates.TemplateResponse( - "entries.html", - { - "request": request, - "settings": await bk.get_settings(), - "entries": list(bk.list_entries(feed_id=feed_id)), - "feed": await bk.get_feed_config(id=feed_id), - "refresh_requested": refresh_requested, - }, - ) + response = { + "settings": await bk.get_settings(), + "entries": list(bk.list_entries(feed_id=feed_id)), + "feed": await bk.get_feed_config(id=feed_id), + "refresh_requested": refresh_requested, + } + + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "entries.html", + {"request": request, **response}, + ) @app.get("/recent/", response_class=HTMLResponse) async def list_recent_feed_entries(request: Request, refresh_requested: bool = False): - - return templates.TemplateResponse( - "entries.html", - { - "request": request, - "settings": await bk.get_settings(), - "entries": list(bk.list_entries(feed_id=None, recent=True)), - "refresh_requested": refresh_requested, - "recent": True, - }, - ) + response = { + "settings": await bk.get_settings(), + "entries": list(bk.list_entries(feed_id=None, recent=True)), + "refresh_requested": refresh_requested, + "recent": True, + } + + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "entries.html", + {"request": request, **response}, + ) @app.get("/read/{feed_entry_id}", response_class=HTMLResponse) async def read(request: Request, feed_entry_id: str, redrive: bool = False): + response = { + "content": await bk.get_entry_content( + feed_entry_id=feed_entry_id, redrive=redrive + ), + "settings": await bk.get_settings(), + } - return templates.TemplateResponse( - "read.html", - { - "request": request, - "content": await bk.get_entry_content( - feed_entry_id=feed_entry_id, redrive=redrive - ), - "settings": await bk.get_settings(), - }, - ) + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "read.html", + {"request": request, **response}, + ) @app.get("/settings/", response_class=HTMLResponse) async def settings( request: Request, update_status: bool = False, update_exception: str = None ): - - return templates.TemplateResponse( - "settings.html", - { - "request": request, - "themes": Themes._member_names_, - "content_handler_choices": await bk.list_content_handler_choices(), - "summarization_handler_choices": await bk.list_summarization_handler_choices(), - "notification_handler_choices": await bk.list_notification_handler_choices(), - "settings": await bk.get_settings(), - "notification": bk.get_handlers(), - "update_status": update_status, - "update_exception": update_exception, - }, - ) + response = { + "themes": Themes._member_names_, + "content_handler_choices": await bk.list_content_handler_choices(), + "summarization_handler_choices": await bk.list_summarization_handler_choices(), + "notification_handler_choices": await bk.list_notification_handler_choices(), + "settings": await bk.get_settings(), + "notification": bk.get_handlers(), + "update_status": update_status, + "update_exception": update_exception, + } + + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "settings.html", {"request": request, **response} + ) @app.get("/settings/{handler}", response_class=HTMLResponse) @@ -199,18 +215,20 @@ async def handler_settings( update_status: bool = False, update_exception: str = None, ): - - return templates.TemplateResponse( - "handler_config.html", - { - "request": request, - "handler": bk.get_handler_config(handler=handler), - "schema": bk.get_handler_schema(handler=handler), - "settings": await bk.get_settings(), - "update_status": update_status, - "update_exception": update_exception, - }, - ) + response = { + "handler": bk.get_handler_config(handler=handler), + "schema": bk.get_handler_schema(handler=handler), + "settings": await bk.get_settings(), + "update_status": update_status, + "update_exception": update_exception, + } + + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "handler_config.html", {"request": request, **response} + ) @app.post("/api/update_handler/", status_code=status.HTTP_200_OK) @@ -410,31 +428,35 @@ async def feeds( async def feed_settings( request: Request, id: str, update_status: bool = False, update_exception: str = None ): - - return templates.TemplateResponse( - "feed_config.html", - { - "request": request, - "settings": await bk.get_settings(), - "feed": await bk.get_feed_config(id=id), - "update_status": update_status, - "update_exception": update_exception, - }, - ) + response = { + "settings": await bk.get_settings(), + "feed": await bk.get_feed_config(id=id), + "update_status": update_status, + "update_exception": update_exception, + } + + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "feed_config.html", {"request": request, **response} + ) @app.get("/feeds/new/", response_class=HTMLResponse) async def new_feed(request: Request, update_exception: str = None): - - return templates.TemplateResponse( - "feed_config.html", - { - "request": request, - "settings": await bk.get_settings(), - "feed": {}, - "update_exception": update_exception, - }, - ) + response = { + "settings": await bk.get_settings(), + "feed": {}, + "update_exception": update_exception, + } + + if request.headers.get("accept") == JSON: + return JSONResponse(content=response, media_type=JSON) + else: + return templates.TemplateResponse( + "feed_config.html", {"request": request, **response} + ) # Utility APIs - mostly used to orchestrate tests, but available for whatever diff --git a/app/context.py b/app/context.py index 5e98f55..df1fc86 100644 --- a/app/context.py +++ b/app/context.py @@ -6,7 +6,7 @@ from typing import Any, List, Mapping, Optional, Type from markdown2 import markdown -from pydantic import BaseModel, validator +from pydantic import BaseModel, Field, validator from readabilipy import simple_json_from_html_string from app.content import content_retrieval_handlers @@ -51,7 +51,7 @@ class GlobalSettings(BaseModel): finished_onboarding: bool = False - db: Any + db: Any = Field(exclude=True) @validator("db") def validate_db(cls, val): diff --git a/go.mod b/go.mod index 730773d..cfca697 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,10 @@ module leozqin/precis_tests go 1.21.11 + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e20fa14 --- /dev/null +++ b/go.sum @@ -0,0 +1,9 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/integration/models.go b/tests/integration/models.go new file mode 100644 index 0000000..9558b48 --- /dev/null +++ b/tests/integration/models.go @@ -0,0 +1,91 @@ +/* +These are partial implementations of either pydantic models or ersponse +models to facilitate testing +*/ + +package precis_test + +type Feed struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type FeedEntry struct { + Id string `json:"id,omitempty"` + FeedID string `json:"feed_id,omitempty"` + FeedName string `json:"feed_name,omitempty"` + Title string `json:"title,omitempty"` + URL string `json:"url,omitempty"` + PublishedAt string `json:"published_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` + Byline string `json:"byline,omitempty"` + Preview string `json:"preview,omitempty"` + Content string `json:"content,omitempty"` + Summary string `json:"summary,omitempty"` + WordCount int `json:"word_count,omitempty"` + ReadingLevel int `json:"reading_level,omitempty"` + ReadingTime int `json:"reading_time,omitempty"` +} + +type Handler struct { + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Configured bool `json:"configured,omitempty"` +} + +type Settings struct { + SendNotification bool `json:"send_notification,omitempty"` + Theme string `json:"theme,omitempty"` + RefreshInterval int `json:"refresh_interval,omitempty"` + ReadingSpeed int `json:"reading_speed,omitempty"` + NotificationHandlerKey string `json:"notification_handler_key,omitempty"` + SummarizationHandlerKey string `json:"summarization_handler_key,omitempty"` + ContentRetrievalHandlerKey string `json:"content_retrieval_handler_key,omitempty"` + RecentHours int `json:"recent_hours,omitempty"` + FinishedOnboarding bool `json:"finished_onboarding,omitempty"` +} + +type AboutResponse struct { + Settings Settings `json:"settings"` + UpdateStatus bool `json:"update_status,omitempty"` + UpdateException string `json:"update_exception,omitempty"` + Version string `json:"version,omitempty"` + PythonVersion string `json:"python_version,omitempty"` + FastAPIVersion string `json:"fastapi_version,omitempty"` + Docker bool `json:"docker,omitempty"` + StorageHandler string `json:"storage_handler,omitempty"` + Github string `json:"github,omitempty"` +} + +type SettingsResponse struct { + Themes []string `json:"themes,omitempty"` + ContentHandlerChoices []string `json:"content_handler_choices,omitempty"` + SummarizationHandlerChoices []string `json:"summarization_handler_choices,omitempty"` + NotificationHandlerChoices []string `json:"notification_handler_choices,omitempty"` + UpdateStatus bool `json:"update_status,omitempty"` + UpdateException string `json:"update_exception,omitempty"` + + Settings Settings `json:"settings,omitempty"` +} + +type OnboardingResponse struct { + Settings Settings `json:"settings"` +} + +type HandlerConfig struct { + Type string `json:"type,omitempty"` + Config string `json:"config,omitempty"` +} + +type HandlerSettingsResponse struct { + Handler HandlerConfig `json:"handler,omitempty"` + Schema string `json:"schema,omitempty"` + Settings Settings `json:"settings,omitempty"` + UpdateStatus bool `json:"update_status,omitempty"` + UpdateException string `json:"update_exception,omitempty"` +} + +type ReadFeedEntryResponse struct { + Content FeedEntry + Settings Settings +} diff --git a/tests/integration/web_api_test.go b/tests/integration/web_api_test.go new file mode 100644 index 0000000..22689b3 --- /dev/null +++ b/tests/integration/web_api_test.go @@ -0,0 +1,210 @@ +/* +These tests test the content of the API responses +*/ +package precis_test + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +var client = &http.Client{} + +func TestAbout(t *testing.T) { + req, err := http.NewRequest("GET", baseURL+"/about", nil) + req.Header.Set("accept", "application/json") + + if err != nil { + t.Fatal(err) + } + + res, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + + if res.StatusCode != http.StatusOK { + t.Errorf("About page returned status code %d, expected %d", res.StatusCode, http.StatusOK) + } + + var about AboutResponse + + if err := json.NewDecoder(res.Body).Decode(&about); err != nil { + t.Fatal(err) + } + + defer res.Body.Close() + + assert.Equal(t, false, about.UpdateStatus) + assert.Empty(t, about.UpdateException) + + assert.NotEmpty(t, about.Version) + assert.NotEmpty(t, about.PythonVersion) + assert.NotEmpty(t, about.FastAPIVersion) + assert.IsType(t, true, about.Docker) + assert.NotEmpty(t, about, about.StorageHandler) + assert.NotEmpty(t, about.Github) + +} + +func TestSettings(t *testing.T) { + req, err := http.NewRequest("GET", baseURL+"/settings", nil) + req.Header.Set("accept", "application/json") + + if err != nil { + t.Fatal(err) + } + + res, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + + if res.StatusCode != http.StatusOK { + t.Errorf("Settings page returned status code %d, expected %d", res.StatusCode, http.StatusOK) + } + + var settings SettingsResponse + + if err := json.NewDecoder(res.Body).Decode(&settings); err != nil { + t.Fatal(err) + } + + defer res.Body.Close() + + assert.NotEmpty(t, settings.Themes) + assert.NotEmpty(t, settings.ContentHandlerChoices) + assert.NotEmpty(t, settings.SummarizationHandlerChoices) + assert.NotEmpty(t, settings.NotificationHandlerChoices) + assert.Equal(t, false, settings.UpdateStatus) + assert.Empty(t, settings.UpdateException) + + /* Test Settings Defaults */ + + var s = settings.Settings + assert.Equal(t, "forest", s.Theme) + assert.Equal(t, true, s.SendNotification) + assert.Equal(t, 5, s.RefreshInterval) + assert.Equal(t, 238, s.ReadingSpeed) + assert.Equal(t, 36, s.RecentHours) + assert.IsType(t, true, s.FinishedOnboarding) + + assert.Contains(t, settings.NotificationHandlerChoices, s.NotificationHandlerKey) + assert.Contains(t, settings.SummarizationHandlerChoices, s.SummarizationHandlerKey) + assert.Contains(t, settings.ContentHandlerChoices, s.ContentRetrievalHandlerKey) +} + +func TestOnboarding(t *testing.T) { + req, err := http.NewRequest("GET", baseURL+"/onboarding/", nil) + req.Header.Set("accept", "application/json") + + if err != nil { + t.Fatal(err) + } + + res, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + + if res.StatusCode != http.StatusOK { + t.Errorf("Onboarding page returned status code %d, expected %d", res.StatusCode, http.StatusOK) + } + + var onboarding OnboardingResponse + + if err := json.NewDecoder(res.Body).Decode(&onboarding); err != nil { + t.Fatal(err) + } + + defer res.Body.Close() + + assert.NotEmpty(t, onboarding.Settings) +} + +func TestHandlerSettings(t *testing.T) { + req, err := http.Get(baseURL + "/util/list-handlers") + if err != nil { + t.Fatal(err) + } + + var handlers []Handler + + if err := json.NewDecoder(req.Body).Decode(&handlers); err != nil { + t.Fatal(err) + } + + for _, handler := range handlers { + req, err := http.NewRequest("GET", baseURL+"/settings/"+handler.Name, nil) + req.Header.Set("accept", "application/json") + + if err != nil { + t.Fatal(err) + } + + res, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + + var requestedHandler HandlerSettingsResponse + + if err := json.NewDecoder(res.Body).Decode(&requestedHandler); err != nil { + t.Fatal(err) + } + + defer res.Body.Close() + + assert.NotEmpty(t, requestedHandler.Settings) + assert.NotEmpty(t, requestedHandler.Schema) + assert.Equal(t, requestedHandler.Handler.Type, handler.Name) + } + +} + +func TestReadEntry(t *testing.T) { + req, err := http.Get(baseURL + "/util/list-feed-entries") + if err != nil { + t.Fatal(err) + } + + var entries []FeedEntry + + if err := json.NewDecoder(req.Body).Decode(&entries); err != nil { + t.Fatal(err) + } + + for _, entry := range entries { + req, err := http.NewRequest("GET", baseURL+"/read/"+entry.Id, nil) + req.Header.Set("accept", "application/json") + + if err != nil { + t.Fatal(err) + } + + res, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + + var feedEntry ReadFeedEntryResponse + + if err := json.NewDecoder(res.Body).Decode(&feedEntry); err != nil { + t.Fatal(err) + } + + defer res.Body.Close() + + requestedEntry := feedEntry.Content + + assert.Equal(t, entry.Id, requestedEntry.Id) + assert.NotEmpty(t, requestedEntry.FeedID) + assert.NotEmpty(t, requestedEntry.FeedName) + assert.NotEmpty(t, requestedEntry.Title) + assert.NotEmpty(t, requestedEntry.URL) + } + +} diff --git a/tests/integration/web_dynamic_test.go b/tests/integration/web_dynamic_test.go index f88c8e7..9c7eb17 100644 --- a/tests/integration/web_dynamic_test.go +++ b/tests/integration/web_dynamic_test.go @@ -7,28 +7,9 @@ package precis_test import ( "encoding/json" "net/http" - "os" "testing" ) -var baseURL = os.Getenv("RSS_BASE_URL") - -type Feed struct { - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` -} - -type FeedEntry struct { - Id string `json:"id,omitempty"` - Title string `json:"title,omitempty"` -} - -type Handler struct { - Name string `json:"name,omitempty"` - Type string `json:"type,omitempty"` - Configured bool `json:"configured,omitempty"` -} - func TestFeedPage(t *testing.T) { req, err := http.Get(baseURL + "/util/list-feeds") if err != nil {