SingleFile integration

backend/lynx/lynx.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pocketbase/pocketbase/tools/security"
 
 	"main/lynx/summarizer"
+	"main/lynx/singlefile"
 )
 
 var parseUrlHandlerFunc = handleParseURL
@@ -91,6 +92,9 @@ func InitializePocketbase(app core.App) {
 		routine.FireAndForget(func() {
 			CurrentSummarizer.MaybeSummarizeLink(app, e.Model.GetId())
 		})
+		routine.FireAndForget(func() {
+			singlefile.MaybeArchiveLink(app, e.Model.GetId())
+		})
 		return nil
 	})
 }

backend/lynx/singlefile/singlefile.go

@@ -0,0 +1,169 @@
+package singlefile
+
+// If the environment variable SINGLEFILE_URL is set, 
+// this will send a request to that url to create an 
+// archive. The resulting html will be saved as a 
+// file attachment to the link.
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/pocketbase/dbx"
+	"github.com/pocketbase/pocketbase/core"
+	"github.com/pocketbase/pocketbase/tools/filesystem"
+)
+
+const (
+	charset   = "abcdefghijklmnopqrstuvwxyz0123456789"
+	randomLen = 10
+)
+
+var seededRand *rand.Rand = rand.New(rand.NewSource(time.Now().UnixNano()))
+
+func generateFilenameSuffix() string {
+	b := make([]byte, randomLen)
+	for i := range b {
+		b[i] = charset[seededRand.Intn(len(charset))]
+	}
+	return string(b)
+}
+
+func MaybeArchiveLink(app core.App, linkID string) {
+	logger := app.Logger().With("action", "createArchive", "linkID", linkID)
+
+	link, err := app.Dao().FindRecordById("links", linkID)
+	if err != nil {
+		logger.Error("Failed to find link", "error", err)
+		return
+	}
+
+	singlefileURL := os.Getenv("SINGLEFILE_URL")
+	if singlefileURL == "" {
+		logger.Info("SINGLEFILE_URL not set, skipping archive creation")
+		return
+	}
+
+	originalURL := link.GetString("original_url")
+	if originalURL == "" {
+		logger.Error("Link has no original_url")
+		return
+	}
+
+	// Create a file using Pocketbase's filesystem
+	// fileKey = the name. This is what is stored on the model
+	// fileName = the full path including the directory for
+	// the link record. This should not be stored on the
+	// model beacuse it's computed by pocketbase
+	fileKey := fmt.Sprintf("archive_%s.html", generateFilenameSuffix())
+	fileName := link.BaseFilesPath() + "/" + fileKey
+	fs, err := app.NewFilesystem()
+	if err != nil {
+		logger.Error("Failed to create filesystem", "error", err)
+		return
+	}
+	defer fs.Close()
+
+	exists, err := fs.Exists(fileName)
+	if exists || err != nil {
+		logger.Info("Skipping archive creation, file already exists")
+		return
+	}
+
+	userID := link.GetString("user")
+	cookiesJSON, err := getUserCookiesJSON(app, userID, originalURL)
+	if err != nil {
+		logger.Error("Failed to get user cookies", "error", err)
+		// Continue without cookies if there's an error
+	}
+
+	formData := url.Values{}
+	formData.Set("url", originalURL)
+	if cookiesJSON != "" {
+		formData.Set("cookies", cookiesJSON)
+	}
+	resp, err := http.PostForm(singlefileURL, formData)
+	if err != nil {
+		logger.Error("Failed to send request to singlefile service", "error", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		logger.Error("Singlefile service returned non-OK status", "statusCode", resp.StatusCode)
+		return
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		logger.Error("Failed to read response body", "error", err)
+		return
+	}
+
+  if len(body) == 0 {
+    logger.Error("Received empty response from singlefile service")
+    return
+  }
+
+	fsFile, err := filesystem.NewFileFromBytes(body, fileName)
+	if err != nil {
+		logger.Error("Failed to create archive file", "error", err)
+		return
+	}
+
+	err = fs.UploadFile(fsFile, fileName)
+	if err != nil {
+		logger.Error("Failed to upload archive file", "error", err)
+		return
+	}
+
+	link.Set("archive", fileKey)
+	if err := app.Dao().SaveRecord(link); err != nil {
+		logger.Error("Failed to update link with archive information", "error", err)
+		return
+	}
+
+	logger.Info("Successfully created archive for link")
+}
+
+func getUserCookiesJSON(app core.App, userID string, urlStr string) (string, error) {
+	parsedURL, err := url.Parse(urlStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse URL: %w", err)
+	}
+
+	cookieRecords, err := app.Dao().FindRecordsByFilter(
+		"user_cookies",
+		"user = {:user} && domain = {:domain}",
+		"-created",
+		10,
+		0,
+		dbx.Params{"user": userID, "domain": parsedURL.Hostname()},
+	)
+	if err != nil {
+		return "", fmt.Errorf("failed to fetch user cookies: %w", err)
+	}
+
+	var cookiesData []string
+	for _, record := range cookieRecords {
+		cookieData := fmt.Sprintf("%s,%s,%s",
+			record.GetString("name"),
+			record.GetString("value"),
+			record.GetString("domain"),
+		)
+		cookiesData = append(cookiesData, cookieData)
+	}
+
+	cookiesJSON, err := json.Marshal(cookiesData)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal cookies to JSON: %w", err)
+	}
+
+	return string(cookiesJSON), nil
+}

backend/lynx/singlefile/singlefile_test.go

@@ -0,0 +1,201 @@
+package singlefile
+
+import (
+  "encoding/json"
+  "net/http"
+  "net/http/httptest"
+  "os"
+  "testing"
+
+  "github.com/pocketbase/pocketbase/core"
+  "github.com/pocketbase/pocketbase/tests"
+)
+
+func TestMaybeArchiveLink(t *testing.T) {
+  testApp, err := tests.NewTestApp("../../test_pb_data")
+  if err != nil {
+    t.Fatal(err)
+  }
+  defer testApp.Cleanup()
+
+  // Variable to track if the server was hit
+  var serverHit bool
+
+  // Create a test server to mock the SingleFile service
+  server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+    serverHit = true
+    switch r.URL.Query().Get("status") {
+    case "error":
+      w.WriteHeader(http.StatusInternalServerError)
+    case "empty":
+      w.WriteHeader(http.StatusOK)
+    default:
+      w.WriteHeader(http.StatusOK)
+      w.Write([]byte("<html><body>Archived content</body></html>"))
+    }
+  }))
+  defer server.Close()
+
+  testCases := []struct {
+    name           string
+    setupEnv       func()
+    cleanupEnv     func()
+    expectedResult func(*testing.T, core.App, string, bool)
+  }{
+    {
+      name: "SINGLEFILE_URL not set",
+      setupEnv: func() {
+        os.Unsetenv("SINGLEFILE_URL")
+      },
+      cleanupEnv: func() {},
+      expectedResult: func(t *testing.T, app core.App, linkID string, hit bool) {
+        link, err := app.Dao().FindRecordById("links", linkID)
+        if err != nil {
+          t.Fatalf("Failed to find link: %v", err)
+        }
+        if link.GetString("archive") != "" {
+          t.Error("Expected archive field to be empty, but it was set")
+        }
+        if hit {
+          t.Error("Expected server not to be hit, but it was")
+        }
+      },
+    },
+    {
+      name: "SINGLEFILE_URL set - successful response",
+      setupEnv: func() {
+        os.Setenv("SINGLEFILE_URL", server.URL)
+      },
+      cleanupEnv: func() {
+        os.Unsetenv("SINGLEFILE_URL")
+      },
+      expectedResult: func(t *testing.T, app core.App, linkID string, hit bool) {
+        link, err := app.Dao().FindRecordById("links", linkID)
+        if err != nil {
+          t.Fatalf("Failed to find link: %v", err)
+        }
+        archive := link.GetString("archive")
+        if archive == "" {
+          t.Error("Expected archive field to be set, but it was empty")
+        }
+
+        fs, err := app.NewFilesystem()
+        if err != nil {
+          t.Fatal(err)
+        }
+        defer fs.Close()
+
+        exists, err := fs.Exists(link.BaseFilesPath() + "/" + archive)
+        if err != nil {
+          t.Fatal(err)
+        }
+        if !exists {
+          t.Error("Expected archive file to exist, but it doesn't")
+        }
+        if !hit {
+          t.Error("Expected server to be hit, but it wasn't")
+        }
+      },
+    },
+    {
+      name: "SINGLEFILE_URL set - unsuccessful response",
+      setupEnv: func() {
+        os.Setenv("SINGLEFILE_URL", server.URL+"?status=error")
+      },
+      cleanupEnv: func() {
+        os.Unsetenv("SINGLEFILE_URL")
+      },
+      expectedResult: func(t *testing.T, app core.App, linkID string, hit bool) {
+        link, err := app.Dao().FindRecordById("links", linkID)
+        if err != nil {
+          t.Fatalf("Failed to find link: %v", err)
+        }
+        if link.GetString("archive") != "" {
+          t.Error("Expected archive field to be empty, but it was set")
+        }
+        if !hit {
+          t.Error("Expected server to be hit, but it wasn't")
+        }
+      },
+    },
+    {
+      name: "SINGLEFILE_URL set - empty response",
+      setupEnv: func() {
+        os.Setenv("SINGLEFILE_URL", server.URL+"?status=empty")
+      },
+      cleanupEnv: func() {
+        os.Unsetenv("SINGLEFILE_URL")
+      },
+      expectedResult: func(t *testing.T, app core.App, linkID string, hit bool) {
+        link, err := app.Dao().FindRecordById("links", linkID)
+        if err != nil {
+          t.Fatalf("Failed to find link: %v", err)
+        }
+        if link.GetString("archive") != "" {
+          t.Error("Expected archive field to be empty, but it was set")
+        }
+        if !hit {
+          t.Error("Expected server to be hit, but it wasn't")
+        }
+      },
+    },
+  }
+
+  for _, tc := range testCases {
+    t.Run(tc.name, func(t *testing.T) {
+      tc.setupEnv()
+      defer tc.cleanupEnv()
+
+      serverHit = false
+
+      MaybeArchiveLink(testApp, "8n3iq8dt6vwi4ph")
+
+      tc.expectedResult(t, testApp, "8n3iq8dt6vwi4ph", serverHit)
+
+      link, err := testApp.Dao().FindRecordById("links", "8n3iq8dt6vwi4ph")
+      if err != nil {
+        t.Fatalf("Failed to find link after test: %v", err)
+      }
+      if archive := link.GetString("archive"); archive != "" {
+        fs, err := testApp.NewFilesystem()
+        if err != nil {
+          t.Fatalf("Failed to create filesystem for cleanup: %v", err)
+        }
+        defer fs.Close()
+
+        err = fs.Delete(link.BaseFilesPath() + "/" + archive)
+        if err != nil {
+          t.Fatalf("Failed to delete archive file: %v", err)
+        }
+
+        // Reset the archive field in the database
+        link.Set("archive", "")
+        if err := testApp.Dao().SaveRecord(link); err != nil {
+          t.Fatalf("Failed to reset archive field: %v", err)
+        }
+      }
+    })
+  }
+}
+
+func TestGetUserCookiesJSON(t *testing.T) {
+  testApp, err := tests.NewTestApp("../../test_pb_data")
+  if err != nil {
+    t.Fatal(err)
+  }
+  defer testApp.Cleanup()
+
+  cookiesJSON, err := getUserCookiesJSON(testApp, "c0qbygabvsrlixp", "https://www.example.com")
+  if err != nil {
+    t.Fatal(err)
+  }
+
+  var cookies []string
+  err = json.Unmarshal([]byte(cookiesJSON), &cookies)
+  if err != nil {
+    t.Fatal(err)
+  }
+  if len(cookies) != 1 {
+    t.Error("Expected exactly one cookie, but got none")
+  }
+}