chore: update template

axum/firecrawl/Cargo.toml

@@ -4,10 +4,10 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-shuttle-runtime = { version = "0.47.0", git = "https://github.com/joshua-mo-143/shuttle.git", branch = "firecrawl" }
-
 axum = "0.7.4"
-shuttle-firecrawl = { version = "0.47.0", git = "https://github.com/joshua-mo-143/shuttle.git", branch = "firecrawl" }
-shuttle-axum = { version = "0.47.0", git = "https://github.com/joshua-mo-143/shuttle.git", branch = "firecrawl" }
+firecrawl = "1.0.0"
+serde = { version = "1.0.210", features = ["derive"] }
+shuttle-axum = "0.48.0"
+shuttle-runtime = "0.48.0"
 tokio = "1.28.2"
-serde_json = "1.0.128"
+url = { version = "2.5.2", features = ["serde"] }

axum/firecrawl/src/main.rs

@@ -1,34 +1,61 @@
-use axum::{routing::get, Router, extract::State, Json, response::IntoResponse};
-use serde_json::json;
-
-use shuttle_firecrawl::firecrawl::FirecrawlApp;
+use axum::{extract::State, response::IntoResponse, routing::get, Json, Router};
+use firecrawl::{
+    scrape::{ScrapeFormats, ScrapeOptions},
+    FirecrawlApp,
+};
+use serde::Deserialize;
+use url::Url;
 
 #[derive(Clone)]
 struct AppState {
     firecrawl: FirecrawlApp,
 }
 
 impl AppState {
-    fn new(firecrawl: FirecrawlApp) -> Self {
+    fn new() -> Self {
+        let firecrawl_api_key =
+            std::env::var("FIRECRAWL_API_KEY").expect("FIRECRAWL_API_KEY to be set as an env var");
+        let firecrawl = FirecrawlApp::new(firecrawl_api_key).unwrap();
         Self { firecrawl }
     }
 }
 
-async fn scrape_page(State(state): State<AppState>) -> impl IntoResponse {
-        let scrape_response = state.firecrawl.scrape_url("https://firecrawl.dev", Some(json!({
-            "formats": ["markdown", "html"]
-        }))).await.unwrap();
+#[derive(Deserialize)]
+struct Request {
+    url: Url,
+}
 
-        Json(scrape_response)
+async fn scrape_page(
+    State(state): State<AppState>,
+    Json(json): Json<Request>,
+) -> impl IntoResponse {
+    let scrape_options = ScrapeOptions {
+        formats: Some(vec![ScrapeFormats::HTML, ScrapeFormats::Markdown]),
+        ..Default::default()
+    };
+
+    let scrape_response = state
+        .firecrawl
+        .scrape_url(&json.url, scrape_options)
+        .await
+        .unwrap();
+
+    Json(scrape_response)
 }
 
 #[shuttle_runtime::main]
-async fn main(#[shuttle_firecrawl::Firecrawl(
-    api_key = "{secrets.FIRECRAWL_API_KEY}"
-)] firecrawl: FirecrawlApp) -> shuttle_axum::ShuttleAxum {
-    let state = AppState::new(firecrawl);
-
-    let router = Router::new().route("/", get(scrape_page)).with_state(state);
+async fn main(
+    #[shuttle_runtime::Secrets] secrets: shuttle_runtime::SecretStore,
+) -> shuttle_axum::ShuttleAxum {
+    secrets.into_iter().for_each(|(key, val)| {
+        std::env::set_var(key, val);
+    });
+
+    let state = AppState::new();
+
+    let router = Router::new()
+        .route("/scrape", get(scrape_page))
+        .with_state(state);
 
     Ok(router.into())
 }

templates.toml

@@ -49,7 +49,7 @@ template = "axum"
 
 [starters.axum-firecrawl]
 title = "Axum with Firecrawl"
-description = "Using the Firecrawl annotation resource with Shuttle"
+description = "Using the Firecrawl AI service with Shuttle to scrape pages"
 path = "axum/firecrawl"
 use_cases = ["Web app"]
 tags = ["axum", "firecrawl"]