Devpost CSV Parsing

Cargo.toml

@@ -17,6 +17,7 @@ dotenv = "0.15.0"
 rand = "0.8.5"
 async-trait = "0.1.59"
 futures = "0.3"
+csv = "1.2.1"
 
 [dependencies.rocket]
 version = "0.5.0-rc.2"

client/dev.Dockerfile

@@ -5,6 +5,4 @@ ENV REACT_APP_JURY_NAME=$REACT_APP_JURY_NAME
 ENV REACT_APP_JURY_URL=$REACT_APP_JURY_URL
 ENV REACT_APP_HUB=$REACT_APP_HUB
 
-RUN yarn install --frozen-lockfile
-
-CMD [ "yarn", "run", "start" ]
+CMD [ "yarn", "run", "docker" ]

client/package.json

@@ -21,6 +21,7 @@
         "tailwindcss": "^3.2.6"
     },
     "scripts": {
+        "docker": "yarn install && yarn start",
         "start": "react-scripts start",
         "build": "react-scripts build",
         "test": "react-scripts test",

src/api/admin.rs

@@ -1,10 +1,13 @@
 use mongodb::Database;
+use rocket::data::{Data, ToByteUnit};
 use rocket::response::stream::{Event, EventStream};
 use rocket::{http::Status, serde::json::Json, State};
 use std::env;
 use std::sync::Arc;
 use tokio::sync::mpsc::{channel, Sender};
 
+use crate::db::admin::insert_projects;
+use crate::util::parse_csv::devpost_integration;
 use crate::{
     db::admin::aggregate_stats,
     util::types::{AdminLogin, Stats},
@@ -29,21 +32,13 @@ pub async fn login(body: Json<AdminLogin<'_>>) -> (Status, String) {
 }
 
 #[rocket::get("/admin/stats")]
-pub async fn get_stats(_password: AdminPassword, db: &State<Arc<Database>>) -> (Status, Json<Stats>) {
+pub async fn get_stats(
+    _password: AdminPassword,
+    db: &State<Arc<Database>>,
+) -> (Status, Json<Stats>) {
     match aggregate_stats(&db).await {
         Ok(stats) => (Status::Ok, Json(stats)),
-        Err(_) => (
-            Status::InternalServerError,
-            Json(Stats {
-                projects: 0,
-                seen: 0,
-                votes: 0,
-                time: 0,
-                avg_mu: 0.0,
-                avg_sigma: 0.0,
-                judges: 0,
-            }),
-        ),
+        Err(_) => (Status::InternalServerError, Json(Stats::default())),
     }
 }
 
@@ -79,3 +74,47 @@ pub async fn req_sync(
         }),
     )
 }
+
+const CSV_HEADER: &str = "Content-Type: text/csv\r\n\r\n";
+const CSV_FOOTER: &str = "\r\n----------------------------";
+
+#[rocket::post("/admin/csv", data = "<csv>")]
+pub async fn add_projects_csv(csv: Data<'_>, db: &State<Arc<Database>>) -> (Status, String) {
+    // TODO: Add admin token check
+
+    // Extract all data and store into a string
+    // TODO: No unwrap
+    let str = csv.open(16.mebibytes()).into_string().await.unwrap();
+
+    // Find the start and end of the CSV data
+    // and cut out the CSV data from the raw string
+    let start = str.find(CSV_HEADER).unwrap() + CSV_HEADER.len();
+    let end = str[start..].find(CSV_FOOTER).unwrap();
+    let cut_str = &str[start..(start + end)];
+
+    // Parse the CSV data
+    let parsed_csv = match devpost_integration(cut_str, db).await {
+        Ok(p) => p,
+        Err(e) => {
+            eprintln!("Unable to parse CSV: {e}",);
+            return (
+                Status::InternalServerError,
+                format!("Unable to parse CSV: {e}").to_string(),
+            );
+        }
+    };
+
+    // Save the parsed CSV data to the database
+    match insert_projects(&db, parsed_csv).await {
+        Ok(_) => (),
+        Err(e) => {
+            eprintln!("Unable to insert projects into database: {e}",);
+            return (
+                Status::InternalServerError,
+                format!("Unable to insert projects into database: {e}").to_string(),
+            );
+        }
+    };
+
+    (Status::Ok, "".to_string())
+}

src/api/judge.rs

@@ -7,7 +7,7 @@ use rocket::http::Status;
 use rocket::serde::json::Json;
 use rocket::State;
 
-use crate::db::judge::{find_judge_by_code, insert_judge, update_judge_token, read_welcome};
+use crate::db::judge::{find_judge_by_code, insert_judge, read_welcome, update_judge_token};
 
 use super::{
     request_types::{Login, NewJudge},
@@ -57,6 +57,9 @@ pub async fn new_judge(
 pub async fn judge_read_welcome(db: &State<Arc<Database>>, token: Token) -> (Status, String) {
     match read_welcome(db, &token.0).await {
         Ok(_) => (Status::Accepted, "{}".to_string()),
-        Err(_) => (Status::InternalServerError, "Internal Server Error".to_string()),
+        Err(_) => (
+            Status::InternalServerError,
+            "Internal Server Error".to_string(),
+        ),
     }
-}
+}

src/api/mod.rs

@@ -1,7 +1,7 @@
-pub mod client;
-pub mod judge;
 pub mod admin;
+pub mod client;
 pub mod guards;
+pub mod judge;
 pub mod request_types;
-pub mod util;
 pub mod tasks;
+pub mod util;

src/api/request_types.rs

@@ -1,6 +1,5 @@
 use serde::Deserialize;
 
-
 #[derive(Deserialize)]
 #[serde()]
 pub struct Login<'r> {
@@ -13,4 +12,4 @@ pub struct NewJudge<'r> {
     pub name: &'r str,
     pub email: &'r str,
     pub notes: &'r str,
-}
+}

src/api/tasks.rs

@@ -111,7 +111,10 @@ async fn mongo_listen(
 async fn debounce_task(sender_list: Arc<Mutex<Vec<Sender<String>>>>, diff: i64) {
     // Sleep for 11 seconds to catch all events that happen
     // Extra second is allocated for overlap with next debounce cycle
-    sleep(Duration::from_millis((11000 - diff).try_into().unwrap_or_else(|_| 0))).await;
+    sleep(Duration::from_millis(
+        (11000 - diff).try_into().unwrap_or_else(|_| 0),
+    ))
+    .await;
     update_senders(sender_list).await;
 }
 

src/api/util.rs

@@ -19,4 +19,4 @@ pub enum AdminPasswordError {
 
 pub fn get_cookie<'r>(request: &'r rocket::Request<'_>, name: &str) -> Option<&'r Cookie<'static>> {
     request.cookies().get(name)
-}
+}

src/db/admin.rs

@@ -96,3 +96,9 @@ pub async fn aggregate_stats(db: &Database) -> Result<Stats, Error> {
         judges,
     })
 }
+
+pub async fn insert_projects(db: &Database, projects: Vec<Project>) -> Result<(), Error> {
+    let collection = db.collection::<Project>("projects");
+    collection.insert_many(projects, None).await?;
+    Ok(())
+}

src/db/mod.rs

@@ -1,4 +1,5 @@
+pub mod admin;
 pub mod init;
-pub mod models;
 pub mod judge;
-pub mod admin;
+pub mod models;
+pub mod options;

src/db/models.rs

@@ -1,14 +1,22 @@
+use chrono::{DateTime, Utc};
 use mongodb::bson::oid::ObjectId;
 use serde::{Deserialize, Serialize};
-use chrono::{Utc, DateTime};
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Options {
+    pub next_table_num: u32,
+}
 
 #[derive(Deserialize, Serialize, Debug)]
 pub struct Project {
     #[serde(rename = "_id", skip_serializing_if = "Option::is_none")]
     pub id: Option<ObjectId>,
     pub name: String,
-    pub location: String,
+    pub location: u64,
     pub description: String,
+    pub try_link: Option<String>,
+    pub video_link: Option<String>,
+    pub challenge_list: Vec<String>,
     pub seen: u64,
     pub votes: u64,
     pub mu: f64,

src/db/options.rs

@@ -0,0 +1,56 @@
+use std::error::Error;
+
+use bson::doc;
+use mongodb::{Collection, Database};
+
+use crate::db::models::Options;
+
+pub async fn get_options(db: &Database) -> Result<Options, Box<dyn Error>> {
+    let collection: Collection<Options> = db.collection("options");
+    let content = collection.find_one(None, None).await?;
+    match content {
+        Some(x) => Ok(x),
+        None => {
+            let options = Options::default();
+            collection.insert_one(options.clone(), None).await?;
+            Ok(options)
+        }
+    }
+}
+
+impl Options {
+    pub fn default() -> Options {
+        Options { next_table_num: 1u32 }
+    }
+
+    /// Increment next table num and return the value
+    pub fn get_next_table_num(&mut self) -> u32 {
+        self.next_table_num += 1;
+        self.next_table_num - 1
+    }
+
+    /// Saves the options to the database
+    pub async fn save(&self, db: &Database) -> Result<(), Box<dyn Error>> {
+        let collection: Collection<Options> = db.collection("options");
+        collection
+            .update_one(
+                doc! {},
+                doc! {
+                    "$set": {
+                        "next_table_num": self.next_table_num,
+                    }
+                },
+                None,
+            )
+            .await?;
+        Ok(())
+    }
+}
+
+impl Clone for Options {
+    fn clone(&self) -> Self {
+        Options {
+            next_table_num: self.next_table_num,
+        }
+    }
+}

src/main.rs

@@ -49,6 +49,7 @@ async fn rocket() -> _ {
                 admin::login,
                 admin::get_stats,
                 admin::req_sync,
+                admin::add_projects_csv,
             ],
         )
         .mount("/", routes![client::home, client::all_options])

src/util/check_env.rs

@@ -3,4 +3,4 @@ use std::env;
 pub fn check() {
     let _ = env::var("MONGODB_URI").expect("MONGODB_URI not defined");
     let _ = env::var("JURY_ADMIN_PASSWORD").expect("JURY_ADMIN_PASSWORD not defined");
-}
+}

src/util/mod.rs

@@ -1,3 +1,4 @@
-pub mod crowd_bt;
 pub mod check_env;
+pub mod crowd_bt;
+pub mod parse_csv;
 pub mod types;