Curl downloader and tar extractor

src/archive.rs

@@ -0,0 +1,36 @@
+use std::{path::Path, process::Command};
+
+pub fn extract<P: AsRef<Path>>(path: &P, name: &str, archive_type: &str) -> Result<(), String> {
+    let mut command = match archive_type.to_lowercase().as_ref() {
+        "tar" | "tar.gz" | "targz" | "tgz" => {
+            let mut cmd = Command::new("tar");
+            cmd.arg("-xvf");
+            cmd.arg(name);
+            cmd
+        }
+        invalid => return Err(format!("Invalid archive type '{invalid}'")),
+    };
+
+    command.current_dir(path);
+
+    command.stdout(std::process::Stdio::piped());
+    command.stderr(std::process::Stdio::piped());
+
+    let spawn = command.spawn().map_err(|e| e.to_string())?;
+    let (result, stdout, stderr) = crate::cli::child_logger(spawn);
+
+    if result.is_err() {
+        return Err("Failed to run tar command".to_string());
+    }
+    let result = result.unwrap();
+
+    if !result.success() {
+        return Err(format!(
+            "Failed to extract archive: \n{}\n{}",
+            stdout.join("\n"),
+            stderr.join("\n")
+        ));
+    }
+
+    Ok(())
+}

src/downloaders.rs

@@ -1,14 +1,19 @@
+use crate::{archive, log};
 use pyo3::prelude::*;
-use std::path::Path;
-use std::process::Command;
+use std::{fs, path::Path, process::Command};
 
 pub trait DownloaderImpl: Sized {
     /// Convert from a Python `Downloader` instance to a Rust [`Downloader`] instance.
-    /// If this is not possible (due to an invalid value, for example), [`None`] is returned.
+    /// If this is not possible (due to an invalid value, for example), [`Err`] is returned
+    /// containing an error message as a [`String`]
     ///
     /// # Note
     /// `object` must be a valid `Downloader` instance in Python.
-    fn from_py(object: &Bound<PyAny>) -> Option<Self>;
+    ///
+    /// # Errors
+    ///
+    /// Errors if the object cannot be converted correctly to a Rust type
+    fn from_py(object: &Bound<PyAny>) -> Result<Self, String>;
 
     /// Download the source code into the specified `path`.
     ///
@@ -30,6 +35,13 @@ pub struct GitClone {
     submodules: bool,
 }
 
+#[derive(Debug, Clone)]
+pub struct Curl {
+    url: String,
+    sha256: Option<String>,
+    archive: Option<String>,
+}
+
 impl GitClone {
     #[must_use]
     pub fn new(url: &str) -> Self {
@@ -43,18 +55,35 @@ impl GitClone {
 }
 
 impl DownloaderImpl for GitClone {
-    fn from_py(object: &Bound<PyAny>) -> Option<Self> {
+    fn from_py(object: &Bound<PyAny>) -> Result<Self, String> {
         let url: String = object
             .getattr("url")
-            .expect("Failed to find attribute .url")
+            .map_err(|_| "Object does not contain an attribute named 'url'")?
             .extract()
-            .expect("Failed to extract url");
+            .map_err(|_| "Failed to convert 'url' to Rust String")?;
 
-        let branch: Option<String> = object.getattr("branch").ok()?.extract().ok();
-        let commit: Option<String> = object.getattr("commit").ok()?.extract().ok();
-        let submodules: bool = object.getattr("submodules").ok()?.extract().ok()?;
+        let branch: Option<String> = match object.getattr("branch") {
+            Ok(x) => x
+                .extract()
+                .map_err(|_| "Failed to convert 'branch' to Rust String")?,
+            Err(_) => None,
+        };
 
-        Some(Self {
+        let commit: Option<String> = match object.getattr("commit") {
+            Ok(x) => x
+                .extract()
+                .map_err(|_| "Failed to convert 'commit' to Rust String")?,
+            Err(_) => None,
+        };
+
+        let submodules: bool = match object.getattr("submodules") {
+            Ok(x) => x
+                .extract()
+                .map_err(|_| "Failed to convert 'submodules' to Rust bool")?,
+            Err(_) => false,
+        };
+
+        Ok(Self {
             url,
             branch,
             commit,
@@ -80,10 +109,12 @@ impl DownloaderImpl for GitClone {
             }
 
             if self.submodules {
-                command.arg("--recurse");
+                command.arg("--recursive");
             }
 
+            // Clone into `path`
             command.arg(path.as_ref());
+
             command.stdout(std::process::Stdio::piped());
             command.stderr(std::process::Stdio::piped());
 
@@ -134,44 +165,87 @@ impl DownloaderImpl for GitClone {
             ));
         }
 
-        // if let Some(commit) = &self.commit {
-        //     let mut command = Command::new("git");
-        //     command.current_dir(path);
-        //     command.arg("checkout");
-        //     command.arg(commit);
-        //     command.stdout(std::process::Stdio::piped());
-        //     command.stderr(std::process::Stdio::piped());
-        //
-        //     let spawn = command.spawn().map_err(|e| e.to_string())?;
-        //     let (result, stdout, stderr) = crate::cli::child_logger(spawn);
-        //
-        //     if result.is_err() || !result.unwrap().success() {
-        //         return Err(format!(
-        //             "Failed to checkout commit {commit:?}: \n{}\n{}",
-        //             stdout.join("\n"),
-        //             stderr.join("\n")
-        //         ));
-        //     }
-        // } else {
-        //     // No commit specified, so pull latest changes
-        //
-        //     let mut command = Command::new("git");
-        //     command.current_dir(path);
-        //     command.arg("pull");
-        //     command.stdout(std::process::Stdio::piped());
-        //     command.stderr(std::process::Stdio::piped());
-        //
-        //     let spawn = command.spawn().map_err(|e| e.to_string())?;
-        //     let (result, stdout, stderr) = crate::cli::child_logger(spawn);
-        //
-        //     if result.is_err() || !result.unwrap().success() {
-        //         return Err(format!(
-        //             "Failed to pull: \n{}\n{}",
-        //             stdout.join("\n"),
-        //             stderr.join("\n")
-        //         ));
-        //     }
-        // }
+        Ok(())
+    }
+}
+
+impl Curl {
+    #[must_use]
+    pub fn new(url: &str) -> Self {
+        Self {
+            url: url.to_string(),
+            sha256: None,
+            archive: None,
+        }
+    }
+}
+
+impl DownloaderImpl for Curl {
+    fn from_py(object: &Bound<PyAny>) -> Result<Self, String> {
+        let url: String = object
+            .getattr("url")
+            .map_err(|_| "Object does not contain an attribute named 'url'")?
+            .extract()
+            .map_err(|_| "Could not convert attribute 'url' to Rust String")?;
+
+        let sha256: Option<String> = match object.getattr("sha256") {
+            Ok(x) => x
+                .extract()
+                .map_err(|_| "Could not convert attribute 'sha256' to Rust String")?,
+            Err(_) => None,
+        };
+
+        let archive: Option<String> = match object.getattr("archive") {
+            Ok(x) => x
+                .extract()
+                .map_err(|_| "Could not convert attribute 'archive' to Rust String")?,
+            Err(_) => None,
+        };
+
+        Ok(Self {
+            url,
+            sha256,
+            archive,
+        })
+    }
+
+    fn download<P: AsRef<Path>>(&self, path: &P) -> Result<(), String> {
+        // Todo: Check if the hashes match. If they do, there is no need to re-download
+
+        // Ensure the directory exists
+        fs::create_dir_all(&path).map_err(|e| e.to_string())?;
+
+        const FILE_NAME: &str = "curl_download_result";
+
+        let mut command = Command::new("curl");
+        command.current_dir(path.as_ref());
+        command.arg("-o");
+        command.arg(FILE_NAME);
+        command.arg(&self.url);
+
+        command.stdout(std::process::Stdio::piped());
+        command.stderr(std::process::Stdio::piped());
+
+        let spawn = command.spawn().map_err(|e| e.to_string())?;
+        let (result, stdout, stderr) = crate::cli::child_logger(spawn);
+
+        if result.is_err() {
+            return Err("Failed to run curl command".to_string());
+        }
+        let result = result.unwrap();
+
+        if !result.success() {
+            return Err(format!(
+                "Failed to download from URL: \n{}\n{}",
+                stdout.join("\n"),
+                stderr.join("\n")
+            ));
+        }
+
+        // Extract the archive if necessary
+        if let Some(archive) = &self.archive {
+            archive::extract(path, FILE_NAME, &archive)?;
+        }
 
         Ok(())
     }
@@ -180,23 +254,24 @@ impl DownloaderImpl for GitClone {
 #[derive(Debug)]
 pub enum Downloader {
     GitClone(GitClone),
-    Curl,
+    Curl(Curl),
 }
 
 impl DownloaderImpl for Downloader {
-    fn from_py(object: &Bound<PyAny>) -> Option<Self> {
+    fn from_py(object: &Bound<PyAny>) -> Result<Self, String> {
         let name = object.get_type().name().unwrap().to_string();
 
         match name.as_str() {
-            "GitClone" => Some(Self::GitClone(GitClone::from_py(object)?)),
-            _ => None,
+            "GitClone" => Ok(Self::GitClone(GitClone::from_py(object)?)),
+            "Curl" => Ok(Self::Curl(Curl::from_py(object)?)),
+            _ => Err("Invalid downloader type".to_string()),
         }
     }
 
     fn download<P: AsRef<Path>>(&self, path: &P) -> Result<(), String> {
         match self {
             Self::GitClone(clone) => clone.download(path),
-            Self::Curl => Err("Not implemented yet".to_string()),
+            Self::Curl(curl) => curl.download(path),
         }
     }
 }

src/lib.rs

@@ -7,6 +7,7 @@ pub mod builders {
     pub mod cmake;
 }
 
+pub mod archive;
 pub mod callbacks;
 pub mod cli;
 pub mod config;

src/module.rs

@@ -94,8 +94,7 @@ impl Module {
                 &extract_object(object, "download")?
                     .call0()
                     .map_err(|err| format!("Failed to call `download` in module class: {err}"))?,
-            )
-            .ok_or_else(|| "Could not extract downloader from module class".to_string())?;
+            )?;
 
             // todo: build requirements
 

src/sccmod/downloaders.py

@@ -1,12 +1,13 @@
 class GitClone:
-    def __init__(self, url, branch=None, commit=None, tag=None, submodules=False):
+    def __init__(self, url, branch=None, commit=None, tag=None, submodules=True):
         self.url = url
         self.branch = branch
         self.commit = commit
         self.submodules = submodules
 
 
 class Curl:
-    def __init__(self, url, sha256=None):
+    def __init__(self, url, archive=None, sha256=None):
         self.url = url
+        self.archive = archive
         self.sha256 = sha256