Minor changes to error handling when fetching posts & media

- handle error when attempting to get posts for sub if data can't be found:
Gert would panic without indicating which sub couldn't be fetched. If trying to get
multiple subs, gert will now print an error and move onto the next
- handle error when post has no media data, common with crossposts mainly.
These would also panic when trying to unwrap media
- Prevent gert from downloading imgur media when that media has been removed
Especially on older subs a lot of media has been removed so gert downloads and saves
the removed image https://i.imgur.com/removed.png, this just skips the saving of those files.

src/download.rs

@@ -239,7 +239,19 @@ impl Downloader {
         let maybe_response = self.session.get(url).send().await;
         if let Ok(response) = maybe_response {
             // debug!("URL Response: {:#?}", response);
+
+            let url = response.url().to_owned();
+            let host_and_path = match url.host_str() {
+                Some(domain) => format!("{}{}", domain, url.path()),
+                None => return Err(GertError::UrlError(url::ParseError::EmptyHost)),
+            };
+
+            if host_and_path.contains("i.imgur.com/removed") {
+                return Err(GertError::ImgurRemovedError);
+            }
+
             let maybe_data = response.bytes().await;
+
             if let Ok(data) = maybe_data {
                 debug!("Bytes length of the data: {:#?}", data.len());
                 let maybe_output = File::create(file_name);
@@ -340,6 +352,11 @@ impl Downloader {
     async fn download_reddit_video(&self, post: &Post) -> Result<()> {
         let post_url = post.data.url.as_ref().unwrap();
         let extension = post_url.split('.').last().unwrap();
+
+        if post.data.media.is_none() {
+            bail!("No media data found for this post {}", post_url);
+        }
+
         let dash_url = &post.data.media.as_ref().unwrap().reddit_video.as_ref().unwrap().dash_url;
 
         let url = match extension {
@@ -552,6 +569,7 @@ impl Downloader {
             || check_path_present(&file_name.replace(".zip", ".jpg"))
         {
             let msg = format!("Media from url {} already downloaded. Skipping...", task.url);
+            error!("{}", msg);
             self.skip(&msg).await;
             return None;
         }
@@ -576,8 +594,15 @@ impl Downloader {
                 None
             }
             Err(e) => {
-                self.fail(anyhow!("Error while downloading media from url {}: {}", task.url, e)).await;
-                None
+                if let GertError::ImgurRemovedError = e {
+                    let msg = format!("Media from url {} has been removed from imgur. Skipping...", task.url);
+                    error!("{}", msg);
+                    self.skip(&msg).await;
+                    None
+                } else {
+                    self.fail(anyhow!("Error while downloading media from url {}: {}", task.url, e)).await;
+                    None
+                }
             }
         }
     }

src/errors.rs

@@ -37,4 +37,6 @@ pub enum GertError {
     FfmpegError(String),
     #[error("Error unzipping file")]
     ZipError(#[from] zip::result::ZipError),
+    #[error("Media has been removed from imgur")]
+    ImgurRemovedError,
 }

src/subreddit.rs

@@ -1,6 +1,6 @@
 use crate::errors::GertError;
 use crate::structures::{Listing, Post};
-use log::debug;
+use log::{debug, error};
 use reqwest::Client;
 use std::fmt::Write;
 
@@ -44,8 +44,8 @@ impl Subreddit<'_> {
             .await
             .expect("Bad response")
             .json::<Listing>()
-            .await
-            .expect("Failed to parse JSON"))
+            .await?
+        )
         // Ok(self.client.get(url).send().await.expect("Bad response").json::<Listing>().await.expect("Failed to parse JSON"))
     }
 
@@ -71,13 +71,26 @@ impl Subreddit<'_> {
         while remaining > 0 {
             debug!("Fetching page {} of {} from r/{} [{}]", page, limit / 100, self.name, feed);
             let limit = if remaining > 100 { 100 } else { remaining };
-            let listing = self.get_feed(feed, limit, period, after).await?;
+            let listing_result = self.get_feed(feed, limit, period, after).await;
 
-            posts.extend(listing.data.children.into_iter().collect::<Vec<Post>>());
-            let last_post = posts.last().unwrap();
-            after = Some(&last_post.data.name);
-            remaining -= limit;
-            page += 1;
+            match listing_result {
+                Ok(listing) => {
+                    if !listing.data.children.is_empty() {
+                        posts.extend(listing.data.children.into_iter().collect::<Vec<Post>>());
+                        let last_post = posts.last().unwrap();
+                        after = Some(&last_post.data.name);
+                        remaining -= limit;
+                        page += 1;
+                    } else {
+                        error!("Failed to fetch posts from r/{}", self.name);
+                        remaining = 0;
+                    }
+                }
+                Err(_error) => {
+                    error!("Failed to fetch posts from r/{}", self.name);
+                    remaining = 0;
+                }
+            }
         }
         Ok(posts)
     }