Skip to content

Commit

Permalink
[deviantart] add 'avatar' and 'background' extractors (#4995)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mikf
mikf committed Jan 2, 2024
1 parent ee65f3d commit 4f36714
Show file tree
Hide file tree
Showing 4 changed files with 154 additions and 12 deletions.
8 changes: 7 additions & 1 deletion docs/configuration.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1401,7 +1401,13 @@ Description
when processing a user profile.

Possible values are
``"gallery"``, ``"scraps"``, ``"journal"``, ``"favorite"``, ``"status"``.
``"avatar"``,
``"background"``,
``"gallery"``,
``"scraps"``,
``"journal"``,
``"favorite"``,
``"status"``.

It is possible to use ``"all"`` instead of listing all values separately.

Expand Down
2 changes: 1 addition & 1 deletion docs/supportedsites.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ Consider all listed sites to potentially be NSFW.
<tr>
<td>DeviantArt</td>
<td>https://www.deviantart.com/</td>
<td>Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
<td>Avatars, Backgrounds, Collections, Deviations, Favorites, Folders, Followed Users, Galleries, Gallery Searches, Journals, Popular Images, Scraps, Search Results, Sta.sh, Status Updates, Tag Searches, User Profiles, Watches</td>
<td><a href="https://github.com/mikf/gallery-dl#oauth">OAuth</a></td>
</tr>
<tr>
Expand Down
56 changes: 49 additions & 7 deletions gallery_dl/extractor/deviantart.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class DeviantartExtractor(Extractor):

def __init__(self, match):
Extractor.__init__(self, match)
self.user = match.group(1) or match.group(2)
self.user = (match.group(1) or match.group(2)).lower()
self.offset = 0

def _init(self):
Expand Down Expand Up @@ -104,7 +104,6 @@ def items(self):
raise exception.StopExtraction()
else:
self.subcategory = "group-" + self.subcategory
self.user = self.user.lower()
self.group = True

for deviation in self.deviations():
Expand Down Expand Up @@ -513,11 +512,13 @@ def initialize(self):
def items(self):
base = "{}/{}/".format(self.root, self.user)
return self._dispatch_extractors((
(DeviantartGalleryExtractor , base + "gallery"),
(DeviantartScrapsExtractor , base + "gallery/scraps"),
(DeviantartJournalExtractor , base + "posts"),
(DeviantartStatusExtractor , base + "posts/statuses"),
(DeviantartFavoriteExtractor, base + "favourites"),
(DeviantartAvatarExtractor , base + "avatar"),
(DeviantartBackgroundExtractor, base + "banner"),
(DeviantartGalleryExtractor , base + "gallery"),
(DeviantartScrapsExtractor , base + "gallery/scraps"),
(DeviantartJournalExtractor , base + "posts"),
(DeviantartStatusExtractor , base + "posts/statuses"),
(DeviantartFavoriteExtractor , base + "favourites"),
), ("gallery",))


Expand All @@ -538,6 +539,47 @@ def deviations(self):
return self._folder_urls(folders, "gallery", DeviantartFolderExtractor)


class DeviantartAvatarExtractor(DeviantartExtractor):
"""Extractor for an artist's avatar"""
subcategory = "avatar"
archive_fmt = "a_{_username}_{index}"
pattern = BASE_PATTERN + r"/avatar"
example = "https://www.deviantart.com/USER/avatar/"

def deviations(self):
profile = self.api.user_profile(self.user.lower())
if profile:
url = profile["user"]["usericon"]
return ({
"author" : profile["user"],
"category" : "avatar",
"index" : text.parse_int(url.rpartition("?")[2]),
"is_deleted" : False,
"is_downloadable": False,
"published_time" : 0,
"title" : "avatar",
"content" : {
"src": url.replace("/avatars/", "/avatars-big/", 1),
},
},)
return ()


class DeviantartBackgroundExtractor(DeviantartExtractor):
"""Extractor for an artist's banner"""
subcategory = "background"
archive_fmt = "b_{index}"
pattern = BASE_PATTERN + r"/ba(?:nner|ckground)"
example = "https://www.deviantart.com/USER/banner/"

def deviations(self):
try:
return (self.api.user_profile(self.user.lower())
["cover_deviation"]["cover_deviation"],)
except Exception:
return ()


class DeviantartFolderExtractor(DeviantartExtractor):
"""Extractor for deviations inside an artist's gallery folder"""
subcategory = "folder"
Expand Down
100 changes: 97 additions & 3 deletions test/results/deviantart.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,23 @@
"#url" : "https://www.deviantart.com/shimoda7",
"#category": ("", "deviantart", "user"),
"#class" : deviantart.DeviantartUserExtractor,
"#pattern" : "/shimoda7/gallery$",
"#urls" : "https://www.deviantart.com/shimoda7/gallery",
},

{
"#url" : "https://www.deviantart.com/shimoda7",
"#category": ("", "deviantart", "user"),
"#class" : deviantart.DeviantartUserExtractor,
"#options" : {"include": "all"},
"#pattern" : "/shimoda7/(gallery(/scraps)?|posts(/statuses)?|favourites)$",
"#count" : 5,
"#urls" : (
"https://www.deviantart.com/shimoda7/avatar",
"https://www.deviantart.com/shimoda7/banner",
"https://www.deviantart.com/shimoda7/gallery",
"https://www.deviantart.com/shimoda7/gallery/scraps",
"https://www.deviantart.com/shimoda7/posts",
"https://www.deviantart.com/shimoda7/posts/statuses",
"https://www.deviantart.com/shimoda7/favourites",
),
},

{
Expand Down Expand Up @@ -195,6 +202,93 @@
"#class" : deviantart.DeviantartGalleryExtractor,
},

{
"#url" : "https://deviantart.com/shimoda7/avatar",
"#category": ("", "deviantart", "avatar"),
"#class" : deviantart.DeviantartAvatarExtractor,
"#urls" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4",
"#sha1_content": "abf2cc79b842315f2e54bfdd93bf794a0f612b6f",

"author" : {
"type" : "premium",
"usericon": "https://a.deviantart.net/avatars/s/h/shimoda7.jpg?4",
"userid" : "9AE51FC7-0278-806C-3FFF-F4961ABF9E2B",
"username": "shimoda7",
},
"content" : {
"src": "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"
},
"da_category" : "avatar",
"date" : "dt:1970-01-01 00:00:00",
"extension" : "jpg",
"filename" : "avatar_by_shimoda7-d4",
"index" : 4,
"index_base36" : "4",
"is_deleted" : False,
"is_downloadable": False,
"is_original" : True,
"published_time" : 0,
"target" : {
"extension": "jpg",
"filename" : "avatar_by_shimoda7-d4",
"src" : "https://a.deviantart.net/avatars-big/s/h/shimoda7.jpg?4"
},
"title" : "avatar",
"username" : "shimoda7",
},

{
"#url" : "https://deviantart.com/gdldev/banner",
"#category": ("", "deviantart", "background"),
"#class" : deviantart.DeviantartBackgroundExtractor,
"#pattern" : r"https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$",
"#sha1_content": "980eaa76ce515f1b6bef60dfadf26a5bbe9c583f",

"allows_comments" : True,
"author" : {
"type" : "regular",
"usericon": "https://a.deviantart.net/avatars/g/d/gdldev.jpg?2",
"userid" : "1A12BA26-33C2-AA0A-7678-0B6DFBA7AC8E",
"username": "gdldev"
},
"category_path" : "",
"content" : {
"filename" : "banner_by_gdldev_dgntyqc.png",
"filesize" : 84510,
"height" : 4000,
"src" : r"re:https://wixmp-\w+\.wixmp\.com/f/b042e0ae-a7ff-420b-a41a-b35503427360/dgntyqc-3deebb65-04b4-4085-992a-aa0c0e7e225d\.png\?token=ey[\w.-]+$",
"transparency": False,
"width" : 6400
},
"da_category" : "Uncategorized",
"date" : "dt:2024-01-02 21:16:06",
"deviationid" : "8C8D6B28-766A-DE21-7F7D-CE055C3BD50A",
"download_filesize": 84510,
"extension" : "png",
"filename" : "banner_by_gdldev-dgntyqc",
"index" : 1007488020,
"index_base36" : "gntyqc",
"is_blocked" : False,
"is_deleted" : False,
"is_downloadable" : True,
"is_favourited" : False,
"is_mature" : False,
"is_original" : True,
"is_published" : False,
"preview" : dict,
"printid" : None,
"published_time" : 1704230166,
"stats" : {
"comments" : 0,
"favourites": 0,
},
"target" : dict,
"thumbs" : list,
"title" : "Banner",
"url" : "https://sta.sh/0198jippkeys",
"username" : "gdldev",
},

{
"#url" : "https://www.deviantart.com/shimoda7/gallery/722019/Miscellaneous",
"#comment" : "user",
Expand Down

0 comments on commit 4f36714

Please sign in to comment.