Pixiv User Background Banner #2495
Comments
|
Also it's possible to replace (removing of BTW, is there a support of banners for naming (Is it possible to add a special rule for it? Since it will not have |
|
yeah something like https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst#extractorpixivuseravatar would be good |
|
API:
Original (requires |
|
its 403 forbidden |
It requires the existence of Referer HTTP request's header with Pixiv's origin value. |
|
how to call it |
|
Edit any visible image HTML element Nevermind. |
|
i dont get it |
|
Okay, a UserScript (updated on 2022.12.21): < Code (click to expand) >// ==UserScript==
// @name Pixiv BG Download Script
// @namespace Pixiv
// @version 0.0.6-2022.12.21
// @match https://www.pixiv.net/en/users/*
// @match https://www.pixiv.net/users/*
// @description Pixiv BG Download Button
// @grant GM_registerMenuCommand
// @grant GM_xmlhttpRequest
// @connect i.pximg.net
// @noframes
// @author [Alt'tiRi]
// @supportURL https://github.com/mikf/gallery-dl/issues/2495#issuecomment-1102505269
// ==/UserScript==
// ------------------------------------------------------------------------------------
// Init
// ------------------------------------------------------------------------------------
const globalFetch = ujs_getGlobalFetch();
const fetch = GM_fetch;
if (globalThis.GM_registerMenuCommand /* undefined in Firefox with VM */ || typeof GM_registerMenuCommand === "function") {
GM_registerMenuCommand("Download BG", downloadBg);
}
// ------------------------------------------------------------------------------------
// Main code
// ------------------------------------------------------------------------------------
function downloadBg() {
const userId = parseUserId();
void downloadBgWithApi(userId);
}
function parseUserId(url = location.href) {
const _url = new URL(url);
const id = _url.pathname.match(/(?<=users\/)\d+/)[0];
return id;
}
async function downloadBgWithApi(userId) {
const titleText = document.title;
try {
document.title = "💤" + titleText;
const resp = await globalFetch("https://www.pixiv.net/ajax/user/" + userId);
const json = await resp.json();
if (!json?.body?.background?.url) {
document.title = "⬜" + titleText;
console.log("[ujs] no bg");
await sleep(1000);
return;
}
const {name: userName, background} = json.body;
const url = background.url.replace("/c/1920x960_80_a2_g5", "");
document.title = "⏳" + titleText;
const {blob, lastModifiedDate, filename} = await fetchResource(url, {headers: {"referer": location.href}});
const filenamePrefix = userId + "_";
const _filename = filename.startsWith(filenamePrefix) ? filename.slice(filenamePrefix.length) : filename;
const name = `[pixiv][bg] ${userId}—${userName}—${lastModifiedDate}—${_filename}`;
download(blob, name, url);
document.title = "✅" + titleText;
await sleep(5000);
} catch (e) {
console.error(e);
document.title = "❌" + titleText;
await sleep(5000);
} finally {
document.title = titleText;
}
}
// ------------------------------------------------------------------------------------
// GM Util
// ------------------------------------------------------------------------------------
function ujs_getGlobalFetch({verbose, strictTrackingProtectionFix} = {}) {
const useFirefoxStrictTrackingProtectionFix = strictTrackingProtectionFix === undefined ? true : strictTrackingProtectionFix; // Let's use by default
const useFirefoxFix = useFirefoxStrictTrackingProtectionFix && typeof wrappedJSObject === "object" && typeof wrappedJSObject.fetch === "function";
// --- [VM/GM + Firefox ~90+ + Enabled "Strict Tracking Protection"] fix --- //
function fixedFirefoxFetch(resource, init = {}) {
verbose && console.log("wrappedJSObject.fetch", resource, init);
if (init.headers instanceof Headers) {
// Since `Headers` are not allowed for structured cloning.
init.headers = Object.fromEntries(init.headers.entries());
}
return wrappedJSObject.fetch(cloneInto(resource, document), cloneInto(init, document));
}
return useFirefoxFix ? fixedFirefoxFetch : globalThis.fetch;
}
// The simplified `fetch` — wrapper for `GM_xmlhttpRequest`
/* Using:
// @grant GM_xmlhttpRequest
const response = await fetch(url);
const {status, statusText} = response;
const lastModified = response.headers.get("last-modified");
const blob = await response.blob();
*/
async function GM_fetch(url, init = {}) {
const defaultInit = {method: "get"};
const {headers, method} = {...defaultInit, ...init};
return new Promise((resolve, _reject) => {
const blobPromise = new Promise((resolve, reject) => {
GM_xmlhttpRequest({
url,
method,
headers,
responseType: "blob",
onload: (response) => resolve(response.response),
onerror: reject,
onreadystatechange: onHeadersReceived
});
});
blobPromise.catch(_reject);
function onHeadersReceived(response) {
const {
readyState, responseHeaders, status, statusText
} = response;
if (readyState === 2) { // HEADERS_RECEIVED
const headers = parseHeaders(responseHeaders);
resolve({
headers,
status,
statusText,
arrayBuffer: () => blobPromise.then(blob => blob.arrayBuffer()),
blob: () => blobPromise,
json: () => blobPromise.then(blob => blob.text()).then(text => JSON.parse(text)),
text: () => blobPromise.then(blob => blob.text()),
});
}
}
});
}
function parseHeaders(headersString) {
class Headers {
get(key) {
return this[key.toLowerCase()];
}
}
const headers = new Headers();
for (const line of headersString.trim().split("\n")) {
const [key, ...valueParts] = line.split(":"); // last-modified: Fri, 21 May 2021 14:46:56 GMT
headers[key.trim().toLowerCase()] = valueParts.join(":").trim();
}
return headers;
}
// ------------------------------------------------------------------------------------
// Util
// ------------------------------------------------------------------------------------
function sleep(time) {
return new Promise(resolve => setTimeout(resolve, time));
}
// Using:
// const {blob, lastModifiedDate, contentType, filename, name, extension, status} = await fetchResource(url);
//
async function fetchResource(url, init = {}) {
const response = await fetch(url, {
cache: "force-cache",
...init,
});
const {status} = response;
const lastModifiedDateSeconds = response.headers.get("last-modified");
const contentType = response.headers.get("content-type");
const lastModifiedDate = dateToDayDateString(lastModifiedDateSeconds);
const extension = extensionFromMime(contentType);
const blob = await response.blob();
const _url = new URL(url);
const {filename} = (_url.origin + _url.pathname).match(/(?<filename>[^\/]+$)/).groups;
const {name} = filename.match(/(?<name>^[^\.]+)/).groups;
return {blob, lastModifiedDate, contentType, filename, name, extension, status};
}
// "Sun, 10 Jan 2021 22:22:22 GMT" -> "2021.01.10"
function dateToDayDateString(dateValue, utc = true) {
const _date = new Date(dateValue);
if (_date.toString() === "Invalid Date") {
throw "Invalid Date";
}
function pad(str) {
return str.toString().padStart(2, "0");
}
const _utc = utc ? "UTC" : "";
const year = _date[`get${_utc}FullYear`]();
const month = _date[`get${_utc}Month`]() + 1;
const date = _date[`get${_utc}Date`]();
return year + "." + pad(month) + "." + pad(date);
}
function extensionFromMime(mimeType) {
let extension = mimeType.match(/(?<=\/).+/)[0];
extension = extension === "jpeg" ? "jpg" : extension;
return extension;
}
function download(blob, name, url) {
const anchor = document.createElement("a");
anchor.setAttribute("download", name || "");
const blobUrl = URL.createObjectURL(blob);
anchor.href = blobUrl + (url ? ("#" + url) : "");
anchor.click();
setTimeout(() => URL.revokeObjectURL(blobUrl), 5000);
} |
|
where did u get that code? |
|
i dont see any download button on the page after installing the script |
It's not in the page.
|
|
i clicked the button but nothing happened tho |
|
Found the typo. Fixed. Should work now. (It worked for me because of the existence of another my script.) It works for me. Tested in Firefox with Violentmonkey, Tampermonkey and in Chrome. |
|
Still doesnt work. I use firefox. |
|
Well, I have fixed multiple bugs (3 total, the last two bugs were Firefox only), maybe you just tried to use an intermediate version, or cached one. If it still does not work for you, I'm don't know the reason. |
|
Still doesnt work for me. maybe i was missing the other script you installed |
|
By the way, in some kind you are right. Okay, finally fixed. |
|
finally it works! now we need somebody who can implement it to gellary-dl |
|
ty! |
|
8475698 looks good, however, I don't think that these are proper: gallery-dl/gallery_dl/extractor/pixiv.py Line 222 in 8475698 gallery-dl/gallery_dl/extractor/pixiv.py Line 242 in 8475698 Avatar and BG can be changed time by time, but this It should additionally use a value from the URL:
to download new versions of bg/ava. BTW, the question: I would like to create a filename like here #2495 (comment)
With using |
|
I think [bg] filename_hash.jpg would be sufficient. you could add date too so it be like [bg] [date] filename_hash.jpg |
- add 'date' metadata to avatar/background files when available and use that in default filenames / archive ids - remove deprecation warnings as their option names clash with subcategory names
I did not consider that it can change over time. Should be fixed in 9adea93.
By putting options for them in an "pixiv":
{
"avatar" : {
"directory": ["{category}", "{user[id]}"],
"filename" : "..."
},
"background": {
"directory": ["{category}", "{user[id]}"],
"filename" : "..."
}
}
Because the |
|
@mikf with "directory": ["{category}", "{user[id]}"] setting. will they both create avatar and background subfolders? sorry im new in this |
|
Look working, one minor thing I still want is "hash" from filename_hash = filename.split("_")[1]It's not important value, but I would prefer to save this just in case. Currently
For subfolders: BTW,
|
Since
That's by design. All other |
|
Probably the last issue is how to prevent running "metadata" postprocessor for "background", "avatar"? Limit it only by "artworks" subcategory. https://github.com/mikf/gallery-dl/blob/master/docs/configuration.rst UPD: "pixiv": {
"artworks": {
"postprocessors": []
}
} |
|
i want to put specific tags folder in a R-18 folder. how to do that? the rest of downloads if dont have specific tags i specify. it will go to NSFW or non NSFW folder |
|
@AlttiRi this concept is explained at the very top and applies for all/most options: @afterdelight conditional directory and filenames: "directory": {
"'女の子' in tags": ["{category}", "{user[id]}", "{rating}", "girl"],
"'足指' in tags" : ["{category}", "{user[id]}", "{rating}", "toes"],
"" : ["{category}", "{user[id]}", "{rating}"]
}
|
|
so i cant write the tags in english? whats that rating for? to seperate r18 and non r18? |
|
|
A bit questionable thing: This user has no bg. I have set Although |
okay man, thanks for the info |
|
how to put if there are two tags in a post such as if it has r-18 and feet goes to |
BugBG downloading fails if the extension is Does it really require guessing the extension? |
|
Guessing the extension sadly is necessary, or at least I haven't found a better way. https://www.pixiv.net/ajax/user/ would return the full URL, but it sometimes return an empty The mobile API only returns the _master1200 version, which almost always has .jpg as extension, but I think I've seen a .gif background that worked without fallback / guessing, which is why .gif wasn't included in the fallback list until now. |
|
Just rechecked with my userscript above (it uses https://www.pixiv.net/ajax/user/) the downloaded BGs by gallery-dl. It (this endpoint) looks working fine. Maybe you did tests without cookies? |
|
Well, yes. gallery-dl doesn't use cookies for Pixiv, only an OAuth access_token. |
hi guys, could you please answer my previous question? thanks. or should i create a new issue? |
|
#2513 bump this one. Or create a new one in Discussions. For me, it's not good idea to have Pixiv works grouped by a tag, instead of an artists. I keep descriptions and tags within metadata html files placed near in a subfolder. BTW, Windows's search can find text context within text (html) files, if you enable indexing of the folder. (By default indexing enabled only for "Download" and a few other system folders.) My config: "pixiv":
{
"tags": "translated",
"directory": ["[gallery-dl]", "[{category}] {user[id]}—{user[name]}"],
"filename": "[{category}] {user[id]}—{id}—{user[name]}—{date:%Y.%m.%d}—{title}—{num}.{extension}",
"include": ["background", "artworks"],
"background": {
"filename" : "[{category}][bg] {user[id]}—{user[name]}—{date:?//%Y.%m.%d}—{filename[-32:]}.{extension}"
},
"avatar": {
"filename" : "[{category}][ava] {user[id]}—{user[name]}—{date:?//%Y.%m.%d}—{filename[-32:]}.{extension}"
},
"artworks": {
"postprocessors": [{
"mtime": true,
"directory": "metadata",
"filename": "[{category}] {user[id]}—{id}—{user[name]}—{date:%Y.%m.%d}—{title}.html",
"name": "metadata",
"mode": "custom",
"format": "<div id='{id}'><h4><a href='https://www.pixiv.net/artworks/{id}'>{title}</a> by <a href='https://www.pixiv.net/users/{user[id]}'>{user[name]}</a></h4><div class='content'>{caption}</div><hr><div>{user[id]}—{id}—{date:%Y.%m.%d %H:%M:%S}{frames[0][delay]:?<br>Ugoira delay: / ms/}</div><hr><div class='tags'>[\"{tags:J\", \"}\"]</div><hr></div>"
}]
}
},Also it's useful to concatinate all html files into one and open it with a browser with bash command: cat *.html > "$TEMP/_temp-catahtml-result.html"; start "$TEMP/_temp-catahtml-result.html"; sleep 0; exit;When you are looking for links in descriptions. |
|
no, what i really want is like this post which contains r18 and hand tag will go to: pixiv-user/r-18/hand/file please guide me |
Please add an option to include download pixiv user background banner. ty
The text was updated successfully, but these errors were encountered: