Upd pikabu parser

NC22 · Dec 16, 2023 · d1f7373 · d1f7373
1 parent cb4b2d8
commit d1f7373
Show file tree

Hide file tree

Showing 10 changed files with 226 additions and 49 deletions.
diff --git a/extension/_locales/en/messages.json b/extension/_locales/en/messages.json
@@ -768,10 +768,10 @@
         "message":"Related doc"
     },   
     "related_doc_limit_one":{  
-        "message":"Take only biggest one image per document"
+        "message":"Take only one best match image"
     },   
     "related_doc_limit_all":{  
-        "message":"Take any image that bigger then preview"
+        "message":"Take all images that bigger then preview"
     }, 
     "restore":{  
         "message":"Restore"

diff --git a/extension/env/init/recorderPopup.js b/extension/env/init/recorderPopup.js
@@ -429,13 +429,15 @@ KellyPopupPage.init = function() {
         sm.loadDB('config', function(fav) { 
 
             if (fav) {
+
                 if (fav.coptions.toolbar && fav.coptions.toolbar.heartHidden) {
                     KellyPopupPage.buttons['support_project'].hidden = true; 
                 }
 
-                // console.log(fav.coptions);
-
                 if (fav.coptions.darkTheme) KellyPopupPage.css.push('darkRecorderPopup');
+
+            } else {
+                 KellyPopupPage.css.push('darkRecorderPopup');
             }
 
             KellyTools.getBrowser().runtime.sendMessage({method: "getResources", items : KellyPopupPage.css}, function(request) {

diff --git a/extension/lib/kellyFavItems.js b/extension/lib/kellyFavItems.js
@@ -1858,7 +1858,12 @@ function KellyFavItems(cfg)
             }
         }
 
-        updatePagination(KellyTools.getElementByClass(sideBarWrap, env.className + '-pagination'));     
+        updatePagination(KellyTools.getElementByClass(sideBarWrap, env.className + '-pagination'));  
+
+        if (env.events.onUpdateImagesBlock) {
+            env.events.onUpdateImagesBlock(handler, imagesBlock, startItem, end, page, totalPages);
+        }
+
         return true;
     }
 

diff --git a/extension/lib/kellyGrabber.js b/extension/lib/kellyGrabber.js
@@ -946,10 +946,10 @@ function KellyGrabber(cfg) {
 
     function showDownloadItemInfoTooltip(downloadIndex, target) {
 
-        if (KellyTools.getElementByClass(document, fav.getGlobal('env').className + '-tooltipster-help')) {
-            KellyTools.log('tooltip is not exist', 'KellyGrabber');
-            return;
-        }
+       // if (KellyTools.getElementByClass(document, fav.getGlobal('env').className + '-tooltipster-help')) {
+       //     KellyTools.log('tooltip is not exist', 'KellyGrabber');
+       //     return;
+       // }
 
         if (!downloads[downloadIndex]) {
             KellyTools.log('cant display item info by downloadIndex : ' + downloadIndex, 'KellyGrabber');

diff --git a/extension/lib/kellyOptions.js b/extension/lib/kellyOptions.js
@@ -333,7 +333,7 @@ function KellyOptions(cfg) {
         if (!fav.coptions.grid.rowHeight || fav.coptions.grid.rowHeight <= 0) fav.coptions.grid.rowHeight = 250;
         if (!fav.coptions.grid.min || fav.coptions.grid.min <= 0) fav.coptions.grid.min = 2;
         if (!fav.coptions.grid.heightDiff || fav.coptions.grid.heightDiff <= 0) fav.coptions.grid.heightDiff = 10;
-        if (!fav.coptions.grid.perPage || fav.coptions.grid.perPage <= 0) fav.coptions.grid.perPage = 60;        
+        if (!fav.coptions.grid.perPage || fav.coptions.grid.perPage <= 0) fav.coptions.grid.perPage = 40;        
         if (fav.coptions.grid.perPage > 1000) fav.coptions.grid.perPage = 1000;        
         if (fav.coptions.grid.heightDiff > 60) fav.coptions.grid.heightDiff = 60;
 

diff --git a/extension/lib/kellyStorageManager.js b/extension/lib/kellyStorageManager.js
@@ -1329,7 +1329,7 @@ function KellyFavStorageManager(cfg) {
                 addToFavSide : false,                
                 addToFavNoConfirm : false,
                 webRequest : true,
-                // darkTheme : true,
+                darkTheme : true,
                 // storageDesc - associations of .storage keys with pretty names | todo - make changable from options [Profiles] tab
             };
         }
@@ -1358,7 +1358,7 @@ function KellyFavStorageManager(cfg) {
                 heightDiff : 10,
                 min : 2, 
                 cssItem : '',
-                perPage : 60,                
+                perPage : 40,                
                 viewerShowAs : 'hd',
                 autoScroll : 1,
                 lazy : true,

diff --git a/extension/lib/recorder/filters/pikabu.js b/extension/lib/recorder/filters/pikabu.js
@@ -22,19 +22,159 @@ KellyRecorderFilterPikabu.onBeforeDownloadValidate = function(handler, data) {
 
 }
 
+KellyRecorderFilterPikabu.previewLoader = {
+    work : [],
+    all : [],
+    images : [],
+    done : 0,
+    aborted : false,
+    abort : function() {
+
+        console.log('[KellyRecorderFilterPikabu] Decoder cleanup');
+
+        KellyRecorderFilterPikabu.previewLoader.aborted = true;
+        var works = KellyRecorderFilterPikabu.previewLoader.work;
+        for (var i = 0; i < works.length; i++) {
+            if (works[i].request) {
+                console.log('[KellyRecorderFilterPikabu] stopped  [' + works[i].src + ']');
+                works[i].request.abort();
+            }
+        }
+
+        KellyRecorderFilterPikabu.previewLoader.done = 0;
+        KellyRecorderFilterPikabu.previewLoader.work = [];
+    },
+    refresher : function(limit) {
+
+        var info = document.getElementById('pikabu-decode-counter');
+        info.innerText = 'Осталось расшифровать : ' + (KellyRecorderFilterPikabu.previewLoader.images.length + KellyRecorderFilterPikabu.previewLoader.work.length);
+
+        if (KellyRecorderFilterPikabu.previewLoader.done >= limit) {
+            K_FAV.updateImagesBlock();                
+            K_FAV.updateImageGrid();
+            KellyRecorderFilterPikabu.previewLoader.done = 0;
+        }
+    },
+    run : function() {
+
+        if (KellyRecorderFilterPikabu.previewLoader.aborted) return;
+
+        var works = KellyRecorderFilterPikabu.previewLoader.work;
+        if (works.length >= 3) return;
+
+        var job = KellyRecorderFilterPikabu.previewLoader.images.pop();
+        if (!job) {
+            KellyRecorderFilterPikabu.previewLoader.refresher(1);
+            if (KellyRecorderFilterPikabu.tooltip) KellyRecorderFilterPikabu.tooltip.show(false);
+            return;
+        }
+
+        works.push(job);
+        console.log('[KellyRecorderFilterPikabu] init work [' + job.src + ']');
+        job.request = KellyTools.fetchRequest(job.src, {responseType : 'arrayBuffer'}, function(urlOrig, arrayBuffer, errorCode, errorText, controller) {
+
+            job.request = false;
+            var jobKey = works.indexOf(job);
+            if (jobKey != -1) {
+                works.splice(jobKey, 1);
+            }
+
+            if (arrayBuffer !== false) { 
+
+                var byteArray = new Uint8Array( arrayBuffer );            
+                var result = KellyRecorderFilterPikabu.deNSFW.descramble(byteArray, parseInt(job.src.split('#nsfw')[1]));
+                if (result.base64) {
+                    job.item.pImage = URL.createObjectURL(KellyTools.base64toBlob(result.base64, result.contentType));
+                    console.log('[KellyRecorderFilterPikabu] Done ...  [' + job.src + ']');
+                    KellyRecorderFilterPikabu.previewLoader.done++;
+                    KellyRecorderFilterPikabu.previewLoader.refresher(10);
+
+                } else {
+                    //data.self.aDProgress.addErrorItem(false, 'Decode ... Err base64 [' + job.imgEl.src + ']');
+                    console.log('[KellyRecorderFilterPikabu] Decode ... Err base64  [' + job.src + ']');
+                }
+
+                byteArray = null;
+            } else {
+                //data.self.aDProgress.addErrorItem(false, 'Decode ... Err arrayBuffer [' + job.imgEl.src + ']');
+                console.log('[KellyRecorderFilterPikabu] Decode ... Err arrayBuffer  [' + job.src + ']');
+
+            }
+
+            setTimeout(KellyRecorderFilterPikabu.previewLoader.run, 500);
+        });
+
+    },
+    add : function(item) {
+
+        if (KellyRecorderFilterPikabu.previewLoader.all.indexOf(item.id) != -1) return;
+
+        KellyRecorderFilterPikabu.previewLoader.all.push(item.id);
+        console.log('[KellyRecorderFilterPikabu] Decode ... [' + item.id + ']');
+        //data.self.aDProgress.addErrorItem(false, 'Decode ... [' + imgEl.src + ']');
+
+        KellyRecorderFilterPikabu.previewLoader.images.push({src : item.pImage, item : item});
+        KellyRecorderFilterPikabu.previewLoader.run();
+    },
+    decodeCurrentPage : function() {
+
+         KellyRecorderFilterPikabu.previewLoader.aborted = false;
+         var dList = K_FAV.getGlobal('filtered');
+         var items = K_FAV.getGlobal('fav').items;
+
+         for (var i = KellyRecorderFilterPikabu.previewLoader.initData.startItem; i <= KellyRecorderFilterPikabu.previewLoader.initData.end; i++) {
+
+             if (items[dList[i]].pImage.indexOf('#nsfw') != -1) {
+                KellyRecorderFilterPikabu.previewLoader.add(items[dList[i]]);
+            }
+         }
+
+
+        if (KellyRecorderFilterPikabu.tooltip) KellyRecorderFilterPikabu.tooltip.show(KellyRecorderFilterPikabu.previewLoader.images.length > 0);  
+    }
+
+}
+
+KellyRecorderFilterPikabu.onUpdateImagesBlock = function(handler, data) {
+
+
+    if (handler.url.indexOf('pikabu.ru') == -1) return;
+
+     KellyRecorderFilterPikabu.previewLoader.initData = data;
+     KellyRecorderFilterPikabu.previewLoader.decodeCurrentPage();
+
+
+}
+
 KellyRecorderFilterPikabu.onRecorderImagesShow = function(handler, data) {
 
     if (handler.url.indexOf('pikabu.ru') == -1) return;
 
     if (KellyDPage.cats['pikabu_nsfw'] && KellyDPage.cats['pikabu_nsfw'].id) {
 
-        var tooltip = KellyTools.getNoticeTooltip(KellyDPage.env.hostClass, KellyDPage.env.className); 
-
-            KellyTools.setHTMLData(tooltip.getContent(), '<div>На странице обнаружены <b>NSFW</b> картинки которые сайт отдает в зашифрованном виде. ' + 
-            'При скачивании расширение попробует их автоматически разблокировать и сохранить в исходном виде. ' + 
-            'Если разработчики сайта поменяли алгоритм и разблокировка перестала работать, дайте знать на <b>radiokellyc@gmail.com</b></div>'); 
+            KellyRecorderFilterPikabu.tooltip = KellyTools.getNoticeTooltip(KellyDPage.env.hostClass, KellyDPage.env.className); 
+            KellyRecorderFilterPikabu.tooltip.updateCfg({removeOnClose : false, closeByBody : false});
+
+            KellyTools.setHTMLData(KellyRecorderFilterPikabu.tooltip.getContent(), '<div><b>Расшифровка превью контента : </b> [<a href="#" id="pikabu-decode-stop">Остановить</a>]</div><div id="pikabu-decode-counter">Инициализация...</div><div>На странице обнаружены <b>NSFW</b> картинки которые сайт отдает в зашифрованном виде, необязательно дожидатся расшифровки превью, данные будут дешифрованы при скачивании</div>'); 
+
+            document.getElementById('pikabu-decode-stop').onclick = function() {
+
+                if (this.getAttribute('data-stoped') == '1') {
+
+                    this.setAttribute('data-stoped', '0');
+                    this.innerText = 'Остановить';
+                    KellyRecorderFilterPikabu.previewLoader.all = [];
+                    KellyRecorderFilterPikabu.previewLoader.decodeCurrentPage();
+
+                } else {
+                    this.setAttribute('data-stoped', '1');
+                    this.innerText = 'Начать';
+                    KellyRecorderFilterPikabu.previewLoader.abort();
+                }
+
+            }
 
-            tooltip.show(true);            
+            // KellyRecorderFilterPikabu.tooltip.show(true);            
     }
 }
 
@@ -50,32 +190,43 @@ KellyRecorderFilterPikabu.addItemByDriver = function(handler, data) {
         if (data.item.relatedSrc.length <= 0) return handler.addDriverAction.SKIP;
         return handler.addDriverAction.ADD;
 
-    // catch decompiled item, oreder is important (must be upper common "article" detector)
+    // catch decompiled item, order is important (must be upper common "article" detector)
 
-    } else if (data.el.tagName == 'IMG' && data.el.getAttribute('data-src') && data.el.getAttribute('data-viewable') && KellyTools.getParentByTag(data.el, 'ARTICLE')) {
-
-        var pdata = data.el.getAttribute('data-src');
-        var fdata = data.el.getAttribute('data-large-image');
-        var fGroups = KellyRecorderFilterPikabu.getPostGroups(false);
-        var pGroups = ['pikabu_post_preview'];
+    } else if (data.el.tagName == 'IMG' && data.el.getAttribute('data-src') && data.el.getAttribute('data-viewable')) {
 
-        var marker = '';        
-        if (data.el.getAttribute('data-scrambler-offset') !== null) {
-            marker = '#nsfw' + data.el.getAttribute('data-scrambler-offset');
-            fGroups.push('pikabu_nsfw');
-            pGroups.push('pikabu_nsfw');
+        var dGroup = false;
+        if (KellyTools.getParentByTag(data.el, 'ARTICLE')) {
+            dGroup = 'pikabu_post_preview';
+        } else if (KellyTools.getParentByClass(data.el, 'comment')) {
+            dGroup = 'pikabu_comment';
         }
 
-        if (fdata && pdata && pdata.indexOf(fdata) == -1) {
-            handler.addSingleSrc(data.item, data.el.getAttribute('data-src') + marker, 'addSrcFromAttributes-src', data.el, pGroups);
-        } else {
-            fGroups.push('pikabu_post_preview');
-        }
+        if (dGroup !== false) {
 
-        handler.addSingleSrc(data.item, data.el.getAttribute('data-large-image') + marker, 'addSrcFromAttributes-src', data.el, fGroups);  
-
-        if (data.item.relatedSrc.length <= 0) return handler.addDriverAction.SKIP;
-        return handler.addDriverAction.ADD;
+            var pdata = data.el.getAttribute('data-src');
+            var fdata = data.el.getAttribute('data-large-image');
+            var fGroups = KellyRecorderFilterPikabu.getPostGroups(false);
+            var pGroups = [];
+                pGroups.push(dGroup);
+
+            var marker = '';        
+            if (data.el.getAttribute('data-scrambler-offset') !== null) {
+                marker = '#nsfw' + data.el.getAttribute('data-scrambler-offset');
+                fGroups.push('pikabu_nsfw');
+                pGroups.push('pikabu_nsfw');
+            }
+
+            if (fdata && pdata && pdata.indexOf(fdata) == -1) { // large version exist (preview src != full src) -> add preview image to list
+                handler.addSingleSrc(data.item, data.el.getAttribute('data-src') + marker, 'addSrcFromAttributes-src', data.el, pGroups);
+            } else {
+                fGroups.push(dGroup);
+            }
+
+            handler.addSingleSrc(data.item, data.el.getAttribute('data-large-image') + marker, 'addSrcFromAttributes-src', data.el, fGroups);  
+
+            if (data.item.relatedSrc.length <= 0) return handler.addDriverAction.SKIP;
+            return handler.addDriverAction.ADD;
+        }
 
     }
 }
@@ -123,7 +274,7 @@ KellyRecorderFilterPikabu.onStartRecord = function(handler, data) {
      handler.additionCats = {
 
         pikabu_comment : {name : 'Comment', color : '#b7dd99', selected : 110},
-        pikabu_post : {name : 'Post', color : '#b7dd99', selected : 110},
+        pikabu_post : {name : 'Post', color : '#b7dd99', selected : 111},
         pikabu_post_preview : {name : 'Post preview', color : '#b7dd99', selected : 100},
         pikabu_nsfw : {name : 'Encoded NSFW', color : '#ff0983'},
         pikabu_post_video : {name : 'Post Video', color : '#218bea'},
@@ -222,8 +373,8 @@ KellyPageWatchdog.validators.push({
     host : 'pikabu.ru', 
     patterns : [
         ['images/previews_comm', 'imageAny'], 
-        ['/post_img/', 'imageAny'], 
-        ['images/big_size_comm', 'imageOriginal'], ['_comm', 'pikabu_comment'], ]
+        ['/post_img/', 'imageAny']
+   ]
 });
 
 KellyPageWatchdog.filters.push(KellyRecorderFilterPikabu);