From aca92c4ee5ee42d247174e0217064baaa0c92804 Mon Sep 17 00:00:00 2001 From: difegue Date: Tue, 30 Jan 2018 23:17:09 +0100 Subject: [PATCH] Bugfixes aplenty + Started work on Namespaces. Signed-off-by: difegue --- lib/LANraragi/Controller/Api.pm | 8 +-- lib/LANraragi/Controller/Edit.pm | 5 +- lib/LANraragi/Controller/Index.pm | 2 +- lib/LANraragi/Controller/Reader.pm | 11 ++-- lib/LANraragi/Controller/Stats.pm | 2 +- lib/LANraragi/Controller/Upload.pm | 10 ++- lib/LANraragi/Model/Backup.pm | 2 +- lib/LANraragi/Model/Config.pm | 7 +- lib/LANraragi/Model/Plugins.pm | 4 +- lib/LANraragi/Model/Reader.pm | 5 +- lib/LANraragi/Model/Utils.pm | 100 ++++++++++++++++------------- lib/Shinobu.pm | 7 +- public/js/index_datatables.js | 2 +- {t => tests}/basic.t | 0 14 files changed, 90 insertions(+), 75 deletions(-) rename {t => tests}/basic.t (100%) mode change 100755 => 100644 diff --git a/lib/LANraragi/Controller/Api.pm b/lib/LANraragi/Controller/Api.pm index 5802db987..76ec94de9 100755 --- a/lib/LANraragi/Controller/Api.pm +++ b/lib/LANraragi/Controller/Api.pm @@ -66,15 +66,15 @@ sub serve_thumbnail { my $redis = $self->LRR_CONF->get_redis(); my $file = $redis->hget($id,"file"); - $file = decode_utf8($file); + $file = LANraragi::Model::Utils::redis_decode($file); my $path = "./public/temp/thumb"; #delete everything in thumb temp to prevent file mismatch errors. unlink glob $path."/*.*"; #Get lsar's output, jam it in an array, and use it as @extracted. + print $file; my $vals = `lsar "$file"`; - #print $vals; my @lsarout = split /\n/, $vals; my @extracted; @@ -101,7 +101,7 @@ sub serve_thumbnail { #While we have the image, grab its SHA-1 hash for potential tag research later. #That way, no need to repeat the costly extraction later. - my $shasum = LANraragi::Model::Utils::shasum($arcimg); + my $shasum = LANraragi::Model::Utils::shasum($arcimg,1); $redis->hset($id,"thumbhash", encode_utf8($shasum)); #Thumbnail generation @@ -140,7 +140,7 @@ sub add_archive { if (index($file,$userdir) == 0) { #utf8 decode the filename - eval { $file = decode_utf8($file) }; + LANraragi::Model::Utils::redis_decode($file); #Archive adding is in the Utils package LANraragi::Model::Utils::add_archive_to_redis($id,$file,$redis); diff --git a/lib/LANraragi/Controller/Edit.pm b/lib/LANraragi/Controller/Edit.pm index f9c81b55e..f93549a31 100755 --- a/lib/LANraragi/Controller/Edit.pm +++ b/lib/LANraragi/Controller/Edit.pm @@ -18,7 +18,7 @@ sub delete_metadata_and_file my $redis = $self->LRR_CONF->get_redis(); my $filename = $redis->hget($id, "file"); - $filename = decode_utf8($filename); + $filename = LANraragi::Model::Utils::redis_decode($filename); #print $filepath; $redis->del($id); @@ -96,11 +96,12 @@ sub index { my $redis = $self->LRR_CONF->get_redis(); + #TODO: Fix this to use new syntax w.namespaces if ($redis->hexists($id,"title")) { my %hash = $redis->hgetall($id); my ($name,$event,$artist,$title,$series,$language,$tags,$file,$thumbhash) = @hash{qw(name event artist title series language tags file thumbhash)}; - ($_ = decode_utf8($_)) for ($name, $event, $artist, $title, $series, $language, $tags, $file); + ($_ = LANraragi::Model::Utils::redis_decode($_)) for ($name, $event, $artist, $title, $series, $language, $tags, $file); $redis->quit(); diff --git a/lib/LANraragi/Controller/Index.pm b/lib/LANraragi/Controller/Index.pm index 99ae83d50..418ab91a8 100755 --- a/lib/LANraragi/Controller/Index.pm +++ b/lib/LANraragi/Controller/Index.pm @@ -33,7 +33,7 @@ sub random_archive if (length($archive)==64 && $redis->type($archive) eq "hash" && $redis->hexists($archive,"file")) { my $arclocation = $redis->hget($archive,"file"); - $arclocation = decode_utf8($arclocation); + $arclocation = LANraragi::Model::Utils::redis_decode($arclocation); if (-e $arclocation) { $archiveexists = 1; } diff --git a/lib/LANraragi/Controller/Reader.pm b/lib/LANraragi/Controller/Reader.pm index f6686c98f..b8728ddfa 100755 --- a/lib/LANraragi/Controller/Reader.pm +++ b/lib/LANraragi/Controller/Reader.pm @@ -22,15 +22,16 @@ sub index { { $self->redirect_to('index'); } #Get a computed archive name if the archive exists - my $artist = "Unknown"; my $arcname = ""; - $artist = $redis->hget($id,"artist"); + my $tags = $redis->hget($id,"tags"); $arcname = $redis->hget($id,"title"); - unless ($artist =~ /^\s*$/) - {$arcname = $arcname." by ".$artist; } + if ($tags =~ /artist:/) { + $tags =~ /.*artist:([^,]*),.*/ ; + $arcname = $arcname." by ".$1; + } - $arcname = decode_utf8($arcname); + $arcname = LANraragi::Model::Utils::redis_decode($arcname); my $force = $self->req->param('force_reload'); my $thumbreload = $self->req->param('reload_thumbnail'); diff --git a/lib/LANraragi/Controller/Stats.pm b/lib/LANraragi/Controller/Stats.pm index d65214415..11b3bf3d5 100755 --- a/lib/LANraragi/Controller/Stats.pm +++ b/lib/LANraragi/Controller/Stats.pm @@ -29,7 +29,7 @@ sub index { if ($redis->hexists($id,"tags")) { $t = $redis->hget($id,"tags"); - $t = decode_utf8($t); + $t = LANraragi::Model::Utils::redis_decode($t); #Split tags by comma @tags = split(/,\s?/, $t); diff --git a/lib/LANraragi/Controller/Upload.pm b/lib/LANraragi/Controller/Upload.pm index 366034e88..01b60576a 100755 --- a/lib/LANraragi/Controller/Upload.pm +++ b/lib/LANraragi/Controller/Upload.pm @@ -23,7 +23,6 @@ sub process_upload { if(exists($acceptedTypes{$uploadMime})) { my $filename = $file->filename; - eval { $filename = decode_utf8($filename) }; my $output_file = $self->LRR_CONF->get_userdir.'/'.$filename; #open up a file on our side @@ -31,7 +30,7 @@ sub process_upload { #if it doesn't already exist, that is. $self->render( json => { operation => "upload", - name => $filename, + name => $file->filename, type => $uploadMime, success => 0, error => "A file bearing this name already exists in the Library." @@ -49,14 +48,13 @@ sub process_upload { #Parse for metadata right now and get the database ID my $redis = $self->LRR_CONF->get_redis(); - #my $utf8_file = Encode::encode_utf8($output_file); - my $id = LANraragi::Model::Utils::sha256_hex($output_file); + my $id = LANraragi::Model::Utils::shasum($output_file,256); LANraragi::Model::Utils::add_archive_to_redis($id,$output_file,$redis); $self->render( json => { operation => "upload", - name => $filename->filename, + name => $file->filename, type => $uploadMime, success => 1, id => $id @@ -67,7 +65,7 @@ sub process_upload { $self->render( json => { operation => "upload", - name => $filename->filename, + name => $file->filename, type => $uploadMime, success => 0, error => "Unsupported Filetype. (".$uploadMime.")" diff --git a/lib/LANraragi/Model/Backup.pm b/lib/LANraragi/Model/Backup.pm index bd781e7a6..ea0de7cb8 100755 --- a/lib/LANraragi/Model/Backup.pm +++ b/lib/LANraragi/Model/Backup.pm @@ -29,7 +29,7 @@ sub build_backup_JSON my %hash = $redis->hgetall($id); my ($event,$artist,$title,$series,$language,$tags) = @hash{qw(event artist title series language tags)}; - ($_ = decode_utf8($_)) for ($event, $artist, $title, $series, $language, $tags); + ($_ = LANraragi::Model::Utils::redis_decode($_)) for ($event, $artist, $title, $series, $language, $tags); $json.=qq( { diff --git a/lib/LANraragi/Model/Config.pm b/lib/LANraragi/Model/Config.pm index c12ae60b9..e452acc3f 100755 --- a/lib/LANraragi/Model/Config.pm +++ b/lib/LANraragi/Model/Config.pm @@ -5,6 +5,7 @@ use warnings; use utf8; use feature "switch"; no warnings 'experimental'; +use Cwd 'abs_path'; use Redis; use Encode; @@ -58,7 +59,7 @@ sub get_redis_conf { if ($redis->hexists("LRR_CONFIG",$param)) { - my $value = decode_utf8($redis->hget("LRR_CONFIG",$param)); + my $value = LANraragi::Model::Utils::redis_decode($redis->hget("LRR_CONFIG",$param)); unless ($value =~ /^\s*$/ ) #failsafe against blank config values { return $value; } @@ -77,7 +78,9 @@ sub get_userdir { #Try to create userdir if it doesn't already exist unless (-e $dir) { mkdir $dir; } - return $dir; + + #Return full path if it's relative, using the /lanraragi directory as a base + return abs_path($dir); }; sub get_tempmaxsize { return &get_redis_conf("tempmaxsize", "500") }; diff --git a/lib/LANraragi/Model/Plugins.pm b/lib/LANraragi/Model/Plugins.pm index cd9469520..090264bb5 100755 --- a/lib/LANraragi/Model/Plugins.pm +++ b/lib/LANraragi/Model/Plugins.pm @@ -32,7 +32,7 @@ sub exec_enabled_plugins_on_file { my %plugincfg = $redis->hgetall($namerds); my ($enabled, $arg) = @plugincfg{qw(enabled arg)}; - ($_ = decode_utf8($_)) for ($enabled, $arg); + ($_ = LANraragi::Model::Utils::redis_decode($_)) for ($enabled, $arg); if ($enabled) { &exec_plugin_on_file($plugin, $id, $arg); @@ -55,7 +55,7 @@ sub exec_plugin_on_file { my %hash = $redis->hgetall($id); my ($name,$event,$artist,$title,$series,$language,$tags,$file,$thumbhash) = @hash{qw(name event artist title series language tags file thumbhash)}; - ($_ = decode_utf8($_)) for ($name, $event, $artist, $title, $series, $language, $tags, $file); + ($_ = LANraragi::Model::Utils::redis_decode($_)) for ($name, $event, $artist, $title, $series, $language, $tags, $file); my %metadata_hash = ( title => $title, diff --git a/lib/LANraragi/Model/Reader.pm b/lib/LANraragi/Model/Reader.pm index 7f79d8e9b..d2717c40c 100755 --- a/lib/LANraragi/Model/Reader.pm +++ b/lib/LANraragi/Model/Reader.pm @@ -37,7 +37,7 @@ sub build_reader_JSON { #Get the path from Redis. my $zipfile = $redis->hget($id,"file"); - $zipfile = decode_utf8($zipfile); + $zipfile = LANraragi::Model::Utils::redis_decode($zipfile); #Get data from the path my ($name,$fpath,$suffix) = fileparse($zipfile, qr/\.[^.]*/); @@ -60,6 +60,7 @@ sub build_reader_JSON { #Has the archive been extracted ? If not, stop here and print an error page. unless (-e $path) { my $errlog = join "
", @$full_buf; + $errlog = decode_utf8($errlog); printf "ERROR while unpacking archive: $errlog"; die $errlog; } @@ -95,7 +96,7 @@ sub build_reader_JSON { { mkdir $dirname."/thumb"; my $path = $images[0]; - my $shasum = LANraragi::Model::Utils::shasum($path); + my $shasum = LANraragi::Model::Utils::shasum($path,1); $redis->hset($id,"thumbhash", encode_utf8($shasum)); LANraragi::Model::Utils::generate_thumbnail($path,$thumbname); diff --git a/lib/LANraragi/Model/Utils.pm b/lib/LANraragi/Model/Utils.pm index 0090baf5d..f3da5014f 100755 --- a/lib/LANraragi/Model/Utils.pm +++ b/lib/LANraragi/Model/Utils.pm @@ -92,23 +92,25 @@ sub generate_thumbnail { $img->Write($thumb_path); } -#This function gives us a SHA-1 hash for the passed file, which is used as an id for some files. +#This function gives us a SHA hash for the passed file, which is used for thumbnail reverse search on E-H. +#First argument is the file, second is the algorithm to use. (1, 224, 256, 384, 512, 512224, or 512256) sub shasum { + my $digest = ""; - eval{ - open(FILE, $_[0]) or die "Can't find file $_[0]\n"; - my $ctx = Digest::SHA->new; - $ctx->addfile(*FILE); - $digest = $ctx->hexdigest; - close(FILE); + eval { + my $ctx = Digest::SHA->new($_[1]); + $ctx->addfile($_[0]); + $digest = $ctx->hexdigest; }; + if($@){ print $@; return ""; } + return $digest; } - + #Remove spaces before and after a word sub remove_spaces { until (substr($_[0],0,1)ne" "){ @@ -118,6 +120,18 @@ sub remove_spaces { chop $_[0];} } +#Final Solution to the Unicode glitches -- Eval'd double-decode for data obtained from Redis. +#This should be a one size fits-all function. +sub redis_decode { + + my $data = $_[0]; + + eval { $data = decode_utf8($data) }; + eval { $data = decode_utf8($data) }; + + return $data; +} + #parse_name(name) #parses an archive name with the regex specified in the configuration file(get_regex and select_from_regex subs) to find metadata. sub parse_name { @@ -128,8 +142,28 @@ sub parse_name { #select_from_regex picks the variables from the regex selection that will be used. my ($event,$artist,$title,$series,$language) = LANraragi::Model::Config->select_from_regex; my $tags = ""; + + unless ($event eq "") { + unless ($tags eq "") { $tags.=", "; } + $tags .= "event:$event"; + } + + unless ($artist eq "") { + unless ($tags eq "") { $tags.=", "; } + $tags .= "artist:$artist "; + } + + unless ($series eq "") { + unless ($tags eq "") { $tags.=", "; } + $tags .= "parody:$series "; + } + + unless ($language eq "") { + unless ($tags eq "") { $tags.=", "; } + $tags .= "language:$language "; + } - return ($event,$artist,$title,$series,$language,$tags); + return ($title,$tags); } #add_archive_to_redis($id,$file,$redis) @@ -140,27 +174,18 @@ sub add_archive_to_redis { my ($name,$path,$suffix) = fileparse($file, qr/\.[^.]*/); #parse_name function is up there - my ($event,$artist,$title,$series,$language,$tags) = &parse_name($name.$suffix); + my ($title,$tags) = &parse_name($name.$suffix); #jam this shit in redis - #prepare the hash which'll be inserted. - my %hash = ( - name => encode_utf8($name), - event => encode_utf8($event), - artist => encode_utf8($artist), - title => encode_utf8($title), - series => encode_utf8($series), - language => encode_utf8($language), - tags => encode_utf8($tags), - file => encode_utf8($file), - isnew => encode_utf8("block"), #New file in collection, so this flag is set. - ); - - #for all keys of the hash, add them to the redis hash $id with the matching keys. - $redis->hset($id, $_, $hash{$_}, sub {}) for keys %hash; + $redis->hset($id, "name", encode_utf8($name)); + $redis->hset($id, "title", encode_utf8($title)); + $redis->hset($id, "tags", encode_utf8($tags)); + $redis->hset($id, "file", encode_utf8($file)); + $redis->hset($id, "isnew", "block"); #New file in collection, so this flag is set. + $redis->wait_all_responses; - return ($name,$event,$artist,$title,$series,$language,$tags,"block"); + return ($name,$title,$tags,"block"); } #build_archive_JSON(id, file, redis, userdir) @@ -172,10 +197,10 @@ sub build_archive_JSON { my ($path, $suffix); #It's not a new archive, but it might have never been clicked on yet, so we'll grab the value for $isnew stored in redis. - my ($name,$event,$artist,$title,$series,$language,$tags,$filecheck,$isnew) = @hash{qw(name event artist title series language tags file isnew)}; + my ($name,$title,$tags,$filecheck,$isnew) = @hash{qw(name title tags file isnew)}; #Parameters have been obtained, let's decode them. - ( eval { $_ = decode_utf8($_) } ) for ($name, $event, $artist, $title, $series, $language, $tags, $filecheck); + ( eval { $_ = LANraragi::Model::Utils::redis_decode($_) } ) for ($name, $title, $tags, $filecheck); #Update the real file path and title if they differ from the saved one just in case the file got manually renamed or some weird shit unless ($file eq $filecheck) @@ -185,20 +210,10 @@ sub build_archive_JSON { $redis->hset($id, "name", encode_utf8($name)); $redis->wait_all_responses; } - - #Grab the suffix to put it in the url for downloads - $suffix = (fileparse($file, qr/\.[^.]*/))[2]; - - #Once we have the data, we can build our json object. - my $urlencoded = $dirname."/".uri_escape($name).$suffix; #Tag display. Simple list separated by hyphens which expands into a caption div with nicely separated tags on hover. + #TODO: Create a JS-side display separating tags by namespaces... my $printedtags = ""; - - unless ($event eq "") - { $printedtags = $event.", ".$tags; } - else - { $printedtags = $tags;} if ($title =~ /^\s*$/) #Workaround if title was incorrectly parsed as blank { $title = " Untitled archive, please edit metadata.";} @@ -206,19 +221,12 @@ sub build_archive_JSON { my $finaljson = qq( { "arcid": "$id", - "url": "$urlencoded", - "artist": "$artist", "title": "$title", - "series": "$series", - "language": "$language", "tags": "$tags", "isnew": "$isnew" }, ); - #Try to UTF8-decode the JSON again, in case it has mangled characters. - eval { $finaljson = decode_utf8($finaljson) }; - return $finaljson; } diff --git a/lib/Shinobu.pm b/lib/Shinobu.pm index 60f27a6e3..c6ff7be5d 100644 --- a/lib/Shinobu.pm +++ b/lib/Shinobu.pm @@ -13,6 +13,7 @@ use strict; use warnings; use utf8; use feature qw(say); +use Cwd; use FindBin; BEGIN { unshift @INC, "$FindBin::Bin/../lib"; } #As this is a new process, reloading the LRR libs into INC is needed. @@ -30,6 +31,7 @@ sub initialize_from_new_process { my $interval = LANraragi::Model::Config::get_interval; say ("Shinobu Background Worker started -- Running every $interval seconds."); + say ("Working dir is ".cwd); while (1) { @@ -77,7 +79,7 @@ sub build_json_cache { foreach $file (@dircontents) { #ID of the archive, used for storing data in Redis. - $id = LANraragi::Model::Utils::sha256_hex($file); + $id = LANraragi::Model::Utils::shasum($file,256); #Craft JSON if archive is in Redis if ($redis->hexists($id,"title")) { @@ -101,11 +103,12 @@ sub new_archive_check { foreach $file (@dircontents) { #ID of the archive, used for storing data in Redis. - $id = LANraragi::Model::Utils::sha256_hex($file); + $id = LANraragi::Model::Utils::shasum($file,256); #Trigger archive addition if title isn't in Redis unless ($redis->hexists($id,"title")) { say ("Adding new file $file"); + say ("ID is $id"); LANraragi::Model::Utils::add_archive_to_redis($id,$file,$redis); #TODO: AutoTagging using enabled plugins goes here! diff --git a/public/js/index_datatables.js b/public/js/index_datatables.js index 1d371f749..23cbe14ca 100644 --- a/public/js/index_datatables.js +++ b/public/js/index_datatables.js @@ -163,7 +163,7 @@ function buildThumbDiv( row, data, index ) { '
'+ '
'+ data.artist+'
'+ - ''+ + ''+ ''+ ''+ ''+ diff --git a/t/basic.t b/tests/basic.t old mode 100755 new mode 100644 similarity index 100% rename from t/basic.t rename to tests/basic.t