From 7104c84c5ac749f0e10d065ad8c1eaa45441e941 Mon Sep 17 00:00:00 2001 From: polak14 <76823966+polak14@users.noreply.github.com> Date: Wed, 2 Nov 2022 13:00:54 +0000 Subject: [PATCH 01/45] Update EHentai.pm updated text search for the new search system (mostly just removing now deprecated flags) image search no longer supports searching tags --- lib/LANraragi/Plugin/Metadata/EHentai.pm | 27 +++++------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/lib/LANraragi/Plugin/Metadata/EHentai.pm b/lib/LANraragi/Plugin/Metadata/EHentai.pm index 5df933036..a26f31826 100644 --- a/lib/LANraragi/Plugin/Metadata/EHentai.pm +++ b/lib/LANraragi/Plugin/Metadata/EHentai.pm @@ -134,21 +134,9 @@ sub lookup_gallery { #search with image SHA hash $URL = $domain - . "?advsearch=1&f_sname=on&f_sdt2=on&f_spf=&f_spt=&f_sfu=on&f_sft=on&f_sfl=on&f_shash=" + . "?f_shash=" . $thumbhash - . "&fs_covers=1&fs_similar=1"; - - #Include expunged galleries in the search if the option is enabled. - if ($expunged) { - $URL = $URL . "&fs_exp=1"; - } - - # Add the language override, if it's defined. - if ( $defaultlanguage ne "" ) { - - # Add f_stags to search in tags for language - $URL = $URL . "&f_stags=on&f_search=" . uri_escape_utf8("language:$defaultlanguage"); - } + . "&fs_similar=on&fs_covers=on"; $logger->debug("Using URL $URL (archive thumbnail hash)"); @@ -159,10 +147,10 @@ sub lookup_gallery { } } - # Regular text search + # Regular text search (advanced options: Disable default filters for: Language, Uploader, Tags) $URL = $domain - . "?advsearch=1&f_sname=on&f_sdt2=on&f_spf=&f_spt=&f_sfu=on&f_sft=on&f_sfl=on" + . "?advsearch=1&f_sfu=on&f_sft=on&f_sfl=on" . "&f_search=" . uri_escape_utf8( qw(") . $title . qw(") ); @@ -179,12 +167,7 @@ sub lookup_gallery { $URL = $URL . "+" . uri_escape_utf8("language:$defaultlanguage"); } - # Add f_stags to search in tags if we added a tag (or two) in the search - if ( $has_artist || $defaultlanguage ne "" ) { - $URL = $URL . "&f_stags=on"; - } - - # Include expunged galleries in the search if the option is enabled. + # Search expunged galleries if the option is enabled. if ($expunged) { $URL = $URL . "&f_sh=on"; } From a38d391a22413d159d68fe688aeaa0c6c8d7c320 Mon Sep 17 00:00:00 2001 From: polak14 <76823966+polak14@users.noreply.github.com> Date: Wed, 2 Nov 2022 13:05:17 +0000 Subject: [PATCH 02/45] Update EHentai.pm "- Selecting "Search Expunged Galleries" will now only search expunged galleries in normal searches. (File searches, GID searches and favorite searches will always display both normal and expunged galleries.)" --- lib/LANraragi/Plugin/Metadata/EHentai.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/LANraragi/Plugin/Metadata/EHentai.pm b/lib/LANraragi/Plugin/Metadata/EHentai.pm index a26f31826..075db4160 100644 --- a/lib/LANraragi/Plugin/Metadata/EHentai.pm +++ b/lib/LANraragi/Plugin/Metadata/EHentai.pm @@ -39,7 +39,7 @@ sub plugin_info { desc => "Save the original title when available instead of the English or romanised title" }, { type => "bool", desc => "Fetch additional timestamp (time posted) and uploader metadata" }, - { type => "bool", desc => "Search expunged galleries as well" }, + { type => "bool", desc => "Search only expunged galleries" }, ], oneshot_arg => "E-H Gallery URL (Will attach tags matching this exact gallery to your archive)", From 01a7f0d7f1aaaa81c57cef2df95de5bcd63404ae Mon Sep 17 00:00:00 2001 From: polak14 <76823966+polak14@users.noreply.github.com> Date: Wed, 2 Nov 2022 18:16:09 +0000 Subject: [PATCH 03/45] searching by gID searching by gID extracted from the title name (when for example downloaded from H@H), will fallback to title name if no gID is found or the search fails due to a false match. --- lib/LANraragi/Plugin/Metadata/EHentai.pm | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/lib/LANraragi/Plugin/Metadata/EHentai.pm b/lib/LANraragi/Plugin/Metadata/EHentai.pm index 075db4160..464a4ed41 100644 --- a/lib/LANraragi/Plugin/Metadata/EHentai.pm +++ b/lib/LANraragi/Plugin/Metadata/EHentai.pm @@ -34,6 +34,7 @@ sub plugin_info { { type => "string", desc => "Forced language to use in searches (Japanese won't work due to EH limitations)" }, { type => "bool", desc => "Save archive title" }, { type => "bool", desc => "Fetch using thumbnail first (falls back to title)" }, + { type => "bool", desc => "Search using gID from title (falls back to title)" }, { type => "bool", desc => "Use ExHentai (enable to search for fjorded content without star cookie)" }, { type => "bool", desc => "Save the original title when available instead of the English or romanised title" @@ -54,7 +55,7 @@ sub get_tags { shift; my $lrr_info = shift; # Global info hash my $ua = $lrr_info->{user_agent}; - my ( $lang, $savetitle, $usethumbs, $enablepanda, $jpntitle, $additionaltags, $expunged ) = @_; # Plugin parameters + my ( $lang, $savetitle, $usethumbs, $search_gid, $enablepanda, $jpntitle, $additionaltags, $expunged ) = @_; # Plugin parameters # Use the logger to output status - they'll be passed to a specialized logfile and written to STDOUT. my $logger = get_plugin_logger(); @@ -82,7 +83,7 @@ sub get_tags { $lrr_info->{archive_title}, $lrr_info->{existing_tags}, $lrr_info->{thumbnail_hash}, - $ua, $domain, $lang, $usethumbs, $expunged + $ua, $domain, $lang, $usethumbs, $search_gid, $expunged ); } @@ -122,7 +123,7 @@ sub get_tags { sub lookup_gallery { - my ( $title, $tags, $thumbhash, $ua, $domain, $defaultlanguage, $usethumbs, $expunged ) = @_; + my ( $title, $tags, $thumbhash, $ua, $domain, $defaultlanguage, $usethumbs, $search_gid, $expunged ) = @_; my $logger = get_plugin_logger(); my $URL = ""; @@ -147,6 +148,23 @@ sub lookup_gallery { } } + # Search using gID if present in title name + my ( $title_gid ) = $title =~ /\[([0-9]+)\]/g; + if ( $search_gid && $title_gid ) { + $URL = + $domain + . "?f_search=" + . uri_escape_utf8("gid:$title_gid"); + + $logger->debug("Found gID: $title_gid, Using URL $URL (gID from archive title)"); + + my ( $gId, $gToken ) = &ehentai_parse( $URL, $ua ); + + if ( $gId ne "" && $gToken ne "" ) { + return ( $gId, $gToken ); + } + } + # Regular text search (advanced options: Disable default filters for: Language, Uploader, Tags) $URL = $domain @@ -168,7 +186,7 @@ sub lookup_gallery { } # Search expunged galleries if the option is enabled. - if ($expunged) { + if ( $expunged ) { $URL = $URL . "&f_sh=on"; } From 43ebf9e1ad47612dab4cc1f6ae18cf2db3585e94 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 14 Nov 2022 18:08:32 +0100 Subject: [PATCH 04/45] (#396) Allow most common http methods when CORS is enabled --- lib/LANraragi/Controller/Login.pm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/LANraragi/Controller/Login.pm b/lib/LANraragi/Controller/Login.pm index c4250fce2..39c9726c3 100644 --- a/lib/LANraragi/Controller/Login.pm +++ b/lib/LANraragi/Controller/Login.pm @@ -73,8 +73,9 @@ sub logged_in_api { sub setup_cors { my $self = shift; - # Set Allow-Origin to wildcard - $self->res->headers->header( 'Access-Control-Allow-Origin' => '*' ); + # Set Allow-Origin to wildcard and Allow-Methods to most common ones + $self->res->headers->header( 'Access-Control-Allow-Origin' => '*' ); + $self->res->headers->header( 'Access-Control-Allow-Methods' => 'GET, OPTIONS, POST, DELETE, PUT' ); # Explicitly say requests with an Authorization header (private API requests) are allowed $self->res->headers->header( 'Access-Control-Allow-Headers' => 'Authorization' ); From 5663a7cc57cb8acbbfcb6b1b841a7f143f4099d5 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sat, 19 Nov 2022 23:31:45 +0100 Subject: [PATCH 05/45] (#706) Bump Karen ref --- tools/build/windows/Karen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build/windows/Karen b/tools/build/windows/Karen index eb529fc43..da59b4a6c 160000 --- a/tools/build/windows/Karen +++ b/tools/build/windows/Karen @@ -1 +1 @@ -Subproject commit eb529fc43b3ef6b8d46b0b2447c05eef09b01df8 +Subproject commit da59b4a6c8c5b5b478b1f624419fea323a77dd69 From 209fcb851a78e84cb3b0067bc16e676062273e39 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sat, 19 Nov 2022 23:55:49 +0100 Subject: [PATCH 06/45] Properly detect "file not found" errors in the Windows wrapper app --- tools/build/windows/Karen | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build/windows/Karen b/tools/build/windows/Karen index da59b4a6c..6a63e5084 160000 --- a/tools/build/windows/Karen +++ b/tools/build/windows/Karen @@ -1 +1 @@ -Subproject commit da59b4a6c8c5b5b478b1f624419fea323a77dd69 +Subproject commit 6a63e508499effed0a680e3984d96d8427383881 From f45d38decc1c76f13e7722e0eef41176bc04ee16 Mon Sep 17 00:00:00 2001 From: zizzdog <120191931+zizzdog@users.noreply.github.com> Date: Sat, 10 Dec 2022 07:19:10 +0800 Subject: [PATCH 07/45] add igneous at cookie (#720) Fixes the problem of using Ehentai login plug-ins in non European and American regions --- lib/LANraragi/Plugin/Login/EHentai.pm | 31 +++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/lib/LANraragi/Plugin/Login/EHentai.pm b/lib/LANraragi/Plugin/Login/EHentai.pm index 6a539ecd4..499b2a17a 100644 --- a/lib/LANraragi/Plugin/Login/EHentai.pm +++ b/lib/LANraragi/Plugin/Login/EHentai.pm @@ -16,13 +16,14 @@ sub plugin_info { type => "login", namespace => "ehlogin", author => "Difegue", - version => "2.2", + version => "2.3", description => "Handles login to E-H. If you have an account that can access fjorded content or exhentai, adding the credentials here will make more archives available for parsing.", parameters => [ { type => "int", desc => "ipb_member_id cookie" }, { type => "string", desc => "ipb_pass_hash cookie" }, - { type => "string", desc => "star cookie (optional, if present you can view fjorded content without exhentai)" } + { type => "string", desc => "star cookie (optional, if present you can view fjorded content without exhentai)" }, + { type => "string", desc => "igneous cookie(optional, if present you can view exhentai without Europe and America IP)" } ] ); @@ -34,8 +35,8 @@ sub do_login { # Login plugins only receive the parameters entered by the user. shift; - my ( $ipb_member_id, $ipb_pass_hash, $star ) = @_; - return get_user_agent( $ipb_member_id, $ipb_pass_hash, $star ); + my ( $ipb_member_id, $ipb_pass_hash, $star ,$igneous ) = @_; + return get_user_agent( $ipb_member_id, $ipb_pass_hash, $star ,$igneous ); } # get_user_agent(ipb cookies) @@ -43,13 +44,13 @@ sub do_login { # Returns the UA object created. sub get_user_agent { - my ( $ipb_member_id, $ipb_pass_hash, $star ) = @_; + my ( $ipb_member_id, $ipb_pass_hash, $star, $igneous ) = @_; my $logger = get_logger( "E-Hentai Login", "plugins" ); my $ua = Mojo::UserAgent->new; if ( $ipb_member_id ne "" && $ipb_pass_hash ne "" ) { - $logger->info("Cookies provided ($ipb_member_id $ipb_pass_hash $star)!"); + $logger->info("Cookies provided ($ipb_member_id $ipb_pass_hash $star $igneous)!"); #Setup the needed cookies with both domains #They should translate to exhentai cookies with the igneous value generated @@ -98,6 +99,15 @@ sub get_user_agent { ) ); + $ua->cookie_jar->add( + Mojo::Cookie::Response->new( + name => 'igneous', + value => $igneous, + domain => 'exhentai.org', + path => '/' + ) + ); + $ua->cookie_jar->add( Mojo::Cookie::Response->new( name => 'star', @@ -107,6 +117,15 @@ sub get_user_agent { ) ); + $ua->cookie_jar->add( + Mojo::Cookie::Response->new( + name => 'igneous', + value => $igneous, + domain => 'e-hentai.org', + path => '/' + ) + ); + $ua->cookie_jar->add( Mojo::Cookie::Response->new( name => 'ipb_coppa', From 665ecaf493e2681898aeea756d6ca3631943f09b Mon Sep 17 00:00:00 2001 From: Difegue Date: Sat, 10 Dec 2022 01:52:08 +0100 Subject: [PATCH 08/45] Add basic Redis password support --- lib/LANraragi/Model/Config.pm | 11 ++++++++++- lrr.conf | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/LANraragi/Model/Config.pm b/lib/LANraragi/Model/Config.pm index 48991c031..0c3b3b0e1 100644 --- a/lib/LANraragi/Model/Config.pm +++ b/lib/LANraragi/Model/Config.pm @@ -20,6 +20,9 @@ my $config = Mojolicious::Plugin::Config->register( Mojolicious->new, { file => # Address and port of your redis instance. sub get_redisad { return $config->{redis_address} } +# Optional password of your redis instance. +sub get_redispassword { return $config->{redis_address} } + # Database that'll be used by LANraragi. Redis databases are numbered, default is 0. sub get_redisdb { return $config->{redis_database} } @@ -29,7 +32,12 @@ sub get_miniondb { return $config->{redis_database_minion} } # Create a Minion object connected to the Minion database. sub get_minion { my $miniondb = get_redisad . "/" . get_miniondb; - return Minion->new( Redis => "redis://$miniondb" ); + my $password = get_redispassword; + + # If the password is non-empty, add the required @ + if ($password) { $password = $password . "@"; } + + return Minion->new( Redis => "redis://$password$miniondb" ); } #get_redis @@ -40,6 +48,7 @@ sub get_redis { #Auto-reconnect on, one attempt every 2ms up to 3 seconds. Die after that. my $redis = Redis->new( server => &get_redisad, + password => &get_redispssword, reconnect => 3 ); diff --git a/lrr.conf b/lrr.conf index 76efd38da..85276924f 100644 --- a/lrr.conf +++ b/lrr.conf @@ -1,5 +1,6 @@ { redis_address => "127.0.0.1:6379", + redis_password => "", redis_database => "0", redis_database_minion => "1" } From 9bf09e1a95d988a35f5deb06aa591d85ce32f806 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 11 Dec 2022 02:30:02 +0100 Subject: [PATCH 09/45] (#555) Calculate tag indexes and spec out new DB layout --- lib/LANraragi/Model/Config.pm | 37 ++++++++++++++----- lib/LANraragi/Model/Stats.pm | 23 ++++++++---- lib/LANraragi/Utils/Database.pm | 1 + lrr.conf | 4 +- .../extending-lanraragi/architecture.md | 25 +++++++++++-- 5 files changed, 70 insertions(+), 20 deletions(-) diff --git a/lib/LANraragi/Model/Config.pm b/lib/LANraragi/Model/Config.pm index 0c3b3b0e1..8c591937e 100644 --- a/lib/LANraragi/Model/Config.pm +++ b/lib/LANraragi/Model/Config.pm @@ -23,12 +23,20 @@ sub get_redisad { return $config->{redis_address} } # Optional password of your redis instance. sub get_redispassword { return $config->{redis_address} } -# Database that'll be used by LANraragi. Redis databases are numbered, default is 0. -sub get_redisdb { return $config->{redis_database} } +# LANraragi uses 4 Redis Databases. Redis databases are numbered, default is 0. -# Database that'll be used by Minion. Redis databases are numbered, default is 1. +# Database used for archive data and tag indexes +sub get_archivedb { return $config->{redis_database} } + +# Database used by Minion sub get_miniondb { return $config->{redis_database_minion} } +# Database used to store config keys +sub get_configdb { return $config->{redis_database_config} } + +# Database used to store search index and cache +sub get_searchdb { return $config->{redis_database_search} } + # Create a Minion object connected to the Minion database. sub get_minion { my $miniondb = get_redisad . "/" . get_miniondb; @@ -40,21 +48,32 @@ sub get_minion { return Minion->new( Redis => "redis://$password$miniondb" ); } -#get_redis -#Create a redis object with the parameters defined at the start of this file and return it sub get_redis { + return get_redis_internal(&get_archivedb); +} + +sub get_redis_config { + return get_redis_internal(&get_configdb); +} + +sub get_redis_search { + return get_redis_internal(&get_searchdb); +} + +sub get_redis_internal { + + my $db = $_[0]; #Default redis server location is localhost:6379. #Auto-reconnect on, one attempt every 2ms up to 3 seconds. Die after that. my $redis = Redis->new( server => &get_redisad, - password => &get_redispssword, + password => &get_redispassword, reconnect => 3 ); - #Database switch if it's not 0 - if ( &get_redisdb != 0 ) { $redis->select(&get_redisdb); } - + #Database switch + $redis->select($db); return $redis; } diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index be42d2fe1..0c6cb04a5 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -46,12 +46,14 @@ sub get_page_stat { # This operation builds two hashes: LRR_URL_MAP, which maps URLs to IDs in the database that have them as a source: tag, # and LRR_STATS, which is a sorted set used to build the statistics/tag cloud JSON. +# It also builds the sets for each distinct tag. sub build_stat_hashes { -# This method does only one atomic write transaction, using Redis' watch/multi mode. -# But we can't use the connection to get other data while it's in transaction mode! So we instantiate a second connection to get the data we need. + # This method does only one atomic write transaction, using Redis' watch/multi mode. + # But we can't use the connection to get other data while it's in transaction mode! + # So we instantiate a second connection to get the data we need. Helps as well now that both connections are made on separate DBs. my $redis = LANraragi::Model::Config->get_redis; - my $redistx = LANraragi::Model::Config->get_redis; + my $redistx = LANraragi::Model::Config->get_redis_search; my $logger = get_logger( "Tag Stats", "lanraragi" ); # 40-character long keys only => Archive IDs @@ -61,8 +63,9 @@ sub build_stat_hashes { # This also allows for the previous stats/map to still be readable until we're done. $redistx->watch( "LRR_STATS", "LRR_URLMAP" ); $redistx->multi; - $redistx->del("LRR_STATS"); - $redistx->del("LRR_URLMAP"); + + # Hose the entire index DB since we're rebuilding it + $redistx->flushdb(); # Iterate on hashes to get their tags $logger->debug("Building stat indexes..."); @@ -86,8 +89,14 @@ sub build_stat_hashes { $redistx->hset( "LRR_URLMAP", $url, $id ); # No need to encode the value, as URLs are already encoded by design } - # Increment tag in stats, all lowercased here to avoid redundancy/dupes - $redistx->zincrby( "LRR_STATS", 1, redis_encode( lc($t) ) ); + # Tag is lowercased here to avoid redundancy/dupes + my $redis_tag = redis_encode( lc($t) ); + + # Increment tag in stats, + $redistx->zincrby( "LRR_STATS", 1, $redis_tag ); + + # Add the archive ID to the set for this tag + $redistx->sadd( $redis_tag, $id ); } } } diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index bc0ea07ea..c065533f5 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -225,6 +225,7 @@ sub delete_archive { # drop_database() # Drops the entire database. Hella dangerous +# TODO: Might be worth it to add versions that only do flushdb on certain databases like the config/archive data one? sub drop_database { my $redis = LANraragi::Model::Config->get_redis; diff --git a/lrr.conf b/lrr.conf index 85276924f..e794e059a 100644 --- a/lrr.conf +++ b/lrr.conf @@ -2,5 +2,7 @@ redis_address => "127.0.0.1:6379", redis_password => "", redis_database => "0", - redis_database_minion => "1" + redis_database_minion => "1", + redis_database_config => "2", + redis_database_search => "3", } diff --git a/tools/Documentation/extending-lanraragi/architecture.md b/tools/Documentation/extending-lanraragi/architecture.md index 97dbed062..dfa6e70cb 100644 --- a/tools/Documentation/extending-lanraragi/architecture.md +++ b/tools/Documentation/extending-lanraragi/architecture.md @@ -154,19 +154,28 @@ In Debug Mode: ## Database Architecture -You can look inside the LRR Redis database at any moment through the `redis-cli` tool. +You can look inside the LRR Redis databases at any moment through the `redis-cli` tool. + +LRR uses three databases to store its own data, and a fourth for the Minion Job Queue. The base architecture is as follows: ``` --Redis Database +-Redis Database 1 - Archive data +| |- **************************************** <- 40-character long ID for every logged archive | |- tags <- Saved tags | |- name <- Name of the archive file, kept for filesystem checks | |- title <- Title of the archive, as set by the User | |- file <- Filesystem path to archive | |- isnew <- Whether the archive has been opened in LRR once or not +| |- filesizes <- Size in bytes of each file in the archive, as a serialized JSON array. This value is only present if the archive has been opened once for reading. +| |- pagecount <- Number of pages of the archive file +| |- progress <- Reading progress, if server-side progress is enabled | +- thumbhash <- SHA-1 hash of the first image of the archive ++ + +-Redis Database 2 - Configuration | |- LRR_PLUGIN_xxxxxxx <- Settings for a plugin with namespace xxxxxxx | @@ -194,11 +203,21 @@ The base architecture is as follows: | |- pagesize <- Amount of archives per Index page | +- apikey <- Key for API requests | -|- LRR_TAGRULES <- Computed Tag Rules, as a Redis list ++- LRR_TAGRULES <- Computed Tag Rules, as a Redis list + + +-Redis Database 3 - Search indexes +| +|- LRR_URLMAP <- Maps archive IDs to their source: tag, used by the Downloader system. +| +|- LRR_STATS <- Redis sorted set used to build the statistics/tag cloud JSON. +| +|- ***:**** <- Each namespaced tag has a matching Redis set, with all the IDs that have this tag in their metadata. This is used for search indexing. | +- LRR_SEARCHCACHE <- Search Cache |- $columnfilter-$filter-$sortkey-$sortorder-$newonly <- Unique ID for a search. The search result is serialized and saved as the value for this ID. +- --title-asc-0 <- Example ID for a search made on titles with no filters. + ``` {% hint style="info" %} From f818fc13c8005c2a77e5e49d1f376796c211c7fa Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 11 Dec 2022 02:38:00 +0100 Subject: [PATCH 10/45] Fix passwording --- lib/LANraragi/Model/Config.pm | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/LANraragi/Model/Config.pm b/lib/LANraragi/Model/Config.pm index 8c591937e..2b7500874 100644 --- a/lib/LANraragi/Model/Config.pm +++ b/lib/LANraragi/Model/Config.pm @@ -21,7 +21,7 @@ my $config = Mojolicious::Plugin::Config->register( Mojolicious->new, { file => sub get_redisad { return $config->{redis_address} } # Optional password of your redis instance. -sub get_redispassword { return $config->{redis_address} } +sub get_redispassword { return $config->{redis_password} } # LANraragi uses 4 Redis Databases. Redis databases are numbered, default is 0. @@ -64,15 +64,19 @@ sub get_redis_internal { my $db = $_[0]; - #Default redis server location is localhost:6379. - #Auto-reconnect on, one attempt every 2ms up to 3 seconds. Die after that. + # Default redis server location is localhost:6379. + # Auto-reconnect on, one attempt every 2ms up to 3 seconds. Die after that. my $redis = Redis->new( server => &get_redisad, - password => &get_redispassword, reconnect => 3 ); - #Database switch + # Auth if password is set + if ( &get_redispassword ne "" ) { + $redis->auth(&get_redispassword); + } + + # Switch to specced database $redis->select($db); return $redis; } From 426c27e41b3f802a9d7c898d21e13ad9c31c8ea4 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 11 Dec 2022 03:03:49 +0100 Subject: [PATCH 11/45] Some cache changes + refactor add_tags to be more generic so we can use it everywhere --- lib/LANraragi.pm | 1 - lib/LANraragi/Controller/Api/Database.pm | 4 +-- lib/LANraragi/Controller/Batch.pm | 7 +++-- lib/LANraragi/Model/Backup.pm | 2 +- lib/LANraragi/Model/Plugins.pm | 2 +- lib/LANraragi/Utils/Database.pm | 40 ++++++++++++++---------- 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/lib/LANraragi.pm b/lib/LANraragi.pm index e643ec545..886c1953e 100644 --- a/lib/LANraragi.pm +++ b/lib/LANraragi.pm @@ -14,7 +14,6 @@ use Config; use LANraragi::Utils::Generic qw(start_shinobu start_minion); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Plugins qw(get_plugins); -use LANraragi::Utils::Database qw(invalidate_cache); use LANraragi::Utils::TempFolder qw(get_temp); use LANraragi::Utils::Routing; use LANraragi::Utils::Minion; diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index 428e3c7ec..af2f7cd7a 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -56,8 +56,8 @@ sub clear_new_all { $redis->hset( $idall, "isnew", "false" ); } - # Bust search cache completely, this is a big change - invalidate_cache(1); + # Bust isnew cache + invalidate_isnew_cache(); $redis->quit(); render_api_response( $self, "clear_new_all" ); } diff --git a/lib/LANraragi/Controller/Batch.pm b/lib/LANraragi/Controller/Batch.pm index 0ca42a912..ab0593667 100644 --- a/lib/LANraragi/Controller/Batch.pm +++ b/lib/LANraragi/Controller/Batch.pm @@ -7,7 +7,7 @@ use Mojo::JSON qw(decode_json); use LANraragi::Utils::Generic qw(generate_themes_header); use LANraragi::Utils::Tags qw(rewrite_tags split_tags_to_array restore_CRLF); -use LANraragi::Utils::Database qw(get_computed_tagrules); +use LANraragi::Utils::Database qw(get_computed_tagrules invalidate_cache); use LANraragi::Utils::Plugins qw(get_plugins get_plugin get_plugin_parameters); use LANraragi::Utils::Logging qw(get_logger); @@ -138,6 +138,9 @@ sub socket { } } ); + + invalidate_cache(); + return; } @@ -195,7 +198,7 @@ sub batch_plugin { # If the plugin exec returned tags, add them unless ( exists $plugin_result{error} ) { - LANraragi::Utils::Database::add_tags( $id, $plugin_result{new_tags} ); + LANraragi::Utils::Database::set_tags( $id, $plugin_result{new_tags}, 1 ); if ( exists $plugin_result{title} ) { LANraragi::Utils::Database::set_title( $id, $plugin_result{title} ); diff --git a/lib/LANraragi/Model/Backup.pm b/lib/LANraragi/Model/Backup.pm index 0764046bd..309f1c54b 100644 --- a/lib/LANraragi/Model/Backup.pm +++ b/lib/LANraragi/Model/Backup.pm @@ -137,7 +137,7 @@ sub restore_from_JSON { } } - #Force a refresh + # Force a refresh invalidate_cache(1); $redis->quit(); } diff --git a/lib/LANraragi/Model/Plugins.pm b/lib/LANraragi/Model/Plugins.pm index e6775a5f3..635f23c80 100644 --- a/lib/LANraragi/Model/Plugins.pm +++ b/lib/LANraragi/Model/Plugins.pm @@ -66,7 +66,7 @@ sub exec_enabled_plugins_on_file { #If the plugin exec returned metadata, add it unless ( exists $plugin_result{error} ) { - LANraragi::Utils::Database::add_tags( $id, $plugin_result{new_tags} ); + LANraragi::Utils::Database::set_tags( $id, $plugin_result{new_tags}, 1 ); # Sum up all the added tags for later reporting. # This doesn't take into account tags that are added twice diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index c065533f5..334238c88 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -312,17 +312,32 @@ sub clean_database { return ( $deleted_arcs, $unlinked_arcs ); } -#add_tags($id, $tags) -#add the $tags to the archive with id $id. -sub add_tags { +sub set_title { + + my ( $id, $newtitle ) = @_; + my $redis = LANraragi::Model::Config->get_redis; + + if ( $newtitle ne "" ) { + $redis->hset( $id, "title", redis_encode($newtitle) ); + } + $redis->quit; +} + +#set_tags($id, $tags, $append) +# Set $tags for the archive with id $id. +# Set $append to 1 if you want to append the tags instead of replacing them. +sub set_tags { - my ( $id, $newtags ) = @_; + my ( $id, $newtags, $append ) = @_; my $redis = LANraragi::Model::Config->get_redis; my $oldtags = $redis->hget( $id, "tags" ); $oldtags = redis_decode($oldtags); - if ( length $newtags ) { + if ($append) { + + # If the new tags are empty, don't do anything + unless ( length $newtags ) return; if ($oldtags) { remove_spaces($oldtags); @@ -331,21 +346,14 @@ sub add_tags { $newtags = $oldtags . "," . $newtags; } } - - $redis->hset( $id, "tags", redis_encode($newtags) ); } - $redis->quit; -} -sub set_title { + # TODO: Update sets depending on the added/removed tags - my ( $id, $newtitle ) = @_; - my $redis = LANraragi::Model::Config->get_redis; - - if ( $newtitle ne "" ) { - $redis->hset( $id, "title", redis_encode($newtitle) ); - } + $redis->hset( $id, "tags", redis_encode($newtags) ); $redis->quit; + + invalidate_cache(); } #This function is used for all ID computation in LRR. From 096c41554c5a86048540335a1dbcf5fe2f58fa6d Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 11 Dec 2022 23:30:59 +0100 Subject: [PATCH 12/45] Refactor to use set_tags and set_title everywhere --- lib/LANraragi/Controller/Batch.pm | 8 ++--- lib/LANraragi/Model/Archive.pm | 18 +++-------- lib/LANraragi/Model/Backup.pm | 8 ++--- lib/LANraragi/Model/Plugins.pm | 5 +-- .../Plugin/Scripts/nHentaiSourceConverter.pm | 20 ++++++------ lib/LANraragi/Utils/Database.pm | 32 +++++++++++++++++-- 6 files changed, 54 insertions(+), 37 deletions(-) diff --git a/lib/LANraragi/Controller/Batch.pm b/lib/LANraragi/Controller/Batch.pm index ab0593667..f3d5a3c6b 100644 --- a/lib/LANraragi/Controller/Batch.pm +++ b/lib/LANraragi/Controller/Batch.pm @@ -7,7 +7,7 @@ use Mojo::JSON qw(decode_json); use LANraragi::Utils::Generic qw(generate_themes_header); use LANraragi::Utils::Tags qw(rewrite_tags split_tags_to_array restore_CRLF); -use LANraragi::Utils::Database qw(get_computed_tagrules invalidate_cache); +use LANraragi::Utils::Database qw(get_computed_tagrules set_tags set_title invalidate_cache); use LANraragi::Utils::Plugins qw(get_plugins get_plugin get_plugin_parameters); use LANraragi::Utils::Logging qw(get_logger); @@ -128,7 +128,7 @@ sub socket { # Merge array with commas my $newtags = join( ', ', @tagarray ); $logger->debug("New tags: $newtags"); - $redis->hset( $id, "tags", $newtags ); + set_tags( $id, $newtags ); $client->send( { json => { @@ -198,10 +198,10 @@ sub batch_plugin { # If the plugin exec returned tags, add them unless ( exists $plugin_result{error} ) { - LANraragi::Utils::Database::set_tags( $id, $plugin_result{new_tags}, 1 ); + set_tags( $id, $plugin_result{new_tags}, 1 ); if ( exists $plugin_result{title} ) { - LANraragi::Utils::Database::set_title( $id, $plugin_result{title} ); + set_title( $id, $plugin_result{title} ); } } diff --git a/lib/LANraragi/Model/Archive.pm b/lib/LANraragi/Model/Archive.pm index 64043af48..f6f00a298 100644 --- a/lib/LANraragi/Model/Archive.pm +++ b/lib/LANraragi/Model/Archive.pm @@ -16,7 +16,8 @@ use LANraragi::Utils::Generic qw(get_tag_with_namespace remove_spaces remove_new use LANraragi::Utils::TempFolder qw(get_temp); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Archive qw(extract_single_file extract_thumbnail); -use LANraragi::Utils::Database qw(redis_encode redis_decode invalidate_cache get_archive_json get_archive_json_multi); +use LANraragi::Utils::Database + qw(redis_encode redis_decode invalidate_cache set_title set_tags get_archive_json get_archive_json_multi); # Functions used when dealing with archives. @@ -335,25 +336,14 @@ sub update_metadata { ( remove_spaces($_) ) for ( $title, $tags ); ( remove_newlines($_) ) for ( $title, $tags ); - # Input new values into redis hash. - my %hash; - - # Prepare the hash which'll be inserted. if ( defined $title ) { - $hash{title} = redis_encode($title); + set_title( $id, $title ); } if ( defined $tags ) { - $hash{tags} = redis_encode($tags); + set_tags( $id, $tags ); } - my $redis = LANraragi::Model::Config->get_redis; - - # For all keys of the hash, add them to the redis hash $id with the matching keys. - $redis->hset( $id, $_, $hash{$_}, sub { } ) for keys %hash; - $redis->wait_all_responses; - $redis->quit(); - # Bust cache invalidate_cache(1); diff --git a/lib/LANraragi/Model/Backup.pm b/lib/LANraragi/Model/Backup.pm index 309f1c54b..4e325376a 100644 --- a/lib/LANraragi/Model/Backup.pm +++ b/lib/LANraragi/Model/Backup.pm @@ -10,7 +10,7 @@ use Mojo::JSON qw(decode_json encode_json); use LANraragi::Model::Category; use LANraragi::Utils::Database; use LANraragi::Utils::Generic qw(remove_newlines); -use LANraragi::Utils::Database qw(redis_encode redis_decode invalidate_cache); +use LANraragi::Utils::Database qw(redis_encode redis_decode invalidate_cache set_title set_tags); use LANraragi::Utils::Logging qw(get_logger); #build_backup_JSON() @@ -122,12 +122,10 @@ sub restore_from_JSON { if ( $redis->exists($id) ) { $logger->info("Restoring metadata for Archive $id..."); - my $title = redis_encode( $archive->{"title"} ); - my $tags = redis_encode( $archive->{"tags"} ); my $thumbhash = redis_encode( $archive->{"thumbhash"} ); - $redis->hset( $id, "title", $title ); - $redis->hset( $id, "tags", $tags ); + set_title( $id, $archive->{"title"} ); + set_tags( $id, $archive->{"tags"} ); if ( $redis->hexists( $id, "thumbhash" ) && $redis->hget( $id, "thumbhash" ) ne "" ) { diff --git a/lib/LANraragi/Model/Plugins.pm b/lib/LANraragi/Model/Plugins.pm index 635f23c80..e25480be2 100644 --- a/lib/LANraragi/Model/Plugins.pm +++ b/lib/LANraragi/Model/Plugins.pm @@ -12,6 +12,7 @@ use Mojo::UserAgent; use Data::Dumper; use LANraragi::Utils::Generic qw(remove_spaces remove_newlines); +use LANraragi::Utils::Database qw(set_tags set_title); use LANraragi::Utils::Archive qw(extract_thumbnail); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Tags qw(rewrite_tags split_tags_to_array); @@ -66,7 +67,7 @@ sub exec_enabled_plugins_on_file { #If the plugin exec returned metadata, add it unless ( exists $plugin_result{error} ) { - LANraragi::Utils::Database::set_tags( $id, $plugin_result{new_tags}, 1 ); + set_tags( $id, $plugin_result{new_tags}, 1 ); # Sum up all the added tags for later reporting. # This doesn't take into account tags that are added twice @@ -76,7 +77,7 @@ sub exec_enabled_plugins_on_file { $addedtags += @added_tags; if ( exists $plugin_result{title} ) { - LANraragi::Utils::Database::set_title( $id, $plugin_result{title} ); + set_title( $id, $plugin_result{title} ); $newtitle = $plugin_result{title}; $logger->debug("Changing title to $newtitle."); diff --git a/lib/LANraragi/Plugin/Scripts/nHentaiSourceConverter.pm b/lib/LANraragi/Plugin/Scripts/nHentaiSourceConverter.pm index a11db37c7..9ead98359 100644 --- a/lib/LANraragi/Plugin/Scripts/nHentaiSourceConverter.pm +++ b/lib/LANraragi/Plugin/Scripts/nHentaiSourceConverter.pm @@ -5,7 +5,7 @@ use warnings; no warnings 'uninitialized'; use LANraragi::Utils::Logging qw(get_plugin_logger); -use LANraragi::Utils::Database qw(invalidate_cache); +use LANraragi::Utils::Database qw(invalidate_cache set_tags); use LANraragi::Model::Config; #Meta-information about your plugin. @@ -13,13 +13,12 @@ sub plugin_info { return ( #Standard metadata - name => "nHentai Source Converter", - type => "script", - namespace => "nhsrcconv", - author => "Guerra24", - version => "1.0", - description => - "Converts \"source:{id}\" tags with 6 or less digits into \"source:nhentai.net/g/{id}\"" + name => "nHentai Source Converter", + type => "script", + namespace => "nhsrcconv", + author => "Guerra24", + version => "1.0", + description => "Converts \"source:{id}\" tags with 6 or less digits into \"source:nhentai.net/g/{id}\"" ); } @@ -35,17 +34,18 @@ sub run_script { my @keys = $redis->keys('????????????????????????????????????????'); #40-character long keys only => Archive IDs my $count = 0; + #Parse the archive list and add them to JSON. foreach my $id (@keys) { my %hash = $redis->hgetall($id); - my ( $tags ) = @hash{qw(tags)}; + my ($tags) = @hash{qw(tags)}; if ( $tags =~ s/source:(\d{1,6})/source:nhentai\.net\/g\/$1/igm ) { $count++; } - $redis->hset( $id, "tags", $tags ); + set_tags( $id, $tags ); } invalidate_cache(); diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index 334238c88..d5c53697f 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -21,7 +21,7 @@ use LANraragi::Utils::Logging qw(get_logger); # Functions for interacting with the DB Model. use Exporter 'import'; our @EXPORT_OK = - qw(redis_encode redis_decode invalidate_cache compute_id get_computed_tagrules save_computed_tagrules get_archive_json get_archive_json_multi); + qw(redis_encode redis_decode invalidate_cache compute_id set_tags set_title get_computed_tagrules save_computed_tagrules get_archive_json get_archive_json_multi); #add_archive_to_redis($id,$file,$redis) # Creates a DB entry for a file path with the given ID. @@ -348,7 +348,8 @@ sub set_tags { } } - # TODO: Update sets depending on the added/removed tags + # Update sets depending on the added/removed tags + update_indexes( $id, $oldtags, $newtags ); $redis->hset( $id, "tags", redis_encode($newtags) ); $redis->quit; @@ -356,6 +357,33 @@ sub set_tags { invalidate_cache(); } +# Splits both old and new tags, and: +# Removes the ID from all sets of the old tags +# Adds it back to all sets of the new tags. +sub update_indexes { + + my ( $id, $oldtags, $newtags ) = @_; + + my $redis = LANraragi::Model::Config->get_redis_search; + $redis->multi; + + my @oldtags = split( ',', $oldtags ); + my @newtags = split( ',', $newtags ); + + foreach my $tag (@oldtags) { + + # Tag is lowercased here to avoid redundancy/dupes + $redis->srem( redis_encode( lc($tag) ), $id ); + } + + foreach my $tag (@newtags) { + $redis->sadd( redis_encode( lc($tag) ), $id ); + } + + $redis->exec; + $redis->quit; +} + #This function is used for all ID computation in LRR. #Takes the path to the file as an argument. sub compute_id { From b3bff4ad625c8fd52ceb115e7e64685cdaf6121c Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 11 Dec 2022 23:42:02 +0100 Subject: [PATCH 13/45] Login now redirects you to the page you tried to access beforehand --- lib/LANraragi/Controller/Login.pm | 9 ++++++--- lib/LANraragi/Model/Stats.pm | 2 +- lib/LANraragi/Utils/Database.pm | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/LANraragi/Controller/Login.pm b/lib/LANraragi/Controller/Login.pm index 39c9726c3..f628168e1 100644 --- a/lib/LANraragi/Controller/Login.pm +++ b/lib/LANraragi/Controller/Login.pm @@ -10,7 +10,8 @@ use LANraragi::Utils::Generic qw(generate_themes_header); sub check { my $self = shift; - my $pw = $self->req->param('password') || ''; + my $pw = $self->req->param('password') || ''; + my $redirect = $self->req->param('redirect') || 'index'; #match password we got with the authen hash stored in redis my $ppr = Authen::Passphrase->from_rfc2307( $self->LRR_CONF->get_password ); @@ -21,7 +22,7 @@ sub check { $self->session( is_logged => 1 ); $self->session( expiration => 60 * 60 * 24 ); - $self->redirect_to('index'); + $self->redirect_to($redirect); } else { $self->LRR_LOGGER->warn( "Failed login attempt with password '$pw' from " . $self->tx->remote_address ); @@ -45,7 +46,9 @@ sub logged_in { return 1 if $self->session('is_logged') || $self->LRR_CONF->enable_pass == 0; - $self->redirect_to('login'); + + my $url = $self->url_for("login"); + $self->redirect_to( $url->query( redirect => $self->req->url->path ) ); return 0; } diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index 0c6cb04a5..6e3a3549f 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -46,7 +46,7 @@ sub get_page_stat { # This operation builds two hashes: LRR_URL_MAP, which maps URLs to IDs in the database that have them as a source: tag, # and LRR_STATS, which is a sorted set used to build the statistics/tag cloud JSON. -# It also builds the sets for each distinct tag. +# It also builds the index sets for each distinct tag. sub build_stat_hashes { # This method does only one atomic write transaction, using Redis' watch/multi mode. diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index d5c53697f..809a5e9ed 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -337,7 +337,7 @@ sub set_tags { if ($append) { # If the new tags are empty, don't do anything - unless ( length $newtags ) return; + unless ( length $newtags ) { return; } if ($oldtags) { remove_spaces($oldtags); From ebdad33cfa24dc2b3a39ef851306fca8fd3dd2ca Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 12 Dec 2022 00:10:01 +0100 Subject: [PATCH 14/45] Use get_redis_search for LRR_URLMAP and LRR_STATS as well --- lib/LANraragi/Model/Stats.pm | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index 6e3a3549f..4ffc5d4c7 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -111,7 +111,7 @@ sub is_url_recorded { my $url = $_[0]; my $logger = get_logger( "Tag Stats", "lanraragi" ); - my $redis = LANraragi::Model::Config->get_redis; + my $redis = LANraragi::Model::Config->get_redis_search; my $id = 0; $logger->debug("Checking if url $url is in the url map."); @@ -133,7 +133,7 @@ sub build_tag_stats { $logger->debug("Serving tag statistics with a minimum weight of $minscore"); # Login to Redis and grab the stats sorted set - my $redis = LANraragi::Model::Config->get_redis; + my $redis = LANraragi::Model::Config->get_redis_search; my %tagcloud = $redis->zrangebyscore( "LRR_STATS", $minscore, "+inf", "WITHSCORES" ); $redis->quit(); diff --git a/package.json b/package.json index b8503b950..1d5afddc1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "lanraragi", - "version": "0.8.7", - "version_name": "Outside", + "version": "0.8.8", + "version_name": "I'm Deranged", "description": "I'm under Japanese influence and my honor's at stake!", "scripts": { "test": "prove -r -l -v tests/", From 904a329974ec19d183e171a5422df45e6580a365 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 12 Dec 2022 00:24:29 +0100 Subject: [PATCH 15/45] prepend tag indexes with INDEX_ so that everything doesn't blow up if a dingus uses lrr_filemap as a tag or something equally wicked --- lib/LANraragi/Model/Stats.pm | 2 +- lib/LANraragi/Utils/Database.pm | 4 ++-- tools/Documentation/extending-lanraragi/architecture.md | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index 4ffc5d4c7..4343efb44 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -96,7 +96,7 @@ sub build_stat_hashes { $redistx->zincrby( "LRR_STATS", 1, $redis_tag ); # Add the archive ID to the set for this tag - $redistx->sadd( $redis_tag, $id ); + $redistx->sadd( "INDEX_" . $redis_tag, $id ); } } } diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index 809a5e9ed..5046e16f1 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -373,11 +373,11 @@ sub update_indexes { foreach my $tag (@oldtags) { # Tag is lowercased here to avoid redundancy/dupes - $redis->srem( redis_encode( lc($tag) ), $id ); + $redis->srem( "INDEX_" . redis_encode( lc($tag) ), $id ); } foreach my $tag (@newtags) { - $redis->sadd( redis_encode( lc($tag) ), $id ); + $redis->sadd( "INDEX_" . redis_encode( lc($tag) ), $id ); } $redis->exec; diff --git a/tools/Documentation/extending-lanraragi/architecture.md b/tools/Documentation/extending-lanraragi/architecture.md index dfa6e70cb..b76c93120 100644 --- a/tools/Documentation/extending-lanraragi/architecture.md +++ b/tools/Documentation/extending-lanraragi/architecture.md @@ -212,7 +212,9 @@ The base architecture is as follows: | |- LRR_STATS <- Redis sorted set used to build the statistics/tag cloud JSON. | -|- ***:**** <- Each namespaced tag has a matching Redis set, with all the IDs that have this tag in their metadata. This is used for search indexing. +|- LRR_UNTAGGED <- List of archive IDs that don't have any tags (except for tags added automatically by the autotagger) +| +|- INDEX_***:**** <- Each tag(namespaced or not) has a matching Redis set, with all the IDs that have this tag in their metadata. This is used for search indexing. | +- LRR_SEARCHCACHE <- Search Cache |- $columnfilter-$filter-$sortkey-$sortorder-$newonly <- Unique ID for a search. The search result is serialized and saved as the value for this ID. From dfe624d4fb54f1b51445eb26cd0e1869fe0ee40b Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 12 Dec 2022 01:39:23 +0100 Subject: [PATCH 16/45] (#709) Implement LRR_UNTAGGED and LRR_TITLES --- lib/LANraragi/Model/Stats.pm | 37 ++++++++++++++++--- lib/LANraragi/Utils/Database.pm | 35 ++++++++++++++++-- .../extending-lanraragi/architecture.md | 4 +- 3 files changed, 66 insertions(+), 10 deletions(-) diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index 4343efb44..93cbe8863 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -44,9 +44,12 @@ sub get_page_stat { return $stat; } -# This operation builds two hashes: LRR_URL_MAP, which maps URLs to IDs in the database that have them as a source: tag, -# and LRR_STATS, which is a sorted set used to build the statistics/tag cloud JSON. -# It also builds the index sets for each distinct tag. +# This operation builds the following hashes: +# - LRR_URL_MAP, which maps URLs to IDs in the database that have them as a source: tag +# - LRR_STATS, which is a sorted set used to build the statistics/tag cloud JSON +# - LRR_UNTAGGED, which is a set used by the untagged archives API +# - LRR_TITLES, which is a set containing all titles in the DB, alongside their ID. (In the "title\0ID" format) +# * It also builds index sets for each distinct tag. sub build_stat_hashes { # This method does only one atomic write transaction, using Redis' watch/multi mode. @@ -61,7 +64,7 @@ sub build_stat_hashes { # Cancel the transaction if the hashes have been modified by another job in the meantime. # This also allows for the previous stats/map to still be readable until we're done. - $redistx->watch( "LRR_STATS", "LRR_URLMAP" ); + $redistx->watch( "LRR_STATS", "LRR_URLMAP", "LRR_UNTAGGED", "LRR_TITLES" ); $redistx->multi; # Hose the entire index DB since we're rebuilding it @@ -74,13 +77,17 @@ sub build_stat_hashes { my $rawtags = $redis->hget( $id, "tags" ); - #Split tags by comma + # Split tags by comma my @tags = split( /,\s?/, redis_decode($rawtags) ); + my $has_tags = 0; foreach my $t (@tags) { remove_spaces($t); remove_newlines($t); + # The following are basic and therefore don't count as "tagged" + $has_tags = 1 unless $t =~ /(artist|parody|series|language|event|group|date_added|timestamp):.*/; + # If the tag is a source: tag, add it to the URL index if ( $t =~ /source:(.*)/i ) { my $url = $1; @@ -92,12 +99,30 @@ sub build_stat_hashes { # Tag is lowercased here to avoid redundancy/dupes my $redis_tag = redis_encode( lc($t) ); - # Increment tag in stats, + # Increment tag in stats $redistx->zincrby( "LRR_STATS", 1, $redis_tag ); # Add the archive ID to the set for this tag $redistx->sadd( "INDEX_" . $redis_tag, $id ); } + + # Flag the ID as untagged if it had no tags + unless ($has_tags) { + $redistx->sadd( "LRR_UNTAGGED", $id ); + } + } + + if ( $redis->hexists( $id, "title" ) ) { + my $title = $redis->hget( $id, "title" ); + + # Decode and lowercase the title + $title = lc( redis_decode($title) ); + remove_spaces($title); + remove_newlines($title); + $title = redis_encode($title); + + # The LRR_TITLES set contains both the title and the id under the form $title\x00$id. + $redistx->sadd( "LRR_TITLES", "$title\0$id" ); } } diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index 5046e16f1..b5caa9b22 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -315,10 +315,27 @@ sub clean_database { sub set_title { my ( $id, $newtitle ) = @_; - my $redis = LANraragi::Model::Config->get_redis; + my $redis = LANraragi::Model::Config->get_redis; + my $redis_search = LANraragi::Model::Config->get_redis_search; if ( $newtitle ne "" ) { + + # Remove old title from search set + my $oldtitle = lc( redis_decode( $redis->hget( $id, "title" ) ) ); + remove_spaces($oldtitle); + remove_newlines($oldtitle); + $oldtitle = redis_encode($oldtitle); + $redis_search->srem( "LRR_TITLES", "$oldtitle\0$id" ); + + # Set actual title in metadata DB $redis->hset( $id, "title", redis_encode($newtitle) ); + + # Set title/ID key in search set + $newtitle = lc($newtitle); + remove_spaces($newtitle); + remove_newlines($newtitle); + $newtitle = redis_encode($newtitle); + $redis_search->sadd( "LRR_TITLES", "$oldtitle\0$id" ); } $redis->quit; } @@ -367,8 +384,9 @@ sub update_indexes { my $redis = LANraragi::Model::Config->get_redis_search; $redis->multi; - my @oldtags = split( ',', $oldtags ); - my @newtags = split( ',', $newtags ); + my @oldtags = split( /,\s?/, $oldtags ); + my @newtags = split( /,\s?/, $newtags ); + my $has_tags = 0; foreach my $tag (@oldtags) { @@ -377,9 +395,20 @@ sub update_indexes { } foreach my $tag (@newtags) { + + # The following are basic and therefore don't count as "tagged" + $has_tags = 1 unless $tag =~ /(artist|parody|series|language|event|group|date_added|timestamp):.*/; + $redis->sadd( "INDEX_" . redis_encode( lc($tag) ), $id ); } + # Add or remove the ID from the untagged list + if ($has_tags) { + $redis->srem( "LRR_UNTAGGED", $id ); + } else { + $redis->sadd( "LRR_UNTAGGED", $id ); + } + $redis->exec; $redis->quit; } diff --git a/tools/Documentation/extending-lanraragi/architecture.md b/tools/Documentation/extending-lanraragi/architecture.md index b76c93120..6b75077fb 100644 --- a/tools/Documentation/extending-lanraragi/architecture.md +++ b/tools/Documentation/extending-lanraragi/architecture.md @@ -212,7 +212,9 @@ The base architecture is as follows: | |- LRR_STATS <- Redis sorted set used to build the statistics/tag cloud JSON. | -|- LRR_UNTAGGED <- List of archive IDs that don't have any tags (except for tags added automatically by the autotagger) +|- LRR_UNTAGGED <- Redis set of archive IDs that don't have any tags (except for tags added automatically by the autotagger) +| +|- LRR_TITLES <- Redis set containing all titles in the DB, alongside their ID. (In the "title\0ID" format) | |- INDEX_***:**** <- Each tag(namespaced or not) has a matching Redis set, with all the IDs that have this tag in their metadata. This is used for search indexing. | From ede94f702410ed7d08285b83e7b36896cd692833 Mon Sep 17 00:00:00 2001 From: Difegue Date: Wed, 14 Dec 2022 23:32:45 +0100 Subject: [PATCH 17/45] Use LRR_UNTAGGED for the untagged archive endpoint and make LRR_TITLES a sorted set --- lib/LANraragi/Controller/Api/Database.pm | 2 +- lib/LANraragi/Controller/Api/Search.pm | 2 +- lib/LANraragi/Model/Archive.pm | 38 ++----------------- lib/LANraragi/Model/Backup.pm | 2 +- lib/LANraragi/Model/Category.pm | 5 +++ lib/LANraragi/Model/Stats.pm | 6 +-- lib/LANraragi/Utils/Database.pm | 13 +++---- .../extending-lanraragi/architecture.md | 2 +- 8 files changed, 22 insertions(+), 48 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index af2f7cd7a..a1ba76529 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -29,7 +29,7 @@ sub serve_tag_stats { sub clean_database { my ( $deleted, $unlinked ) = LANraragi::Utils::Database::clean_database; - #Force a refresh + # Force a refresh invalidate_cache(1); shift->render( diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index 9ac081e75..5a7d4460c 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -88,7 +88,7 @@ sub handle_api { } sub clear_cache { - invalidate_cache(1); + invalidate_cache(); render_api_response( shift, "clear_cache" ); } diff --git a/lib/LANraragi/Model/Archive.pm b/lib/LANraragi/Model/Archive.pm index f6f00a298..835276eb4 100644 --- a/lib/LANraragi/Model/Archive.pm +++ b/lib/LANraragi/Model/Archive.pm @@ -100,41 +100,11 @@ sub generate_opds_catalog { } # Return a list of archive IDs that have no tags. -# Tags added automatically by the autotagger are ignored. +# TODO: Move this simple call to the API and integrate the smembers call in the search engine itself sub find_untagged_archives { - my $redis = LANraragi::Model::Config->get_redis; - my @keys = $redis->keys('????????????????????????????????????????'); - my @untagged; - - #Parse the archive list. - foreach my $id (@keys) { - my $archive = $redis->hget( $id, "file" ); - if ( $archive && -e $archive ) { - - my $title = $redis->hget( $id, "title" ); - $title = redis_decode($title); - - my $tagstr = $redis->hget( $id, "tags" ); - $tagstr = redis_decode($tagstr); - my @tags = split( /,\s?/, $tagstr ); - my $nondefaulttags = 0; - - foreach my $t (@tags) { - remove_spaces($t); - remove_newlines($t); - - # The following are basic and therefore don't count as "tagged" - $nondefaulttags += 1 unless $t =~ /(artist|parody|series|language|event|group|date_added|timestamp):.*/; - } - - #If the archive has no tags, or the tags namespaces are only from - #filename parsing (probably), add it to the list. - if ( !@tags || $nondefaulttags == 0 ) { - push @untagged, $id; - } - } - } + my $redis = LANraragi::Model::Config->get_redis_search; + my @untagged = $redis->smembers("LRR_UNTAGGED"); $redis->quit; return @untagged; } @@ -345,7 +315,7 @@ sub update_metadata { } # Bust cache - invalidate_cache(1); + invalidate_cache(); # No errors. return ""; diff --git a/lib/LANraragi/Model/Backup.pm b/lib/LANraragi/Model/Backup.pm index 4e325376a..b9920c222 100644 --- a/lib/LANraragi/Model/Backup.pm +++ b/lib/LANraragi/Model/Backup.pm @@ -136,7 +136,7 @@ sub restore_from_JSON { } # Force a refresh - invalidate_cache(1); + invalidate_cache(); $redis->quit(); } diff --git a/lib/LANraragi/Model/Category.pm b/lib/LANraragi/Model/Category.pm index 0e4b6d563..7f0e24ef2 100644 --- a/lib/LANraragi/Model/Category.pm +++ b/lib/LANraragi/Model/Category.pm @@ -75,6 +75,11 @@ sub get_category { my $logger = get_logger( "Categories", "lanraragi" ); my $redis = LANraragi::Model::Config->get_redis; + if ( $cat_id eq "" ) { + $logger->debug("No category ID provided."); + return (); + } + unless ( length($cat_id) == 14 && $redis->exists($cat_id) ) { $logger->warn("$cat_id doesn't exist in the database!"); return (); diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index 93cbe8863..cc72155e8 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -48,7 +48,7 @@ sub get_page_stat { # - LRR_URL_MAP, which maps URLs to IDs in the database that have them as a source: tag # - LRR_STATS, which is a sorted set used to build the statistics/tag cloud JSON # - LRR_UNTAGGED, which is a set used by the untagged archives API -# - LRR_TITLES, which is a set containing all titles in the DB, alongside their ID. (In the "title\0ID" format) +# - LRR_TITLES, which is a lexicographically sorted set containing all titles in the DB, alongside their ID. (In the "title\0ID" format) # * It also builds index sets for each distinct tag. sub build_stat_hashes { @@ -121,8 +121,8 @@ sub build_stat_hashes { remove_newlines($title); $title = redis_encode($title); - # The LRR_TITLES set contains both the title and the id under the form $title\x00$id. - $redistx->sadd( "LRR_TITLES", "$title\0$id" ); + # The LRR_TITLES lexicographically sorted set contains both the title and the id under the form $title\x00$id. + $redistx->zadd( "LRR_TITLES", 0, "$title\0$id" ); } } diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index b5caa9b22..ac89afced 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -325,7 +325,7 @@ sub set_title { remove_spaces($oldtitle); remove_newlines($oldtitle); $oldtitle = redis_encode($oldtitle); - $redis_search->srem( "LRR_TITLES", "$oldtitle\0$id" ); + $redis_search->zrem( "LRR_TITLES", "$oldtitle\0$id" ); # Set actual title in metadata DB $redis->hset( $id, "title", redis_encode($newtitle) ); @@ -335,7 +335,7 @@ sub set_title { remove_spaces($newtitle); remove_newlines($newtitle); $newtitle = redis_encode($newtitle); - $redis_search->sadd( "LRR_TITLES", "$oldtitle\0$id" ); + $redis_search->zadd( "LRR_TITLES", 0, "$oldtitle\0$id" ); } $redis->quit; } @@ -464,17 +464,16 @@ sub redis_decode { } # Bust the current search cache key in Redis. -# Add "1" as a parameter to perform a cache warm after the wipe. +# Add "1" as a parameter to rebuild stat hashes as well. (Use with caution!) sub invalidate_cache { - my $do_warm = shift; - my $redis = LANraragi::Model::Config->get_redis; + my $rebuild_indexes = shift; + my $redis = LANraragi::Model::Config->get_redis_search; $redis->del("LRR_SEARCHCACHE"); $redis->hset( "LRR_SEARCHCACHE", "created", time ); $redis->quit(); # Re-warm the cache to ensure sufficient speed on the main index - if ($do_warm) { - LANraragi::Model::Config->get_minion->enqueue( warm_cache => [] => { priority => 3 } ); + if ($rebuild_indexes) { LANraragi::Model::Config->get_minion->enqueue( build_stat_hashes => [] => { priority => 3 } ); } } diff --git a/tools/Documentation/extending-lanraragi/architecture.md b/tools/Documentation/extending-lanraragi/architecture.md index 6b75077fb..fd7c551a3 100644 --- a/tools/Documentation/extending-lanraragi/architecture.md +++ b/tools/Documentation/extending-lanraragi/architecture.md @@ -214,7 +214,7 @@ The base architecture is as follows: | |- LRR_UNTAGGED <- Redis set of archive IDs that don't have any tags (except for tags added automatically by the autotagger) | -|- LRR_TITLES <- Redis set containing all titles in the DB, alongside their ID. (In the "title\0ID" format) +|- LRR_TITLES <- Redis lexicographically sorted set containing all titles in the DB, alongside their ID. (In the "title\0ID" format) | |- INDEX_***:**** <- Each tag(namespaced or not) has a matching Redis set, with all the IDs that have this tag in their metadata. This is used for search indexing. | From 59abb7728dd63121d3c0839af933a9a6cde9a916 Mon Sep 17 00:00:00 2001 From: Difegue Date: Wed, 14 Dec 2022 23:33:11 +0100 Subject: [PATCH 18/45] Stop using the warm cache job, as it hopefully won't be needed anymore --- lib/LANraragi.pm | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/LANraragi.pm b/lib/LANraragi.pm index 886c1953e..43022a6df 100644 --- a/lib/LANraragi.pm +++ b/lib/LANraragi.pm @@ -141,10 +141,9 @@ sub startup { LANraragi::Utils::Minion::add_tasks( $self->minion ); $self->LRR_LOGGER->debug("Registered tasks with Minion."); - # Warm search cache + # Rebuild stat hashes # /!\ Enqueuing tasks must be done either before starting the worker, or once the IOLoop is started! # Anything else can cause weird database lockups. - $self->minion->enqueue('warm_cache'); $self->minion->enqueue('build_stat_hashes'); # Start a Minion worker in a subprocess From afdf0d1cab60bb91d70b8ea28084737f2568c834 Mon Sep 17 00:00:00 2001 From: Difegue Date: Thu, 15 Dec 2022 00:40:36 +0100 Subject: [PATCH 19/45] Add LRR_NEW index, remove old specific cachebust code Mojo 9.20 is now required because of the trace log level --- lib/LANraragi/Controller/Api/Archive.pm | 14 ++------ lib/LANraragi/Controller/Api/Database.pm | 1 - lib/LANraragi/Controller/Batch.pm | 4 +-- lib/LANraragi/Model/Stats.pm | 18 +++++++--- lib/LANraragi/Utils/Database.pm | 46 +++++++++++++----------- lib/LANraragi/Utils/Logging.pm | 5 +++ tools/cpanfile | 2 +- 7 files changed, 49 insertions(+), 41 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Archive.pm b/lib/LANraragi/Controller/Api/Archive.pm index 45ecaf1e4..0edfb1ee4 100644 --- a/lib/LANraragi/Controller/Api/Archive.pm +++ b/lib/LANraragi/Controller/Api/Archive.pm @@ -8,7 +8,7 @@ use Mojo::JSON qw(decode_json); use Scalar::Util qw(looks_like_number); use LANraragi::Utils::Generic qw(render_api_response); -use LANraragi::Utils::Database qw(get_archive_json); +use LANraragi::Utils::Database qw(get_archive_json set_isnew); use LANraragi::Model::Archive; use LANraragi::Model::Category; @@ -127,17 +127,7 @@ sub clear_new { my $self = shift; my $id = check_id_parameter( $self, "clear_new" ) || return; - my $redis = $self->LRR_CONF->get_redis(); - - # Just set isnew to false for the provided ID. - if ( $redis->hget( $id, "isnew" ) ne "false" ) { - - # Bust search cache...partially! - LANraragi::Utils::Database::invalidate_isnew_cache(); - - $redis->hset( $id, "isnew", "false" ); - } - $redis->quit(); + set_isnew( $id, "false" ); $self->render( json => { diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index a1ba76529..97080da70 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -57,7 +57,6 @@ sub clear_new_all { } # Bust isnew cache - invalidate_isnew_cache(); $redis->quit(); render_api_response( $self, "clear_new_all" ); } diff --git a/lib/LANraragi/Controller/Batch.pm b/lib/LANraragi/Controller/Batch.pm index f3d5a3c6b..24c2380ff 100644 --- a/lib/LANraragi/Controller/Batch.pm +++ b/lib/LANraragi/Controller/Batch.pm @@ -7,7 +7,7 @@ use Mojo::JSON qw(decode_json); use LANraragi::Utils::Generic qw(generate_themes_header); use LANraragi::Utils::Tags qw(rewrite_tags split_tags_to_array restore_CRLF); -use LANraragi::Utils::Database qw(get_computed_tagrules set_tags set_title invalidate_cache); +use LANraragi::Utils::Database qw(get_computed_tagrules set_tags set_title set_isnew invalidate_cache); use LANraragi::Utils::Plugins qw(get_plugins get_plugin get_plugin_parameters); use LANraragi::Utils::Logging qw(get_logger); @@ -88,7 +88,7 @@ sub socket { } if ( $operation eq "clearnew" ) { - $redis->hset( $id, "isnew", "false" ); + set_isnew( $id, "false" ); $client->send( { json => { diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index cc72155e8..35cc8e2ed 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -48,6 +48,7 @@ sub get_page_stat { # - LRR_URL_MAP, which maps URLs to IDs in the database that have them as a source: tag # - LRR_STATS, which is a sorted set used to build the statistics/tag cloud JSON # - LRR_UNTAGGED, which is a set used by the untagged archives API +# - LRR_NEW, which contains all archives that have isnew=true # - LRR_TITLES, which is a lexicographically sorted set containing all titles in the DB, alongside their ID. (In the "title\0ID" format) # * It also builds index sets for each distinct tag. sub build_stat_hashes { @@ -60,18 +61,19 @@ sub build_stat_hashes { my $logger = get_logger( "Tag Stats", "lanraragi" ); # 40-character long keys only => Archive IDs - my @keys = $redis->keys('????????????????????????????????????????'); + my @keys = $redis->keys('????????????????????????????????????????'); + my $archive_count = scalar @keys; # Cancel the transaction if the hashes have been modified by another job in the meantime. # This also allows for the previous stats/map to still be readable until we're done. - $redistx->watch( "LRR_STATS", "LRR_URLMAP", "LRR_UNTAGGED", "LRR_TITLES" ); + $redistx->watch( "LRR_STATS", "LRR_URLMAP", "LRR_UNTAGGED", "LRR_TITLES", "LRR_NEW" ); $redistx->multi; # Hose the entire index DB since we're rebuilding it $redistx->flushdb(); # Iterate on hashes to get their tags - $logger->debug("Building stat indexes..."); + $logger->info("Building stat indexes... ($archive_count archives)"); foreach my $id (@keys) { if ( $redis->hexists( $id, "tags" ) ) { @@ -92,7 +94,7 @@ sub build_stat_hashes { if ( $t =~ /source:(.*)/i ) { my $url = $1; trim_url($url); - $logger->debug("Adding $url as an URL for $id"); + $logger->trace("Adding $url as an URL for $id"); $redistx->hset( "LRR_URLMAP", $url, $id ); # No need to encode the value, as URLs are already encoded by design } @@ -108,6 +110,7 @@ sub build_stat_hashes { # Flag the ID as untagged if it had no tags unless ($has_tags) { + $logger->trace("Adding $id to LRR_UNTAGGED"); $redistx->sadd( "LRR_UNTAGGED", $id ); } } @@ -124,10 +127,15 @@ sub build_stat_hashes { # The LRR_TITLES lexicographically sorted set contains both the title and the id under the form $title\x00$id. $redistx->zadd( "LRR_TITLES", 0, "$title\0$id" ); } + + if ( $redis->hget( $id, "isnew" ) eq "true" ) { + $logger->trace("Adding $id to LRR_ISNEW"); + $redistx->sadd( "LRR_NEW", $id ); + } } $redistx->exec; - $logger->debug("Done!"); + $logger->info("Stat indexes built! ($archive_count archives)"); $redis->quit; $redistx->quit; } diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index ac89afced..e679576b8 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -21,7 +21,7 @@ use LANraragi::Utils::Logging qw(get_logger); # Functions for interacting with the DB Model. use Exporter 'import'; our @EXPORT_OK = - qw(redis_encode redis_decode invalidate_cache compute_id set_tags set_title get_computed_tagrules save_computed_tagrules get_archive_json get_archive_json_multi); + qw(redis_encode redis_decode invalidate_cache compute_id set_tags set_title set_isnew get_computed_tagrules save_computed_tagrules get_archive_json get_archive_json_multi); #add_archive_to_redis($id,$file,$redis) # Creates a DB entry for a file path with the given ID. @@ -44,7 +44,7 @@ sub add_archive_to_redis { $redis->hset( $id, "file", $file ); # New file in collection, so this flag is set. - $redis->hset( $id, "isnew", "true" ); + set_isnew( $id, "true" ); $redis->quit; return $name; @@ -374,6 +374,29 @@ sub set_tags { invalidate_cache(); } +#set_isnew($id, $isnew) +# Set $isnew for the archive with id $id. +sub set_isnew { + + my ( $id, $isnew ) = @_; + my $redis = LANraragi::Model::Config->get_redis(); + my $redis_search = LANraragi::Model::Config->get_redis_search(); + + # Just set isnew for the provided ID. + my $newval = $isnew ne "false" ? "true" : "false"; + + $redis->hset( $id, "isnew", $newval ); + + if ( $newval eq "true" ) { + $redis_search->sadd( "LRR_NEW", $id ); + } else { + $redis_search->srem( "LRR_NEW", $id ); + } + + $redis_search->quit; + $redis->quit; +} + # Splits both old and new tags, and: # Removes the ID from all sets of the old tags # Adds it back to all sets of the new tags. @@ -467,7 +490,7 @@ sub redis_decode { # Add "1" as a parameter to rebuild stat hashes as well. (Use with caution!) sub invalidate_cache { my $rebuild_indexes = shift; - my $redis = LANraragi::Model::Config->get_redis_search; + my $redis = LANraragi::Model::Config->get_redis_search; $redis->del("LRR_SEARCHCACHE"); $redis->hset( "LRR_SEARCHCACHE", "created", time ); $redis->quit(); @@ -478,23 +501,6 @@ sub invalidate_cache { } } -# Go through the search cache and only invalidate keys that rely on isNew. -sub invalidate_isnew_cache { - - my $redis = LANraragi::Model::Config->get_redis; - my %cache = $redis->hgetall("LRR_SEARCHCACHE"); - - foreach my $cachekey ( keys(%cache) ) { - - # A cached search uses isNew if the second to last number is equal to 1 - # i.e, "--title-asc-1-0" has to be pruned - if ( $cachekey =~ /.*-.*-.*-.*-1-\d?/ ) { - $redis->hdel( "LRR_SEARCHCACHE", $cachekey ); - } - } - $redis->quit(); -} - sub save_computed_tagrules { my ($tagrules) = @_; my $redis = LANraragi::Model::Config->get_redis; diff --git a/lib/LANraragi/Utils/Logging.pm b/lib/LANraragi/Utils/Logging.pm index 60584f707..9b0c9c639 100644 --- a/lib/LANraragi/Utils/Logging.pm +++ b/lib/LANraragi/Utils/Logging.pm @@ -56,6 +56,11 @@ sub get_logger { $log->level('debug'); } + # Step down into trace if we're launched from npm run dev-server + if ( $ENV{MOJO_MODE} eq "development" ) { + $log->level('trace'); + } + #Copy logged messages to STDOUT with the matching name $log->on( message => sub { diff --git a/tools/cpanfile b/tools/cpanfile index ff53d9658..359c61fe0 100755 --- a/tools/cpanfile +++ b/tools/cpanfile @@ -36,7 +36,7 @@ requires 'Test::Trap', 0.3.4; requires 'Test::Deep', 1.130; # Mojo stuff -requires 'Mojolicious', 9.27; +requires 'Mojolicious', 9.30; requires 'Mojolicious::Plugin::TemplateToolkit', 0.005; requires 'Mojolicious::Plugin::RenderFile', 0.12; requires 'Mojolicious::Plugin::Status', 1.15; From b2e26d68bac741abf244e77c50f463c9bd75bf5e Mon Sep 17 00:00:00 2001 From: Difegue Date: Thu, 15 Dec 2022 00:41:25 +0100 Subject: [PATCH 20/45] (#555) Rewrite search to use all our new indexes (WIP, no sort) --- lib/LANraragi/Controller/Api/Search.pm | 6 +- lib/LANraragi/Model/Search.pm | 313 ++++++++++++------------- 2 files changed, 158 insertions(+), 161 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index 5a7d4460c..2717882b3 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -103,6 +103,7 @@ sub get_random_archives { my $category = $req->param('category') || ""; my $random_count = $req->param('count') || 5; + # TODO rework this # Use the search engine to get IDs matching the filter/category selection, with start=-1 to get all data # This method could be extended later to also use isnew/untagged filters. my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( $filter, $category, -1, "title", 0, "", "" ); @@ -126,10 +127,7 @@ sub get_random_archives { # Creates a Datatables-compatible json from the given data. sub get_datatables_object { - my ( $draw, $redis, $total, $filtered, @keys ) = @_; - - # Get IDs from keys - my @ids = map { $_->{id} } @keys; + my ( $draw, $redis, $total, $filtered, @ids ) = @_; # Get archive data my @data = get_archive_json_multi(@ids); diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 1b5f667a7..31b9d3368 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -25,186 +25,213 @@ use LANraragi::Model::Category; sub do_search { my ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $untaggedonly ) = @_; - my $sortorder_inv = $sortorder ? 0 : 1; - my $redis = LANraragi::Model::Config->get_redis; + my $redis = LANraragi::Model::Config->get_redis_search; my $logger = get_logger( "Search Engine", "lanraragi" ); # Search filter results - my $total = $redis->hlen("LRR_FILEMAP") + 0; # Total number of archives (as int) - my @filtered = (); + my $total = $redis->hlen("LRR_FILEMAP") + 0; # Total number of archives (as int) # Look in searchcache first - my $cachekey = encode_utf8("$category_id-$filter-$sortkey-$sortorder-$newonly-$untaggedonly"); - my $cachekey_inv = encode_utf8("$category_id-$filter-$sortkey-$sortorder_inv-$newonly-$untaggedonly"); + my $sortorder_inv = $sortorder ? 0 : 1; + my $cachekey = encode_utf8("$category_id-$filter-$sortkey-$sortorder-$newonly-$untaggedonly"); + my $cachekey_inv = encode_utf8("$category_id-$filter-$sortkey-$sortorder_inv-$newonly-$untaggedonly"); + my ( $cachehit, @filtered ) = check_cache( $cachekey, $cachekey_inv ); + + unless ($cachehit) { + $logger->debug("No cache available, doing a full DB parse."); + @filtered = search_uncached( $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly ); + + # Cache this query in the search database + eval { $redis->hset( "LRR_SEARCHCACHE", $cachekey, nfreeze \@filtered ); }; + } + $redis->quit(); + + # If start is negative, return all possible data + # Kind of a hack for the random API, not sure how this could be improved... + # (The paging has always been there mostly to make datatables behave after all.) + if ( $start == -1 ) { + return ( $total, $#filtered + 1, @filtered ); + } + + # Only get the first X keys + my $keysperpage = LANraragi::Model::Config->get_pagesize; + + # Return total keys and the filtered ones + my $end = min( $start + $keysperpage - 1, $#filtered ); + return ( $total, $#filtered + 1, @filtered[ $start .. $end ] ); +} + +sub check_cache { + + my ( $cachekey, $cachekey_inv ) = @_; + my $redis = LANraragi::Model::Config->get_redis_search; + my $logger = get_logger( "Search Cache", "lanraragi" ); + + my @filtered = (); + my $cachehit = 0; $logger->debug("Search request: $cachekey"); - if ( $redis->exists("LRR_SEARCHCACHE") - && $redis->hexists( "LRR_SEARCHCACHE", $cachekey ) ) { + if ( $redis->exists("LRR_SEARCHCACHE") && $redis->hexists( "LRR_SEARCHCACHE", $cachekey ) ) { $logger->debug("Using cache for this query."); + $cachehit = 1; # Thaw cache and use that as the filtered list my $frozendata = $redis->hget( "LRR_SEARCHCACHE", $cachekey ); @filtered = @{ thaw $frozendata }; - } elsif ( $redis->exists("LRR_SEARCHCACHE") - && $redis->hexists( "LRR_SEARCHCACHE", $cachekey_inv ) ) { + } elsif ( $redis->exists("LRR_SEARCHCACHE") && $redis->hexists( "LRR_SEARCHCACHE", $cachekey_inv ) ) { $logger->debug("A cache key exists with the opposite sortorder."); + $cachehit = 1; # Thaw cache, invert the list to match the sortorder and use that as the filtered list my $frozendata = $redis->hget( "LRR_SEARCHCACHE", $cachekey_inv ); @filtered = reverse @{ thaw $frozendata }; + } - } else { - $logger->debug("No cache available, doing a full DB parse."); + $redis->quit(); + return ( $cachehit, @filtered ); +} + +# Grab all our IDs, then filter them down according to the following filters and tokens' ID groups. +sub search_uncached { + + my ( $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly ) = @_; + my $redis = LANraragi::Model::Config->get_redis_search; + my $logger = get_logger( "Search Core", "lanraragi" ); + + # Compute search filters + my @tokens = compute_search_filter($filter); - # Get all archives from redis - or just use IDs from the category if possible. - my ( $filter_cat, @keys ) = get_source_data( $redis, $category_id ); + # Prepare array: For each token, we'll have a list of matching archive IDs. + # We intersect those lists as we proceed to get the final result. + # Start with all our IDs. + my @filtered = LANraragi::Model::Config->get_redis->keys('????????????????????????????????????????'); - # Compute search filters - my @tokens = compute_search_filter($filter); - my @tokens_cat = compute_search_filter($filter_cat); + # If we're using a category, we'll need to get its source data first. + my %category = LANraragi::Model::Category::get_category($category_id); - # Setup parallel processing - my $numCpus = Sys::CpuAffinity::getNumCpus(); - my $pl = Parallel::Loops->new($numCpus); - my @shared = (); - $pl->share( \@shared ); + if (%category) { - # If the untagged filter is enabled, call the untagged files API - my %untagged = (); - if ($untaggedonly) { + # We're using a category! Update its lastused value. + $redis->hset( $category_id, "last_used", time() ); - # Map the array to a hash to easily check if it contains our id - %untagged = map { $_ => 1 } LANraragi::Model::Archive::find_untagged_archives(); + # If the category is dynamic, get its search predicate and add it to the tokens. + # If it's static however, we can use its ID list as the base for our result array. + if ( $category{search} ne "" ) { + my @cat_tokens = compute_search_filter( $category{search} ); + push @tokens, @cat_tokens; + } else { + @filtered = intersect_arrays( $category{archives}, \@filtered, 0 ); } + } - my @sections = split_workload_by_cpu( $numCpus, @keys ); + # If the untagged filter is enabled, call the untagged files API + if ($untaggedonly) { + my @untagged = LANraragi::Model::Archive::find_untagged_archives(); + @filtered = intersect_arrays( \@untagged, \@filtered, 0 ); + } - # Go through tags and apply search filter in subprocesses - $pl->foreach( - \@sections, - sub { + # Check new filter + if ($newonly) { + my @new = $redis->smembers("LRR_NEW"); + @filtered = intersect_arrays( \@new, \@filtered, 0 ); + } - my @ids = @$_; + # Iterate through each token and intersect the results with the previous ones. + unless ( scalar @tokens == 0 || scalar @filtered == 0 ) { + foreach my $token (@tokens) { - # Get all the info for the given IDs as an atomic operation - $redis = LANraragi::Model::Config->get_redis; - $redis->multi; - foreach my $id (@$_) { + my $tag = $token->{tag}; + my $isneg = $token->{isneg}; + my $isexact = $token->{isexact}; - # Check untagged filter first as it requires no DB hits - if ( !$untaggedonly || exists( $untagged{$id} ) ) { - $redis->hgetall($id); - } else { - $logger->debug("$id doesn't exist in the untagged_archives set, skipping."); - @ids = grep { $_ ne $id } @ids; # Remove id from array to avoid messing up the mapping post-multi - } - } - my @data = $redis->exec; - $redis->quit; + $logger->debug("Searching for $tag, isneg=$isneg, isexact=$isexact"); - for my $i ( 0 .. $#data ) { + my @ids = (); - # MULTI returns data in the same order the operations were sent, - # so we can get the ID from the original array this way. - my %hash; + # Tags are always considered exact for now, so just check if an index for it exists + if ( $redis->exists("INDEX_$tag") ) { - if ( $data[$i] ) { - %hash = @{ $data[$i] }; - } else { - next; - } - my $id = $ids[$i]; + # Get the list of IDs for this tag + @ids = $redis->smembers("INDEX_$tag"); + } - my ( $tags, $title, $file, $isnew ) = @hash{qw(tags title file isnew)}; + # Append fuzzy title search + my $namesearch = $isexact ? $tag : "*$tag*"; + my $scan = -1; + while ( $scan > 0 ) { - $title = redis_decode($title); - $tags = redis_decode($tags); + # First iteration + if ( $scan == -1 ) { $scan = 0; } + $logger->debug("Scanning for $namesearch, cursor=$scan"); - # Check new filter first - if ( $newonly && $isnew && $isnew ne "true" ) { - next; - } + my @result = $redis->zscan( "LRR_TITLES", $scan, "MATCH", $namesearch ); + $scan = $result[0]; - # Check category search and base search filter - my $concat = $tags ? $title ? $title . "," . $tags : $tags : $title; - if ( $file - && matches_search_filter( $concat, @tokens_cat ) - && matches_search_filter( $concat, @tokens ) ) { + foreach my $title ( @{ $result[1] } ) { + $logger->debug("Found title match: $title"); - # Push id to array - push @shared, { id => $id, title => $title, tags => $tags }; - } + # Strip everything before \x00 to get the ID out of the key + my $id = substr( $title, index( $title, "\x00" ) + 1 ); + push @ids, $id; } } - ); - # Remove the extra reference/objects Parallel::Loops adds to the array, - # as that'll cause memory leaks when we serialize/deserialize them with Storable. - # This is done by simply copying the parallelized array to @filtered. - @filtered = @shared; + if ( scalar @ids == 0 && !$isneg ) { - if ( $#filtered > 0 ) { + # No more results, we can end search here + $logger->debug("No results for this token, halting search."); + @filtered = (); + last; + } else { + $logger->debug( "Found " . scalar @ids . " results for this token." ); - if ( !$sortkey ) { - $sortkey = "title"; + # Intersect the new list with the previous ones + @filtered = intersect_arrays( \@ids, \@filtered, $isneg ); } - - # Sort by the required metadata, asc or desc - @filtered = sort_results( $sortkey, $sortorder, @filtered ); } - - # Cache this query in Redis - eval { $redis->hset( "LRR_SEARCHCACHE", $cachekey, nfreeze \@filtered ); }; } - $redis->quit(); - # If start is negative, return all possible data - # Kind of a hack for the random API, not sure how this could be improved... - # (The paging has always been there mostly to make datatables behave after all.) - if ( $start == -1 ) { - return ( $total, $#filtered + 1, @filtered ); - } + if ( $#filtered > 0 ) { - # Only get the first X keys - my $keysperpage = LANraragi::Model::Config->get_pagesize; - - # Return total keys and the filtered ones - my $end = min( $start + $keysperpage - 1, $#filtered ); - return ( $total, $#filtered + 1, @filtered[ $start .. $end ] ); -} + if ( !$sortkey ) { + $sortkey = "title"; + } -sub get_source_data { + # TODO Sort by the required metadata, asc or desc + #@filtered = sort_results( $sortkey, $sortorder, @filtered ); + } - my ( $redis, $category_id ) = @_; - my @keys = (); - my $filter_cat = ""; + return @filtered; +} - if ( $category_id ne "" ) { - my %category = LANraragi::Model::Category::get_category($category_id); +# intersect_arrays(@array1, @array2, $isneg) +# Intersect two arrays and return the result. If $isneg is true, return the difference instead. +sub intersect_arrays { - if (%category) { + my ( $array1, $array2, $isneg ) = @_; - # We're using a category! Update its lastused value. - $redis->hset( $category_id, "last_used", time() ); + # Special case: If array1 is empty, just return array2 as we don't have anything to intersect yet + if ( scalar @$array1 == 0 ) { + return @$array2; + } - # If the category is dynamic, get its search predicate - $filter_cat = $category{search}; + # If array2 is empty, die since this sub shouldn't even be used in that case + if ( scalar @$array2 == 0 ) { + die "intersect_arrays called with an empty array2"; + } - # If it's static however, we can use its ID list as the source data. - if ( $filter_cat eq "" ) { - @keys = @{ $category{archives} }; - } else { - @keys = $redis->keys('????????????????????????????????????????'); - } - } + my %hash = map { $_ => 1 } @$array1; + my @result; + if ($isneg) { + @result = grep { !exists $hash{$_} } @$array2; } else { - @keys = $redis->keys('????????????????????????????????????????'); + @result = grep { exists $hash{$_} } @$array2; } - return ( $filter_cat, @keys ); + return @result; } # compute_search_filter($filter) @@ -213,6 +240,7 @@ sub get_source_data { sub compute_search_filter { my $filter = shift; + my $logger = get_logger( "Search Core", "lanraragi" ); my @tokens = (); if ( !$filter ) { $filter = ""; } @@ -266,14 +294,16 @@ sub compute_search_filter { } # Escape already present regex characters - $tag = quotemeta($tag); + $logger->debug("Pre-escaped tag: $tag"); + + #$tag = quotemeta($tag); - # Replace placeholders(with an extra backslash in em thanks to quotemeta) with regex-friendly variants, - # ? _ => . - $tag =~ s/\\\?|\_/\./g; + # Replace placeholders with glob-style patterns, + # ? or _ => ? + $tag =~ s/\_/\?/g; - # * % => .* - $tag =~ s/\\\*|\\\%/\.\*/g; + # * or % => * + $tag =~ s/\%/\*/g; push @tokens, { tag => $tag, @@ -284,37 +314,6 @@ sub compute_search_filter { return @tokens; } -# matches_search_filter($computed_filter, $tags) -# Search engine core. -sub matches_search_filter { - - my ( $tags, @tokens ) = @_; - - foreach my $token (@tokens) { - - my $tag = $token->{tag}; - my $isneg = $token->{isneg}; - my $isexact = $token->{isexact}; - - # For each token, we check if the tag is present. - my $tagpresent = 0; - if ($isexact) { # The tag must necessarily be complete if isexact = 1 - # Check for comma + potential space before and comma after the tag, or start/end of string to account for the first/last tag. - $tagpresent = $tags =~ m/(.*\,\s*|^)$tag(\,.*|$)/i; - } else { - $tagpresent = $tags =~ m/.*$tag.*/i; - } - - #present=true & isneg=true => false - #present=false & isneg=false => false - return 0 if ( $tagpresent == $isneg ); - - } - - # All filters passed! - return 1; -} - sub sort_results { my ( $sortkey, $sortorder, @filtered ) = @_; From 623a8c046cd105bd13ad808d200d851c55e012b0 Mon Sep 17 00:00:00 2001 From: Difegue Date: Thu, 15 Dec 2022 00:47:07 +0100 Subject: [PATCH 21/45] (#563) Use LRR_TITLES for the dreaded total entry count Why do people look at this number so much --- lib/LANraragi/Model/Search.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 31b9d3368..fb6f1bcab 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -30,7 +30,7 @@ sub do_search { my $logger = get_logger( "Search Engine", "lanraragi" ); # Search filter results - my $total = $redis->hlen("LRR_FILEMAP") + 0; # Total number of archives (as int) + my $total = $redis->zcard("LRR_TITLES") + 0; # Total number of archives (as int) # Look in searchcache first my $sortorder_inv = $sortorder ? 0 : 1; From a8a4858c03cedd4071d2b1e8850244e6b0de9f9b Mon Sep 17 00:00:00 2001 From: Difegue Date: Fri, 16 Dec 2022 01:17:27 +0100 Subject: [PATCH 22/45] Add npm run kill-workers convenience command, using the PID textfiles made for s6 shutdown_from_pid doesn't seem to work well at SIGTERM on some linuxes, maybe we should just revert to killing from the textfile outright? --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 1d5afddc1..0b2393f54 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "lint": "eslint --ext .js public/", "start": "perl ./script/launcher.pl -f ./script/lanraragi", "dev-server": "perl ./script/launcher.pl -m -v ./script/lanraragi", + "kill-workers": "(kill -15 `cat ./public/temp/shinobu.pid-s6` || true) && (kill -15 `cat ./public/temp/minion.pid-s6` || true)", "docker-build": "docker build -t difegue/lanraragi -f ./tools/build/docker/Dockerfile .", "critic": "perlcritic ./lib/* ./script/* ./tools/install.pl", "backup-db": "perl ./script/backup", From 5ede28722c92dde8b065dedc9f2450838589edbc Mon Sep 17 00:00:00 2001 From: Difegue Date: Fri, 16 Dec 2022 01:19:42 +0100 Subject: [PATCH 23/45] Fix search bugs + search delimiter is now commas instead of space this makes tag search act similarly to the way we enter tag metadata in the app to begin with, and makes for simpler code... --- lib/LANraragi/Model/Search.pm | 30 ++++++++++++++++++++---------- lib/LANraragi/Utils/Database.pm | 4 ++-- public/js/index_datatables.js | 2 +- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index fb6f1bcab..c17e6c0f9 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -13,7 +13,7 @@ use Sys::CpuAffinity; use Parallel::Loops; use Mojo::JSON qw(decode_json); -use LANraragi::Utils::Generic qw(split_workload_by_cpu); +use LANraragi::Utils::Generic qw(split_workload_by_cpu remove_spaces); use LANraragi::Utils::Database qw(redis_decode); use LANraragi::Utils::Logging qw(get_logger); @@ -151,26 +151,30 @@ sub search_uncached { my @ids = (); # Tags are always considered exact for now, so just check if an index for it exists + # TODO Add fuzzy search for tags? Probably needed if only to have wildcards work... if ( $redis->exists("INDEX_$tag") ) { # Get the list of IDs for this tag @ids = $redis->smembers("INDEX_$tag"); + $logger->trace( "Found tag index for $tag, containing " . scalar @ids . " IDs" ); } # Append fuzzy title search my $namesearch = $isexact ? $tag : "*$tag*"; my $scan = -1; - while ( $scan > 0 ) { + while ( $scan != 0 ) { # First iteration if ( $scan == -1 ) { $scan = 0; } - $logger->debug("Scanning for $namesearch, cursor=$scan"); + $logger->trace("Scanning for $namesearch, cursor=$scan"); - my @result = $redis->zscan( "LRR_TITLES", $scan, "MATCH", $namesearch ); + my @result = $redis->zscan( "LRR_TITLES", $scan, "MATCH", $namesearch, "COUNT", 100 ); $scan = $result[0]; foreach my $title ( @{ $result[1] } ) { - $logger->debug("Found title match: $title"); + + if ( $title eq "0" ) { next; } # Skip scores + $logger->trace("Found title match: $title"); # Strip everything before \x00 to get the ID out of the key my $id = substr( $title, index( $title, "\x00" ) + 1 ); @@ -181,11 +185,11 @@ sub search_uncached { if ( scalar @ids == 0 && !$isneg ) { # No more results, we can end search here - $logger->debug("No results for this token, halting search."); + $logger->trace("No results for this token, halting search."); @filtered = (); last; } else { - $logger->debug( "Found " . scalar @ids . " results for this token." ); + $logger->trace( "Found " . scalar @ids . " results for this token." ); # Intersect the new list with the previous ones @filtered = intersect_arrays( \@ids, \@filtered, $isneg ); @@ -256,13 +260,18 @@ sub compute_search_filter { my $char = chop $b; my $isneg = 0; + # Skip spaces + while ( $char eq " " && $b ne "" ) { + $char = chop $b; + } + if ( $char eq "-" ) { $isneg = 1; $char = chop $b; } - # Get characters until the next space, or the next " if the following char is " - my $delimiter = ' '; + # Get characters until the next comma, or the next " if the following char is " + my $delimiter = ','; if ( $char eq '"' ) { $delimiter = '"'; $char = chop $b; @@ -296,7 +305,8 @@ sub compute_search_filter { # Escape already present regex characters $logger->debug("Pre-escaped tag: $tag"); - #$tag = quotemeta($tag); + # Escape characters according to redis zscan rules + $tag =~ s/([\[\]\^\\])/\\$1/g; # Replace placeholders with glob-style patterns, # ? or _ => ? diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index e679576b8..61d0de61c 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -13,7 +13,7 @@ use Cwd; use Unicode::Normalize; use LANraragi::Model::Plugins; -use LANraragi::Utils::Generic qw(flat remove_spaces); +use LANraragi::Utils::Generic qw(flat remove_spaces remove_newlines); use LANraragi::Utils::Tags qw(unflat_tagrules tags_rules_to_array restore_CRLF); use LANraragi::Utils::Archive qw(get_filelist); use LANraragi::Utils::Logging qw(get_logger); @@ -335,7 +335,7 @@ sub set_title { remove_spaces($newtitle); remove_newlines($newtitle); $newtitle = redis_encode($newtitle); - $redis_search->zadd( "LRR_TITLES", 0, "$oldtitle\0$id" ); + $redis_search->zadd( "LRR_TITLES", 0, "$newtitle\0$id" ); } $redis->quit; } diff --git a/public/js/index_datatables.js b/public/js/index_datatables.js index fcd7e070d..7e3217da7 100644 --- a/public/js/index_datatables.js +++ b/public/js/index_datatables.js @@ -90,7 +90,7 @@ IndexTable.doSearch = function (page) { // Update search input field $("#search-input").val(IndexTable.currentSearch); - IndexTable.dataTable.search(IndexTable.currentSearch.replace(",", "")); + IndexTable.dataTable.search(IndexTable.currentSearch); // Add the current search terms to the title tab document.title = IndexTable.originalTitle + ((IndexTable.currentSearch !== "") ? ` - ${IndexTable.currentSearch}` : ""); From af6e4d4d7b0fa481efa71e60e9db27fca71bbc32 Mon Sep 17 00:00:00 2001 From: Difegue Date: Fri, 16 Dec 2022 01:53:47 +0100 Subject: [PATCH 24/45] (#555) Add sort functionality back title sorting is optimized thanks to the title sorted set, other namespaces less so. --- lib/LANraragi/Model/Search.pm | 47 +++++++++++++++++++++++++++-------- package.json | 2 +- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index c17e6c0f9..2a1d9633c 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -203,8 +203,32 @@ sub search_uncached { $sortkey = "title"; } - # TODO Sort by the required metadata, asc or desc - #@filtered = sort_results( $sortkey, $sortorder, @filtered ); + if ( $sortkey eq "title" ) { + my @ordered = (); + + # For title sorting, we can just use the LRR_TITLES set, which is sorted lexicographically. + if ($sortorder) { + @ordered = $redis->zrevrangebylex( "LRR_TITLES", "+", "-" ); + } else { + @ordered = $redis->zrangebylex( "LRR_TITLES", "-", "+" ); + } + + # Remove the titles from the keys, which are stored as "title\x00id" + @ordered = map { substr( $_, index( $_, "\x00" ) + 1 ) } @ordered; + + $logger->trace( "Example element from ordered list: " . $ordered[0] ); + + # Just intersect the ordered list with the filtered one to get the final result + @filtered = intersect_arrays( \@filtered, \@ordered, 0 ); + } else { + + # For other sorting, we need to get the metadata for each archive and sort it manually. + # Just use the old sort algorithm at this point. + @filtered = sort_results( $sortkey, $sortorder, @filtered ); + + # We could theoretically use the tag indexes for this by scanning them all + # to find the filtered IDs and then ordering on those namespace/ID pairs, but that's a lot of work for little gain. + } } return @filtered; @@ -328,24 +352,25 @@ sub sort_results { my ( $sortkey, $sortorder, @filtered ) = @_; + my $redis = LANraragi::Model::Config->get_redis; + @filtered = sort { - #Use either tags or title depending on the sortkey - my $meta1 = $a->{title}; - my $meta2 = $b->{title}; + my $tags_a = $redis->hget( $a, "tags" ); + my $tags_b = $redis->hget( $b, "tags" ); + + # Not a very good way to make items end at the bottom... + my $meta1 = "zzzz"; + my $meta2 = "zzzz"; if ( $sortkey ne "title" ) { my $re = qr/$sortkey/; - if ( $a->{tags} =~ m/.*${re}:(.*)(\,.*|$)/ ) { + if ( $tags_a =~ m/.*${re}:(.*)(\,.*|$)/ ) { $meta1 = $1; - } else { - $meta1 = "zzzz"; # Not a very good way to make items end at the bottom... } - if ( $b->{tags} =~ m/.*${re}:(.*)(\,.*|$)/ ) { + if ( $tags_b =~ m/.*${re}:(.*)(\,.*|$)/ ) { $meta2 = $1; - } else { - $meta2 = "zzzz"; } } diff --git a/package.json b/package.json index 0b2393f54..a28df3ffe 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "lint": "eslint --ext .js public/", "start": "perl ./script/launcher.pl -f ./script/lanraragi", "dev-server": "perl ./script/launcher.pl -m -v ./script/lanraragi", - "kill-workers": "(kill -15 `cat ./public/temp/shinobu.pid-s6` || true) && (kill -15 `cat ./public/temp/minion.pid-s6` || true)", + "kill-workers": "(kill -15 `cat ./public/temp/shinobu.pid-s6` || true) && (kill -15 `cat ./public/temp/minion.pid-s6` || true) && (pkill -9 -f ./script/lanraragi || true)", "docker-build": "docker build -t difegue/lanraragi -f ./tools/build/docker/Dockerfile .", "critic": "perlcritic ./lib/* ./script/* ./tools/install.pl", "backup-db": "perl ./script/backup", From deffcc500dda0d326f1dcf0905689e645058218b Mon Sep 17 00:00:00 2001 From: Difegue Date: Fri, 16 Dec 2022 02:08:22 +0100 Subject: [PATCH 25/45] misc. fixes --- lib/LANraragi/Model/Search.pm | 2 ++ lib/LANraragi/Model/Upload.pm | 10 +++++++--- lib/LANraragi/Utils/Database.pm | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 2a1d9633c..6e4e5f221 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -329,6 +329,8 @@ sub compute_search_filter { # Escape already present regex characters $logger->debug("Pre-escaped tag: $tag"); + remove_spaces($tag); + # Escape characters according to redis zscan rules $tag =~ s/([\[\]\^\\])/\\$1/g; diff --git a/lib/LANraragi/Model/Upload.pm b/lib/LANraragi/Model/Upload.pm index 5aa9bc30e..e2e99bc47 100644 --- a/lib/LANraragi/Model/Upload.pm +++ b/lib/LANraragi/Model/Upload.pm @@ -51,8 +51,9 @@ sub handle_incoming_file { #Check if the ID is already in the database, and #that the file it references still exists on the filesystem - my $redis = LANraragi::Model::Config->get_redis(); - my $isdupe = $redis->exists($id) && -e $redis->hget( $id, "file" ); + my $redis = LANraragi::Model::Config->get_redis; + my $redis_search = LANraragi::Model::Config->get_redis_search; + my $isdupe = $redis->exists($id) && -e $redis->hget( $id, "file" ); # Stop here if file is a dupe. if ( -e $output_file || $isdupe ) { @@ -91,7 +92,9 @@ sub handle_incoming_file { $logger->debug("Adding $url as an URL for $id"); trim_url($url); $logger->debug("Trimmed: $url"); - $redis->hset( "LRR_URLMAP", $url, $id ); # No need to encode the value, as URLs are already encoded by design + + # No need to encode the value, as URLs are already encoded by design + $redis_search->hset( "LRR_URLMAP", $url, $id ); } } } @@ -112,6 +115,7 @@ sub handle_incoming_file { LANraragi::Utils::Database::add_timestamp_tag( $redis, $id ); LANraragi::Utils::Database::add_pagecount( $redis, $id ); $redis->quit(); + $redis_search->quit(); $logger->debug("Running autoplugin on newly uploaded file $id..."); diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index 61d0de61c..085a727a8 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -338,6 +338,7 @@ sub set_title { $redis_search->zadd( "LRR_TITLES", 0, "$newtitle\0$id" ); } $redis->quit; + $redis_search->quit; } #set_tags($id, $tags, $append) From 4b1f333f7ebd7bdd20014b33b033358d3dad5b52 Mon Sep 17 00:00:00 2001 From: Difegue Date: Fri, 16 Dec 2022 23:22:10 +0100 Subject: [PATCH 26/45] Fix some warnings and noisy logs --- lib/LANraragi/Model/Plugins.pm | 2 +- lib/LANraragi/Model/Stats.pm | 3 ++- lib/LANraragi/Utils/Logging.pm | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/LANraragi/Model/Plugins.pm b/lib/LANraragi/Model/Plugins.pm index e25480be2..e7220a681 100644 --- a/lib/LANraragi/Model/Plugins.pm +++ b/lib/LANraragi/Model/Plugins.pm @@ -112,7 +112,7 @@ sub exec_login_plugin { $logger->error("Plugin doesn't implement do_login!"); } } else { - $logger->info("No login plugin specified, returning empty UserAgent."); + $logger->debug("No login plugin specified, returning empty UserAgent."); } return $ua; diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index 35cc8e2ed..eac3f2424 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -128,7 +128,8 @@ sub build_stat_hashes { $redistx->zadd( "LRR_TITLES", 0, "$title\0$id" ); } - if ( $redis->hget( $id, "isnew" ) eq "true" ) { + my $isnew = $redis->hget( $id, "isnew" ); + if ( $isnew && $isnew eq "true" ) { $logger->trace("Adding $id to LRR_ISNEW"); $redistx->sadd( "LRR_NEW", $id ); } diff --git a/lib/LANraragi/Utils/Logging.pm b/lib/LANraragi/Utils/Logging.pm index 9b0c9c639..b6b5a5501 100644 --- a/lib/LANraragi/Utils/Logging.pm +++ b/lib/LANraragi/Utils/Logging.pm @@ -57,7 +57,7 @@ sub get_logger { } # Step down into trace if we're launched from npm run dev-server - if ( $ENV{MOJO_MODE} eq "development" ) { + if ( $ENV{MOJO_MODE} && $ENV{MOJO_MODE} eq "development" ) { $log->level('trace'); } From 6f37c9584eca4d17ddc8b00eca9dbb42c3e373a0 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 00:40:42 +0100 Subject: [PATCH 27/45] Add fuzzy tag search + add a key to the search DB to indicate whether indexes are present or not --- lib/LANraragi/Model/Search.pm | 25 +++++++++++++++++++------ lib/LANraragi/Model/Stats.pm | 3 +++ lib/LANraragi/Utils/Logging.pm | 2 +- lib/LANraragi/Utils/Minion.pm | 23 ----------------------- package.json | 2 +- 5 files changed, 24 insertions(+), 31 deletions(-) diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 6e4e5f221..95d8745cc 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -29,6 +29,11 @@ sub do_search { my $redis = LANraragi::Model::Config->get_redis_search; my $logger = get_logger( "Search Engine", "lanraragi" ); + unless ( $redis->exists("LAST_JOB_TIME") ) { + $logger->error("Search engine is not initialized yet. Please wait a few seconds."); + return ( -1, -1, () ); + } + # Search filter results my $total = $redis->zcard("LRR_TITLES") + 0; # Total number of archives (as int) @@ -47,9 +52,7 @@ sub do_search { } $redis->quit(); - # If start is negative, return all possible data - # Kind of a hack for the random API, not sure how this could be improved... - # (The paging has always been there mostly to make datatables behave after all.) + # If start is negative, return all possible data. if ( $start == -1 ) { return ( $total, $#filtered + 1, @filtered ); } @@ -150,13 +153,23 @@ sub search_uncached { my @ids = (); - # Tags are always considered exact for now, so just check if an index for it exists - # TODO Add fuzzy search for tags? Probably needed if only to have wildcards work... - if ( $redis->exists("INDEX_$tag") ) { + # For exact tag searches, just check if an index for it exists + if ( $isexact && $redis->exists("INDEX_$tag") ) { # Get the list of IDs for this tag @ids = $redis->smembers("INDEX_$tag"); $logger->trace( "Found tag index for $tag, containing " . scalar @ids . " IDs" ); + } else { + + # Get index keys that match this tag + my @keys = $redis->keys("INDEX_*$tag*"); + + # Get the list of IDs for each key + foreach my $key (@keys) { + my @keyids = $redis->smembers($key); + $logger->trace( "Found index $key for $tag, containing " . scalar @ids . " IDs" ); + push @ids, @keyids; + } } # Append fuzzy title search diff --git a/lib/LANraragi/Model/Stats.pm b/lib/LANraragi/Model/Stats.pm index eac3f2424..d78027846 100644 --- a/lib/LANraragi/Model/Stats.pm +++ b/lib/LANraragi/Model/Stats.pm @@ -135,6 +135,9 @@ sub build_stat_hashes { } } + # Add a stamp to the stats hash to indicate when it was last updated + $redistx->set( "LAST_JOB_TIME", time() ); + $redistx->exec; $logger->info("Stat indexes built! ($archive_count archives)"); $redis->quit; diff --git a/lib/LANraragi/Utils/Logging.pm b/lib/LANraragi/Utils/Logging.pm index b6b5a5501..37d9f2e8e 100644 --- a/lib/LANraragi/Utils/Logging.pm +++ b/lib/LANraragi/Utils/Logging.pm @@ -57,7 +57,7 @@ sub get_logger { } # Step down into trace if we're launched from npm run dev-server - if ( $ENV{MOJO_MODE} && $ENV{MOJO_MODE} eq "development" ) { + if ( $ENV{LRR_DEVSERVER} ) { $log->level('trace'); } diff --git a/lib/LANraragi/Utils/Minion.pm b/lib/LANraragi/Utils/Minion.pm index 724b1f613..dc3b1bec0 100644 --- a/lib/LANraragi/Utils/Minion.pm +++ b/lib/LANraragi/Utils/Minion.pm @@ -84,29 +84,6 @@ sub add_tasks { } ); - $minion->add_task( - warm_cache => sub { - my ( $job, @args ) = @_; - my $logger = get_logger( "Minion", "minion" ); - - $logger->info("Warming up search cache..."); - - # Cache warm performs a search for the base index (no search) - LANraragi::Model::Search::do_search( "", "", 0, "title", "asc", 0, 0 ); - - # And for every category defined by the user. - my @categories = LANraragi::Model::Category->get_category_list; - for my $category (@categories) { - my $catid = %{$category}{"id"}; - $logger->debug("Warming category $catid"); - LANraragi::Model::Search::do_search( "", $catid, 0, "title", "asc", 0, 0 ); - } - - $logger->info("Done!"); - $job->finish; - } - ); - $minion->add_task( build_stat_hashes => sub { my ( $job, @args ) = @_; diff --git a/package.json b/package.json index a28df3ffe..aa53bf89b 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "lanraragi-installer": "perl ./tools/install.pl", "lint": "eslint --ext .js public/", "start": "perl ./script/launcher.pl -f ./script/lanraragi", - "dev-server": "perl ./script/launcher.pl -m -v ./script/lanraragi", + "dev-server": "export LRR_DEVSERVER && perl ./script/launcher.pl -m -v ./script/lanraragi", "kill-workers": "(kill -15 `cat ./public/temp/shinobu.pid-s6` || true) && (kill -15 `cat ./public/temp/minion.pid-s6` || true) && (pkill -9 -f ./script/lanraragi || true)", "docker-build": "docker build -t difegue/lanraragi -f ./tools/build/docker/Dockerfile .", "critic": "perlcritic ./lib/* ./script/* ./tools/install.pl", From 1464579acb926e3efc76bb7714c4aeaccbb009e0 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 00:41:00 +0100 Subject: [PATCH 28/45] Fix random search --- lib/LANraragi/Controller/Api/Search.pm | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index 2717882b3..ca5128da9 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -101,25 +101,25 @@ sub get_random_archives { my $filter = $req->param('filter'); my $category = $req->param('category') || ""; + my $isnew = $req->param('isnew') || ""; + my $untaggedonly = $req->param('untaggedonly') || ""; my $random_count = $req->param('count') || 5; - # TODO rework this # Use the search engine to get IDs matching the filter/category selection, with start=-1 to get all data - # This method could be extended later to also use isnew/untagged filters. - my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( $filter, $category, -1, "title", 0, "", "" ); + my ( $total, $filtered, @ids ) = + LANraragi::Model::Search::do_search( $filter, $category, -1, "title", 0, $isnew, $untaggedonly ); my @random_ids; $random_count = min( $random_count, scalar(@ids) ); - while ( $random_count > 0 ) { - my $random_id = $ids[ int( rand( scalar @ids ) ) ]; - next if ( grep { $_ eq $random_id } @random_ids ); - - push @random_ids, $random_id; - $random_count--; + # Get random IDs out of the array + for ( 1 .. $random_count ) { + my $random_index = int( rand( scalar(@ids) ) ); + push( @random_ids, splice( @ids, $random_index, 1 ) ); } - $self->render( json => { data => \@random_ids } ); + my @data = get_archive_json_multi( $redis, @random_ids ); + $self->render( json => { data => \@data } ); $redis->quit(); } From 1e25065fd6561e7b97c64919bcd259d00d81828f Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 01:49:16 +0100 Subject: [PATCH 29/45] Update search unit tests and use them to fix remaining bugs --- lib/LANraragi/Model/Search.pm | 24 +++++--- tests/mocks.pl | 113 ++++++++++++++++++++++++++++++++-- tests/search.t | 48 +++++++++------ 3 files changed, 152 insertions(+), 33 deletions(-) diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 95d8745cc..01ccd3247 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -45,6 +45,7 @@ sub do_search { unless ($cachehit) { $logger->debug("No cache available, doing a full DB parse."); + $logger->trace("No cache sdfsdavailable, doing a full DB parse."); @filtered = search_uncached( $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly ); # Cache this query in the search database @@ -158,11 +159,14 @@ sub search_uncached { # Get the list of IDs for this tag @ids = $redis->smembers("INDEX_$tag"); - $logger->trace( "Found tag index for $tag, containing " . scalar @ids . " IDs" ); + $logger->debug( "Found tag index for $tag, containing " . scalar @ids . " IDs" ); } else { - # Get index keys that match this tag - my @keys = $redis->keys("INDEX_*$tag*"); + # Get index keys that match this tag. + # If the tag has a namespace, We don't add a wildcard at the start of the tag to keep it intact. + # Otherwise, we add a wildcard at the start to match all namespaces. + my $indexkey = $tag =~ /:/ ? "INDEX_$tag*" : "INDEX_*$tag*"; + my @keys = $redis->keys($indexkey); # Get the list of IDs for each key foreach my $key (@keys) { @@ -173,7 +177,7 @@ sub search_uncached { } # Append fuzzy title search - my $namesearch = $isexact ? $tag : "*$tag*"; + my $namesearch = $isexact ? "$tag\x00*" : "*$tag*"; my $scan = -1; while ( $scan != 0 ) { @@ -211,6 +215,7 @@ sub search_uncached { } if ( $#filtered > 0 ) { + $logger->debug( "Found " . $#filtered . " results after filtering." ); if ( !$sortkey ) { $sortkey = "title"; @@ -286,7 +291,7 @@ sub compute_search_filter { if ( !$filter ) { $filter = ""; } # Special characters: - # "" for exact search (or $ but is that one really useful) + # "" for exact search (or $, but is that one really useful now?) # ?/_ for any character # * % for multiple characters # - to exclude the next tag @@ -322,12 +327,13 @@ sub compute_search_filter { $char = chop $b; } - #If last char is $, enable isexact + #If last char is $ or delimiter was ", enable isexact if ( $delimiter eq '"' ) { + $isexact = 1; + + # Quotes then $ is an accepted syntax, even though it does nothing $char = chop $b; - if ( $char eq "\$" ) { - $isexact = 1; - } else { + unless ( $char eq "\$" ) { $b = $b . $char; } } else { diff --git a/tests/mocks.pl b/tests/mocks.pl index 1e656fc0c..a5e4bd9fa 100644 --- a/tests/mocks.pl +++ b/tests/mocks.pl @@ -38,7 +38,7 @@ sub setup_redis_mock { "isnew": "false", "pagecount": 2, "progress": 10, - "tags": "character:segata sanshiro", + "tags": "character:segata sanshiro, male:very cool", "title": "Saturn Backup Cartridge - Japanese Manual", "file": "package.json" }, @@ -46,7 +46,7 @@ sub setup_redis_mock { "isnew": "false", "pagecount": 0, "progress": 34, - "tags": "character:segata", + "tags": "character:segata, female:very cool too", "title": "Saturn Backup Cartridge - American Manual", "file": "package.json" }, @@ -78,7 +78,7 @@ sub setup_redis_mock { "isnew": "false", "pagecount": 0, "progress": 0, - "tags": "parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color", + "tags": "parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color, male:very cool too", "title": "Fate GO MEMO", "file": "package.json" } @@ -91,7 +91,12 @@ sub setup_redis_mock { 'keys', # $redis->keys => get keys matching predicate in datamodel sub { shift; - my $expr = $_[0] =~ s/\?/\./gr; # Replace redis' '?' wildcards with regex '.'s + + # Replace redis' '?' wildcards with regex '.'s + my $expr = $_[0] =~ s/\?/\./gr; + + # Replace redis' '*' wildcards with regex '.*'s + $expr = $expr =~ s/\*/\.\*/gr; return grep { /$expr/ } keys %datamodel; } ); @@ -100,6 +105,9 @@ sub setup_redis_mock { $redis->mock( 'hset', sub { 1 } ); $redis->mock( 'quit', sub { 1 } ); $redis->mock( 'select', sub { 1 } ); + $redis->mock( 'flushdb', sub { 1 } ); + $redis->mock( 'zincrby', sub { 1 } ); + $redis->mock( 'watch', sub { 1 } ); $redis->mock( 'hlen', sub { 1337 } ); $redis->mock( 'dbsize', sub { 1337 } ); @@ -134,6 +142,101 @@ sub setup_redis_mock { } ); + $redis->mock( + 'sadd', # $redis->sadd => add value to list named by key in datamodel + sub { + my $self = shift; + my ( $key, $value ) = @_; + + if ( !exists $datamodel{$key} ) { + $datamodel{$key} = []; + } + + push @{ $datamodel{$key} }, $value; + } + ); + + $redis->mock( + 'zadd', # $redis->zadd => add value to list named by key in datamodel + sub { + my $self = shift; + my ( $key, $weight, $value ) = @_; + + if ( !exists $datamodel{$key} ) { + $datamodel{$key} = []; + } + + push @{ $datamodel{$key} }, $value; + } + ); + + $redis->mock( + 'zcard', # $redis->zcard => number of values in list named by key in datamodel + sub { + my $self = shift; + my ($key) = @_; + + if ( !exists $datamodel{$key} ) { + $datamodel{$key} = []; + } + + return scalar @{ $datamodel{$key} }; + } + ); + + $redis->mock( + 'zrangebylex', # $redis->zrangebylex => get all values of key in datamodel + sub { + my $self = shift; + my ( $key, $start, $end ) = @_; + + if ( !exists $datamodel{$key} ) { + $datamodel{$key} = []; + } + + # Return array, ordered alphabetically + return sort @{ $datamodel{$key} }; + } + ); + + $redis->mock( + 'zscan', # $redis->zscan => get all values of key in datamodel + sub { + my $self = shift; + my ( $key, $cursor, $match, $matchexpr, $count, $countnumber ) = @_; + + if ( !exists $datamodel{LRR_TITLES} ) { + $datamodel{LRR_TITLES} = []; + } + + # Search for the match expression in the list of titles + # Replace redis' '?' wildcards with regex '.'s + my $expr = $matchexpr =~ s/\?/\./gr; + + # Replace redis' '*' wildcards with regex '.*'s + $expr = $expr =~ s/\*/\.\*/gr; + + my @matches = grep { /$expr/i } @{ $datamodel{LRR_TITLES} }; + + # Return 0 for cursor to indicate we scanned everything, and + return ( 0, \@matches ); + } + ); + + $redis->mock( + 'smembers', # $redis->smembers => return all values of key in datamodel + sub { + my $self = shift; + my ($key) = @_; + + if ( !exists $datamodel{$key} ) { + $datamodel{$key} = []; + } + + return @{ $datamodel{$key} }; + } + ); + $redis->mock( 'hgetall', # $redis->hgetall => get all values of key in datamodel sub { @@ -156,7 +259,7 @@ sub setup_redis_mock { sub get_logger_mock { my $mock = Test::MockObject->new(); - $mock->mock( 'debug', sub { }, 'info', sub { } ); + $mock->mock( 'debug', sub { }, 'info', sub { }, 'trace', sub { } ); return $mock; } diff --git a/tests/search.t b/tests/search.t index 5981fd5f1..58c246aa1 100644 --- a/tests/search.t +++ b/tests/search.t @@ -13,6 +13,7 @@ use Data::Dumper; use LANraragi::Model::Search; use LANraragi::Model::Config; +use LANraragi::Model::Stats; # Mock Redis my $cwd = getcwd; @@ -23,6 +24,9 @@ my $redis = LANraragi::Model::Config->get_redis; is( $redis->hget( "28697b96f0ac5858be2614ed10ca47742c9522fd", "title" ), "Fate GO MEMO", 'Redis mock test' ); +# Build search hashes +LANraragi::Model::Stats::build_stat_hashes(); + # Search queries my $search = ""; my ( $total, $filtered, @ids ); @@ -36,11 +40,19 @@ is( $filtered, 6, qq(Empty search(full index)) ); $search = qq(Ghost in the Shell); do_test_search(); -is( %{ $ids[0] }{title}, "Ghost in the Shell 1.5 - Human-Error Processor vol01ch01", qq(Basic search ($search)) ); +is( $ids[0], "4857fd2e7c00db8b0af0337b94055d8445118630", qq(Basic search ($search)) ); -$search = qq("Fate GO MEMO"); +$search = qq("male:very cool"); +do_test_search(); +is( $filtered, 1, qq(Exact namespace search ($search)) ); + +$search = qq(male:very cool); do_test_search(); -is( $filtered, 2, qq(Non-exact quoted search ($search)) ); +is( $filtered, 2, qq(Fuzzy namespace search ($search)) ); + +$search = qq(*male:very cool); +do_test_search(); +is( $filtered, 3, qq(Very fuzzy namespace search ($search)) ); $search = qq("Fate GO MEMO ?"); do_test_search(); @@ -50,25 +62,25 @@ $search = qq("Fate GO MEMO _"); do_test_search(); is( $filtered, 1, qq(Wildcard search ($search)) ); -$search = qq("Saturn*Cartridge*Japanese"); +$search = qq("Saturn*Cartridge*Japanese Manual"); do_test_search(); -is( %{ $ids[0] }{title}, "Saturn Backup Cartridge - Japanese Manual", qq(Multiple wildcard search ($search)) ); +is( $ids[0], "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf", qq(Multiple wildcard search ($search)) ); -$search = qq("Saturn\%American"); +$search = qq("Saturn\%American\%"); do_test_search(); -is( %{ $ids[0] }{title}, "Saturn Backup Cartridge - American Manual", qq(Multiple wildcard search ($search)) ); +is( $ids[0], "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg", qq(Multiple wildcard search ($search)) ); -$search = qq("artist:wada rco" character:ereshkigal); +$search = qq(artist:wada rco, character:ereshkigal); do_test_search(); -ok( $filtered eq 1 && %{ $ids[0] }{title} eq "Fate GO MEMO 2", qq(Tag inclusion search ($search)) ); +ok( $filtered eq 1 && $ids[0] eq "2810d5e0a8d027ecefebca6237031a0fa7b91eb3", qq(Tag inclusion search ($search)) ); -$search = qq("artist:wada rco" -character:ereshkigal); +$search = qq(artist:wada rco, -character:ereshkigal); do_test_search(); -ok( $filtered eq 1 && %{ $ids[0] }{title} eq "Fate GO MEMO", qq(Tag exclusion search ($search)) ); +ok( $filtered eq 1 && $ids[0] eq "28697b96f0ac5858be2614ed10ca47742c9522fd", qq(Tag exclusion search ($search)) ); -$search = qq("artist:wada rco" -"character:waver velvet"); +$search = qq(artist:wada rco, -character:waver velvet); do_test_search(); -ok( $filtered eq 1 && %{ $ids[0] }{title} eq "Fate GO MEMO", qq(Tag exclusion with quotes ($search)) ); +ok( $filtered eq 1 && $ids[0] eq "28697b96f0ac5858be2614ed10ca47742c9522fd", qq(Tag exclusion with quotes ($search)) ); $search = qq("artist:wada rco" "-character:waver velvet"); do_test_search(); @@ -76,10 +88,9 @@ is( $filtered, 0, qq(Incorrect tag exclusion ($search)) ); $search = qq(character:segata\$); do_test_search(); -ok( $filtered eq 1 && %{ $ids[0] }{title} eq "Saturn Backup Cartridge - American Manual", - qq(Exact search without quotes ($search)) ); +ok( $filtered eq 1 && $ids[0] eq "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg", qq(Exact search without quotes ($search)) ); -$search = qq("Fate GO MEMO"\$); +$search = qq("Fate GO MEMO"); do_test_search(); is( $filtered, 1, qq(Exact search with quotes ($search)) ); @@ -95,10 +106,9 @@ $search = qq("character:segata"); is( $filtered, 1, qq(Search with favorite search category applied ($search) + (SET_1589138380: American)) ); ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( "", "", 0, 0, 0, 1, 0 ); -ok( $filtered eq 1 && %{ $ids[0] }{title} eq "Rohan Kishibe goes to Gucci", qq(Search with new filter applied) ); +ok( $filtered eq 1 && $ids[0] eq "e4c422fd10943dc169e3489a38cdbf57101a5f7e", qq(Search with new filter applied) ); ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( "", "", 0, 0, 0, 0, 1 ); -ok( $filtered eq 2 && %{ $ids[0] }{title} eq "Ghost in the Shell 1.5 - Human-Error Processor vol01ch01", - qq(Search with untagged filter applied) ); +ok( $filtered eq 2 && $ids[0] eq "4857fd2e7c00db8b0af0337b94055d8445118630", qq(Search with untagged filter applied) ); done_testing(); From f35f41e46d316c44890e1fa3a08fcc297b6621db Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 02:23:19 +0100 Subject: [PATCH 30/45] (#709) Carousel now auto-reloads with every new search + has a nicer empty view Also update FA to v6 --- package.json | 2 +- public/js/index.js | 9 +++++++-- public/js/index_datatables.js | 3 +++ templates/index.html.tt2 | 11 ++++++++--- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index aa53bf89b..7ba467e58 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ }, "homepage": "https://github.com/Difegue/LANraragi#readme", "dependencies": { - "@fortawesome/fontawesome-free": "^5.15.4", + "@fortawesome/fontawesome-free": "^6.2.1", "@jcubic/tagger": "^0.4.2", "allcollapsible": "^1.1.0", "awesomplete": "^1.1.5", diff --git a/public/js/index.js b/public/js/index.js index 7168b192c..06eca266a 100644 --- a/public/js/index.js +++ b/public/js/index.js @@ -311,9 +311,10 @@ Index.handleCustomSort = function () { Index.updateCarousel = function (e) { e?.preventDefault(); - + $("#carousel-empty").hide(); $("#carousel-loading").show(); $(".swiper-wrapper").hide(); + $("#reload-carousel").addClass("fa-spin"); // Hit a different API endpoint depending on the requested localStorage carousel type @@ -346,10 +347,14 @@ Index.updateCarousel = function (e) { (results) => { Index.swiper.virtual.removeAllSlides(); const slides = results.data - .map((archive) => LRR.buildThumbnailDiv(archive, false)); + .map((archive) => LRR.buildThumbnailDiv(archive)); Index.swiper.virtual.appendSlide(slides); Index.swiper.virtual.update(); + if (results.data.length === 0) { + $("#carousel-empty").show(); + } + $("#carousel-loading").hide(); $(".swiper-wrapper").show(); $("#reload-carousel").removeClass("fa-spin"); diff --git a/public/js/index_datatables.js b/public/js/index_datatables.js index 7e3217da7..31800a6a8 100644 --- a/public/js/index_datatables.js +++ b/public/js/index_datatables.js @@ -107,6 +107,9 @@ IndexTable.doSearch = function (page) { // Re-load categories so the most recently selected/created ones appear first Index.loadCategories(); + + // Re-load carousel + Index.updateCarousel(); }; // #region Compact View diff --git a/templates/index.html.tt2 b/templates/index.html.tt2 index 274c34174..c4c586d23 100644 --- a/templates/index.html.tt2 +++ b/templates/index.html.tt2 @@ -24,7 +24,7 @@ - + @@ -113,7 +113,12 @@ style="height: 344px; display:flex; justify-content: center; align-items: center;"> - + @@ -192,7 +197,7 @@ heading: 'You\'re using the default password and that\'s super baka of you', text: 'Login with password "kamimamita" and change that shit on the double.
...Or just disable it!
Why not check the configuration options afterwards, while you\'re at it? ', icon: 'warning', - hideAfter: 25000, + hideAfter: 25000, closeOnClick: false, draggable: false, }); From e3165bfef03c830561f12904005c13333afb916d Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 21:42:58 +0100 Subject: [PATCH 31/45] Cleanup remaining unused redis objects and update docs for random API --- lib/LANraragi/Controller/Api/Search.pm | 24 ++++++------- lib/LANraragi/Utils/Database.pm | 2 +- .../api-documentation/search-api.md | 35 +++++++++---------- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index ca5128da9..6d091c73b 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -10,9 +10,8 @@ use LANraragi::Utils::Database qw(invalidate_cache get_archive_json_multi); # Undocumented API matching the Datatables spec. sub handle_datatables { - my $self = shift; - my $redis = $self->LRR_CONF->get_redis(); - my $req = $self->req; + my $self = shift; + my $req = $self->req; my $draw = $req->param('draw'); my $start = $req->param('start'); @@ -54,7 +53,7 @@ sub handle_datatables { my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( $filter, $categoryfilter, $start, $sortkey, $sortorder, $newfilter, $untaggedfilter ); - $self->render( json => get_datatables_object( $draw, $redis, $total, $filtered, @ids ) ); + $self->render( json => get_datatables_object( $draw, $total, $filtered, @ids ) ); $redis->quit(); } @@ -62,9 +61,8 @@ sub handle_datatables { # Public search API with saner parameters. sub handle_api { - my $self = shift; - my $redis = $self->LRR_CONF->get_redis(); - my $req = $self->req; + my $self = shift; + my $req = $self->req; my $filter = $req->param('filter'); my $category = $req->param('category') || ""; @@ -82,7 +80,7 @@ sub handle_api { $untaggedf eq "true" ); - $self->render( json => get_datatables_object( 0, $redis, $total, $filtered, @ids ) ); + $self->render( json => get_datatables_object( 0, $total, $filtered, @ids ) ); $redis->quit(); } @@ -95,9 +93,8 @@ sub clear_cache { # Pull random archives out of the given search sub get_random_archives { - my $self = shift; - my $redis = $self->LRR_CONF->get_redis(); - my $req = $self->req; + my $self = shift; + my $req = $self->req; my $filter = $req->param('filter'); my $category = $req->param('category') || ""; @@ -118,16 +115,15 @@ sub get_random_archives { push( @random_ids, splice( @ids, $random_index, 1 ) ); } - my @data = get_archive_json_multi( $redis, @random_ids ); + my @data = get_archive_json_multi(@random_ids); $self->render( json => { data => \@data } ); - $redis->quit(); } # get_datatables_object($draw, $total, $totalsearched, @pagedkeys) # Creates a Datatables-compatible json from the given data. sub get_datatables_object { - my ( $draw, $redis, $total, $filtered, @ids ) = @_; + my ( $draw, $total, $filtered, @ids ) = @_; # Get archive data my @data = get_archive_json_multi(@ids); diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index 085a727a8..3a9e2fc53 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -130,7 +130,7 @@ sub get_archive_json { return $arcdata; } -# get_archive_json_multi(redis, ids) +# get_archive_json_multi(ids) # Uses Redis' MULTI to get an archive JSON for each ID. sub get_archive_json_multi { my @ids = @_; diff --git a/tools/Documentation/api-documentation/search-api.md b/tools/Documentation/api-documentation/search-api.md index c93be270d..1e43a2b67 100644 --- a/tools/Documentation/api-documentation/search-api.md +++ b/tools/Documentation/api-documentation/search-api.md @@ -142,18 +142,11 @@ ID of the category you want to restrict this search to. {% endswagger-parameter %} {% swagger-parameter name="filter" type="string" required="false" in="query" %} -Search query. This follows the same rules as the queries in - -`/api/search` - -. +Search query. This follows the same rules as the queries in `/api/search`. {% endswagger-parameter %} {% swagger-parameter name="count" type="int" required="false" in="query" %} -How many archives you want to pull randomly. Defaults to 5. - -\ - +How many archives you want to pull randomly. Defaults to 5. If the search doesn't return enough data to match your count, you will get the full search shuffled randomly. {% endswagger-parameter %} @@ -163,23 +156,27 @@ If the search doesn't return enough data to match your count, you will get the f { "data": [ { - "id": "28697b96f0ac5858be2614ed10ca47742c9522fd", - "tags": "parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color", - "title": "Fate GO MEMO" - }, { - "id": "2810d5e0a8d027ecefebca6237031a0fa7b91eb3", - "tags": "parody:fate grand order, character:abigail williams, character:artoria pendragon alter, character:asterios, character:ereshkigal, character:gilgamesh, character:hans christian andersen, character:hassan of serenity, character:hector, character:helena blavatsky, character:irisviel von einzbern, character:jeanne alter, character:jeanne darc, character:kiara sessyoin, character:kiyohime, character:lancer, character:martha, character:minamoto no raikou, character:mochizuki chiyome, character:mordred pendragon, character:nitocris, character:oda nobunaga, character:osakabehime, character:penthesilea, character:queen of sheba, character:rin tosaka, character:saber, character:sakata kintoki, character:scheherazade, character:sherlock holmes, character:suzuka gozen, character:tamamo no mae, character:ushiwakamaru, character:waver velvet, character:xuanzang, character:zhuge liang, group:wadamemo, artist:wada rco, artbook, full color", - "title": "Fate GO MEMO 2" + "arcid": "2810d5e0a8d027ecefebca6237031a0fa7b91eb3", + "isnew": "none", + "extension": "rar", + "tags": "parody:fate grand order, character:abigail williams, character:artoria pendragon alter, character:asterios, character:ereshkigal, character:gilgamesh, character:hans christian andersen, character:hassan of serenity, character:hector, character:helena blavatsky, character:irisviel von einzbern, character:jeanne alter, character:jeanne darc, character:kiara sessyoin, character:kiyohime, character:lancer, character:martha, character:minamoto no raikou, character:mochizuki chiyome, character:mordred pendragon, character:nitocris, character:oda nobunaga, character:osakabehime, character:penthesilea, character:queen of sheba, character:rin tosaka, character:saber, character:sakata kintoki, character:scheherazade, character:sherlock holmes, character:suzuka gozen, character:tamamo no mae, character:ushiwakamaru, character:waver velvet, character:xuanzang, character:zhuge liang, group:wadamemo, artist:wada rco, artbook, full color", + "title": "Fate GO MEMO 2" }, { - "id": "4857fd2e7c00db8b0af0337b94055d8445118630", + "arcid": "4857fd2e7c00db8b0af0337b94055d8445118630", + "isnew": "none", + "extension": "pdf", "tags": "artist:shirow masamune", "title": "Ghost in the Shell 1.5 - Human-Error Processor vol01ch01" }, { - "id": "e4c422fd10943dc169e3489a38cdbf57101a5f7e", + "arcid": "e4c422fd10943dc169e3489a38cdbf57101a5f7e", + "isnew": "none", + "extension": "epub", "tags": "parody: jojo's bizarre adventure", "title": "Rohan Kishibe goes to Gucci" }, { - "id": "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf", + "arcid": "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf", + "isnew": "none", + "extension": "lzma", "tags": "character:segata sanshiro", "title": "Saturn Backup Cartridge - Japanese Manual" } From ef1c08797d79945f1cad83fcacac944c0ab8368a Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 21:50:33 +0100 Subject: [PATCH 32/45] Add query parameters to login redirects --- lib/LANraragi/Controller/Login.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/LANraragi/Controller/Login.pm b/lib/LANraragi/Controller/Login.pm index f628168e1..b4a4a76c9 100644 --- a/lib/LANraragi/Controller/Login.pm +++ b/lib/LANraragi/Controller/Login.pm @@ -48,7 +48,7 @@ sub logged_in { || $self->LRR_CONF->enable_pass == 0; my $url = $self->url_for("login"); - $self->redirect_to( $url->query( redirect => $self->req->url->path ) ); + $self->redirect_to( $url->query( redirect => $self->req->url->path_query ) ); return 0; } From 9334b191c4fda983cf0b06622a7d3ae97759b838 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 21:57:54 +0100 Subject: [PATCH 33/45] Fix tags with case not working in search --- lib/LANraragi/Controller/Api/Search.pm | 4 ---- lib/LANraragi/Model/Search.pm | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index 6d091c73b..aab4f5779 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -54,8 +54,6 @@ sub handle_datatables { LANraragi::Model::Search::do_search( $filter, $categoryfilter, $start, $sortkey, $sortorder, $newfilter, $untaggedfilter ); $self->render( json => get_datatables_object( $draw, $total, $filtered, @ids ) ); - $redis->quit(); - } # Public search API with saner parameters. @@ -81,8 +79,6 @@ sub handle_api { ); $self->render( json => get_datatables_object( 0, $total, $filtered, @ids ) ); - $redis->quit(); - } sub clear_cache { diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 01ccd3247..a026777e9 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -361,7 +361,7 @@ sub compute_search_filter { $tag =~ s/\%/\*/g; push @tokens, - { tag => $tag, + { tag => lc($tag), isneg => $isneg, isexact => $isexact }; From 5ea6a29c52546956f802ee1652e88af537fe37e2 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 22:24:24 +0100 Subject: [PATCH 34/45] Fix clear all new not emptying the matching search index --- lib/LANraragi/Controller/Api/Database.pm | 11 ++-- tests/samples/opds/opds_sample.xml | 50 +++++++++---------- .../basic-operations/searching.md | 7 ++- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Database.pm b/lib/LANraragi/Controller/Api/Database.pm index 97080da70..fd3ed2ceb 100644 --- a/lib/LANraragi/Controller/Api/Database.pm +++ b/lib/LANraragi/Controller/Api/Database.pm @@ -45,8 +45,9 @@ sub clean_database { #Clear new flag in all archives. sub clear_new_all { - my $self = shift; - my $redis = $self->LRR_CONF->get_redis(); + my $self = shift; + my $redis = $self->LRR_CONF->get_redis(); + my $redis_search = $self->LRR_CONF->get_redis_search(); # Get all archives thru redis # 40-character long keys only => Archive IDs @@ -56,8 +57,12 @@ sub clear_new_all { $redis->hset( $idall, "isnew", "false" ); } - # Bust isnew cache $redis->quit(); + + # Bust isnew cache + $redis_search->del("LRR_NEW"); + $redis_search->quit(); + render_api_response( $self, "clear_new_all" ); } diff --git a/tests/samples/opds/opds_sample.xml b/tests/samples/opds/opds_sample.xml index eefdbba9e..12cdf4b21 100644 --- a/tests/samples/opds/opds_sample.xml +++ b/tests/samples/opds/opds_sample.xml @@ -1,28 +1,28 @@ + xmlns:dcterms="http://purl.org/dc/terms/" + xmlns:opds="http://opds-spec.org/2010/catalog"> urn:lrr:0 + href="/api/opds" + type="application/atom+xml;profile=opds-catalog;kind=acquisition"/> + href="/api/opds" + type="application/atom+xml;profile=opds-catalog;kind=acquisition"/> LANraragi 2010-01-10T10:03:10Z Welcome to this Library running LANraragi! /favicon.ico - 9.9.9 - http://github.org/Difegue/LANraragi + 9.9.9 + http://github.org/Difegue/LANraragi - + Fate GO MEMO urn:lrr:28697b96f0ac5858be2614ed10ca47742c9522fd @@ -38,19 +38,19 @@ - parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color + parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color, male:very cool too - + - + Fate GO MEMO 2 urn:lrr:2810d5e0a8d027ecefebca6237031a0fa7b91eb3 @@ -71,14 +71,14 @@ - + - + Ghost in the Shell 1.5 - Human-Error Processor vol01ch01 urn:lrr:4857fd2e7c00db8b0af0337b94055d8445118630 @@ -99,14 +99,14 @@ - + - + Rohan Kishibe goes to Gucci urn:lrr:e4c422fd10943dc169e3489a38cdbf57101a5f7e @@ -127,14 +127,14 @@ - + - + Saturn Backup Cartridge - American Manual urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebg @@ -150,19 +150,19 @@ - character:segata + character:segata, female:very cool too - + - + Saturn Backup Cartridge - Japanese Manual urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebf @@ -178,18 +178,18 @@ - character:segata sanshiro + character:segata sanshiro, male:very cool - + - - + + diff --git a/tools/Documentation/basic-operations/searching.md b/tools/Documentation/basic-operations/searching.md index e78e92a66..4658ead7e 100644 --- a/tools/Documentation/basic-operations/searching.md +++ b/tools/Documentation/basic-operations/searching.md @@ -4,10 +4,13 @@ The search bar in LANraragi tries to not be too dumb and will actively suggest t ![Search suggestions](../.screenshots/search.png) +If you want to queue multiple terms/tags, you have to use **commas** to separate them, much like how tags are entered in the metadata field when editing an archive. +You can mix both tags and a specific title in a search if you want. + You can also use the following special characters in a search: **Quotation Marks ("...")**\ -Exact string search. Allows a search term to include spaces. Everything placed inside a pair of quotation marks is treated as a singular term. Wildcard characters are still interpreted as wildcards. +Exact string search. Everything placed inside a pair of quotation marks is treated as a singular term. Wildcard characters are still interpreted as wildcards. **Question Mark (?), Underscore (\_)**\ Wildcard. Can match any single character. @@ -19,4 +22,4 @@ Wildcard. Can match any sequence of characters (including none). Exclusion. When placed before a term, prevents search results from including that term. **Dollar Sign ($)**\ -Add at the end of a tag to perform an exact tag search rather than displaying all elements that start with the term. Only matches tags regardless of search parameters and can be used as an exclusion to ignore misc tags in the search query. +Add at the end of a tag to perform an exact tag search rather than displaying all elements that start with the term. Can be used as an exclusion to ignore misc tags in the search query. From fac6f869e823aa4349a3438cae7e7ba5ee820ad8 Mon Sep 17 00:00:00 2001 From: Difegue Date: Sun, 18 Dec 2022 23:27:41 +0100 Subject: [PATCH 35/45] Remove Model::find_untagged_archive --- lib/LANraragi/Controller/Api/Archive.pm | 10 +++++++--- lib/LANraragi/Model/Search.pm | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Archive.pm b/lib/LANraragi/Controller/Api/Archive.pm index 0edfb1ee4..505373ffc 100644 --- a/lib/LANraragi/Controller/Api/Archive.pm +++ b/lib/LANraragi/Controller/Api/Archive.pm @@ -36,9 +36,13 @@ sub serve_archivelist { } sub serve_untagged_archivelist { - my $self = shift; - my @idlist = LANraragi::Model::Archive::find_untagged_archives; - $self->render( json => \@idlist ); + my $self = shift; + my $redis = $self->LRR_CONF->get_redis_search; + + my @untagged = $redis->smembers("LRR_UNTAGGED"); + $redis->quit; + + $self->render( json => \@untagged ); } sub serve_metadata { diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index a026777e9..244e36379 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -132,7 +132,7 @@ sub search_uncached { # If the untagged filter is enabled, call the untagged files API if ($untaggedonly) { - my @untagged = LANraragi::Model::Archive::find_untagged_archives(); + my @untagged = $redis->smembers("LRR_UNTAGGED"); @filtered = intersect_arrays( \@untagged, \@filtered, 0 ); } From c04c31006a22e54e364ef25f810885d606f8bf45 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 00:44:32 +0100 Subject: [PATCH 36/45] Refactor OPDS API + Add Categories and PSE 1.1 support --- README.md | 6 +- lib/LANraragi/Controller/Api/Other.pm | 21 +- lib/LANraragi/Model/Archive.pm | 81 +--- lib/LANraragi/Model/Opds.pm | 137 ++++++ lib/LANraragi/Model/Search.pm | 2 +- lib/LANraragi/Utils/Routing.pm | 6 +- templates/opds.html.tt2 | 124 +++--- templates/opds_entry.html.tt2 | 31 +- tests/mocks.pl | 6 +- tests/modules.t | 82 ++-- tests/opds.t | 17 +- tests/samples/opds/opds_sample.xml | 414 ++++++++++-------- .../advanced-usage/external-readers.md | 9 +- 13 files changed, 526 insertions(+), 410 deletions(-) create mode 100644 lib/LANraragi/Model/Opds.pm diff --git a/README.md b/README.md index 817572ab9..5980091b2 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,6 @@ Open source server for archival of comics/manga, running on Mojolicious + Redis. #### [📄 Documentation](https://sugoi.gitbook.io/lanraragi/v/dev) | [⏬ Download](https://github.com/Difegue/LANraragi/releases/latest) | [🎞 Demo](https://lrr.tvc-16.science) | [🪟🌃 Windows Nightlies](https://nightly.link/Difegue/LANraragi/workflows/push-continous-delivery/dev) | [💵 Sponsor Development](https://ko-fi.com/T6T2UP5N) -## The 2021 User Survey results have landed! - -Check [here](https://tvc-16.science/lrr-survey-3-results.html) for more info. - ## Screenshots |Main Page, Thumbnail View | Main Page, List View | @@ -43,7 +39,7 @@ Check [here](https://tvc-16.science/lrr-survey-3-results.html) for more info. * Read archives directly from your web browser: the server reads from within compressed files using temporary folders. -* Read your archives in dedicated reader software using the built-in OPDS Catalog +* Read your archives in dedicated reader software using the built-in OPDS Catalog (now with PSE support!) * Use the Client API to interact with LANraragi from other programs diff --git a/lib/LANraragi/Controller/Api/Other.pm b/lib/LANraragi/Controller/Api/Other.pm index 35f7b76aa..b4b595bcf 100644 --- a/lib/LANraragi/Controller/Api/Other.pm +++ b/lib/LANraragi/Controller/Api/Other.pm @@ -5,6 +5,7 @@ use Mojo::JSON qw(encode_json decode_json); use Redis; use LANraragi::Model::Stats; +use LANraragi::Model::Opds; use LANraragi::Utils::TempFolder qw(get_tempsize clean_temp_full); use LANraragi::Utils::Generic qw(render_api_response); use LANraragi::Utils::Plugins qw(get_plugin get_plugins get_plugin_parameters use_plugin); @@ -37,11 +38,25 @@ sub serve_serverinfo { ); } -sub serve_opds { +# Basic OPDS catalog +sub serve_opds_catalog { my $self = shift; + $self->render( text => LANraragi::Model::Opds::generate_opds_catalog($self), format => 'xml' ); +} + +sub serve_opds_item { + my $self = shift; + my $id = $self->stash('id'); + $self->render( text => LANraragi::Model::Opds::generate_opds_item( $self, $id ), format => 'xml' ); +} + +# OPDS-PSE specific endpoint +sub serve_opds_page { + my $self = shift; + my $id = $self->stash('id'); + my $page = $self->req->param('page') || 1; - # TODO: Move to LRR::Model::Opds - $self->render( text => LANraragi::Model::Archive::generate_opds_catalog($self), format => 'xml' ); + LANraragi::Model::Opds::render_archive_page( $self, $id, $page ); } #Remove temp dir. diff --git a/lib/LANraragi/Model/Archive.pm b/lib/LANraragi/Model/Archive.pm index 835276eb4..4fb4e33aa 100644 --- a/lib/LANraragi/Model/Archive.pm +++ b/lib/LANraragi/Model/Archive.pm @@ -10,9 +10,8 @@ use Time::HiRes qw(usleep); use File::Basename; use File::Copy "cp"; use File::Path qw(make_path); -use Mojo::Util qw(xml_escape); -use LANraragi::Utils::Generic qw(get_tag_with_namespace remove_spaces remove_newlines render_api_response); +use LANraragi::Utils::Generic qw(remove_spaces remove_newlines render_api_response); use LANraragi::Utils::TempFolder qw(get_temp); use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Utils::Archive qw(extract_single_file extract_thumbnail); @@ -31,84 +30,6 @@ sub generate_archive_list { return get_archive_json_multi(@keys); } -sub generate_opds_catalog { - - my $mojo = shift; - my $redis = $mojo->LRR_CONF->get_redis; - my @keys = (); - - # Detailed pages just return a single entry instead of all the archives. - if ( $mojo->req->param('id') ) { - @keys = ( $mojo->req->param('id') ); - } else { - @keys = $redis->keys('????????????????????????????????????????'); - } - - my @list = (); - - foreach my $id (@keys) { - my $file = $redis->hget( $id, "file" ); - if ( -e $file ) { - my $arcdata = get_archive_json( $redis, $id ); - unless ($arcdata) { next; } - - my $tags = $arcdata->{tags}; - - # Infer a few OPDS-related fields from the tags - $arcdata->{dateadded} = get_tag_with_namespace( "date_added", $tags, "2010-01-10T10:01:11Z" ); - $arcdata->{author} = get_tag_with_namespace( "artist", $tags, "" ); - $arcdata->{language} = get_tag_with_namespace( "language", $tags, "" ); - $arcdata->{circle} = get_tag_with_namespace( "group", $tags, "" ); - $arcdata->{event} = get_tag_with_namespace( "event", $tags, "" ); - - # Application/zip is universally hated by all readers so it's better to use x-cbz and x-cbr here. - if ( $file =~ /^(.*\/)*.+\.(pdf)$/ ) { - $arcdata->{mimetype} = "application/pdf"; - } elsif ( $file =~ /^(.*\/)*.+\.(rar|cbr)$/ ) { - $arcdata->{mimetype} = "application/x-cbr"; - } elsif ( $file =~ /^(.*\/)*.+\.(epub)$/ ) { - $arcdata->{mimetype} = "application/epub+zip"; - } else { - $arcdata->{mimetype} = "application/x-cbz"; - } - - for ( values %{$arcdata} ) { $_ = xml_escape($_); } - - push @list, $arcdata; - } - } - - $redis->quit; - - if ( $mojo->req->param('id') ) { - @keys = ( $mojo->req->param('id') ); - } else { - @keys = $redis->keys('????????????????????????????????????????'); - } - - # Sort list to get reproducible results - @list = sort { lc( $a->{title} ) cmp lc( $b->{title} ) } @list; - - return $mojo->render_to_string( - template => $mojo->req->param('id') ? "opds_entry" : "opds", - arclist => \@list, - arc => $mojo->req->param('id') ? $list[0] : "", - title => $mojo->LRR_CONF->get_htmltitle, - motd => $mojo->LRR_CONF->get_motd, - version => $mojo->LRR_VERSION - ); -} - -# Return a list of archive IDs that have no tags. -# TODO: Move this simple call to the API and integrate the smembers call in the search engine itself -sub find_untagged_archives { - - my $redis = LANraragi::Model::Config->get_redis_search; - my @untagged = $redis->smembers("LRR_UNTAGGED"); - $redis->quit; - return @untagged; -} - sub update_thumbnail { my ( $self, $id ) = @_; diff --git a/lib/LANraragi/Model/Opds.pm b/lib/LANraragi/Model/Opds.pm new file mode 100644 index 000000000..c1a760f0f --- /dev/null +++ b/lib/LANraragi/Model/Opds.pm @@ -0,0 +1,137 @@ +package LANraragi::Model::Opds; + +use strict; +use warnings; +use utf8; + +use Redis; +use Mojo::Util qw(xml_escape); + +use LANraragi::Utils::Generic qw(get_tag_with_namespace); +use LANraragi::Utils::Archive qw(get_filelist); +use LANraragi::Utils::Database qw(get_archive_json ); +use LANraragi::Model::Category; +use LANraragi::Model::Search; + +sub generate_opds_catalog { + + my $mojo = shift; + my $cat_id = $mojo->req->param('category') || ""; + my @cats = LANraragi::Model::Category->get_category_list; + + # Use the search engine to get the list of archives to show in the catalog. + my ( $total, $filtered, @keys ) = LANraragi::Model::Search::do_search( "", $cat_id, -1, "title", 0, 0, 0 ); + + my @list = (); + + foreach my $id (@keys) { + my $arcdata = get_opds_data($id); + push @list, $arcdata if $arcdata; + } + + foreach my $cat (@cats) { + + # If the category doesn't have a search string, we can add the total count of archives to the entry. + if ( $cat->{search} eq "" ) { + $cat->{count} = scalar @{ $cat->{archives} }; + } + + if ( $cat->{id} eq $cat_id ) { + $cat->{active} = 1; + } + } + + # Sort lists to get reproducible results + @list = sort { lc( $a->{title} ) cmp lc( $b->{title} ) } @list; + @cats = sort { lc( $a->{name} ) cmp lc( $b->{name} ) } @cats; + + return $mojo->render_to_string( + template => "opds", + arclist => \@list, + catlist => \@cats, + nocat => $cat_id eq "", + title => $mojo->LRR_CONF->get_htmltitle, + motd => $mojo->LRR_CONF->get_motd, + version => $mojo->LRR_VERSION + ); +} + +sub generate_opds_item { + + my ( $mojo, $id ) = @_; + + # Detailed pages just return a single entry instead of all the archives. + my $arcdata = get_opds_data($id); + + return $mojo->render_to_string( + template => "opds_entry", + arc => $arcdata, + title => $mojo->LRR_CONF->get_htmltitle, + motd => $mojo->LRR_CONF->get_motd, + version => $mojo->LRR_VERSION + ); +} + +sub get_opds_data { + + my $id = shift; + my $redis = LANraragi::Model::Config->get_redis; + + my $file = $redis->hget( $id, "file" ); + unless ( -e $file ) { return; } + + my $arcdata = get_archive_json( $redis, $id ); + unless ($arcdata) { return; } + + my $tags = $arcdata->{tags}; + + # Infer a few OPDS-related fields from the tags + $arcdata->{dateadded} = get_tag_with_namespace( "date_added", $tags, "2010-01-10T10:01:11Z" ); + $arcdata->{author} = get_tag_with_namespace( "artist", $tags, "" ); + $arcdata->{language} = get_tag_with_namespace( "language", $tags, "" ); + $arcdata->{circle} = get_tag_with_namespace( "group", $tags, "" ); + $arcdata->{event} = get_tag_with_namespace( "event", $tags, "" ); + + # Application/zip is universally hated by all readers so it's better to use x-cbz and x-cbr here. + if ( $file =~ /^(.*\/)*.+\.(pdf)$/ ) { + $arcdata->{mimetype} = "application/pdf"; + } elsif ( $file =~ /^(.*\/)*.+\.(rar|cbr)$/ ) { + $arcdata->{mimetype} = "application/x-cbr"; + } elsif ( $file =~ /^(.*\/)*.+\.(epub)$/ ) { + $arcdata->{mimetype} = "application/epub+zip"; + } else { + $arcdata->{mimetype} = "application/x-cbz"; + } + + for ( values %{$arcdata} ) { $_ = xml_escape($_); } + + return $arcdata; +} + +sub render_archive_page { + + my ( $mojo, $id, $page ) = @_; + + my $redis = $mojo->LRR_CONF->get_redis; + my $archive = $redis->hget( $id, "file" ); + + # Parse archive to get its list of images + my ( $images, $sizes ) = get_filelist($archive); + + my @images = @$images; + + # If the page number is invalid, use the first page. + if ( $page > scalar @images ) { + $page = 1; + } + + # If the page number is valid, render the page. + my $image = $images[ $page - 1 ]; + + # Use the same code as /api/page to serve the file. + # This is clean, but might serve other types than JPEG depending on how the archive is built.. + # We could force resizing here to always have JPEG. (TODO?) + LANraragi::Model::Archive::serve_page( $mojo, $id, $image ); +} + +1; diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 244e36379..54d1de933 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -20,7 +20,7 @@ use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Model::Archive; use LANraragi::Model::Category; -# do_search (filter, filter2, page, key, order, newonly, untaggedonly) +# do_search (filter, category_id, page, key, order, newonly, untaggedonly) # Performs a search on the database. sub do_search { diff --git a/lib/LANraragi/Utils/Routing.pm b/lib/LANraragi/Utils/Routing.pm index 4d078fbcc..5f8bc533a 100644 --- a/lib/LANraragi/Utils/Routing.pm +++ b/lib/LANraragi/Utils/Routing.pm @@ -86,8 +86,12 @@ sub apply_routes { $logged_in->get('/logs/mojo')->to('logging#print_mojo'); $logged_in->get('/logs/redis')->to('logging#print_redis'); + # OPDS API + $public_api->get('/api/opds')->to('api-other#serve_opds_catalog'); + $public_api->get('/api/opds/:id')->to('api-other#serve_opds_item'); + $public_api->get('/api/opds/:id/pse')->to('api-other#serve_opds_page'); + # Miscellaneous API - $public_api->get('/api/opds')->to('api-other#serve_opds'); $public_api->get('/api/info')->to('api-other#serve_serverinfo'); $logged_in_api->get('/api/plugins/:type')->to('api-other#list_plugins'); $logged_in_api->post('/api/plugins/use')->to('api-other#use_plugin_sync'); diff --git a/templates/opds.html.tt2 b/templates/opds.html.tt2 index b7ff45ac3..5592726ca 100644 --- a/templates/opds.html.tt2 +++ b/templates/opds.html.tt2 @@ -1,57 +1,69 @@ - - - urn:lrr:0 - - - - - - [% title %] - 2010-01-10T10:03:10Z - [% motd %] - /favicon.ico - - [% version %] - http://github.org/Difegue/LANraragi - - - [% FOREACH arc IN arclist %] - - [% arc.title %] - urn:lrr:[% arc.arcid %] - [% arc.dateadded %] - [% arc.dateadded %] - - [% arc.author %] - - [% arc.series %] - [% arc.language %] - [% arc.circle %] - [% arc.event %] - [% IF arc.isnew %] - - [% ELSE %] - - [% END %] - [% arc.tags %] - - - - - - - - - - [% END %] - - + + + urn:lrr:0 + + + + + [% title %] + 2010-01-10T10:03:10Z + [% motd %] + /favicon.ico + + [% version %] + http://github.org/Difegue/LANraragi + + + + + [% FOREACH cat IN catlist %] + + [% END %] + + [% FOREACH arc IN arclist %] + + [% arc.title %] + urn:lrr:[% arc.arcid %] + [% arc.dateadded %] + [% arc.dateadded %] + + [% arc.author %] + + [% arc.series %] + [% arc.language %] + [% arc.circle %] + [% arc.event %] + [% IF arc.isnew == "false" %] + + [% ELSE %] + + [% END %] + [% arc.tags %] + + + + + + + + + + [% END %] + + \ No newline at end of file diff --git a/templates/opds_entry.html.tt2 b/templates/opds_entry.html.tt2 index ca1139899..9c60c326c 100644 --- a/templates/opds_entry.html.tt2 +++ b/templates/opds_entry.html.tt2 @@ -1,10 +1,14 @@ - + - - + + [% arc.title %] urn:lrr:[% arc.arcid %] @@ -17,16 +21,19 @@ [% arc.language %] [% arc.circle %] [% arc.event %] - [% IF arc.isnew %] - + [% IF arc.isnew == "false" %] + [% ELSE %] - + [% END %] [% arc.tags %] - - - - + + + + + \ No newline at end of file diff --git a/tests/mocks.pl b/tests/mocks.pl index a5e4bd9fa..cbad7564c 100644 --- a/tests/mocks.pl +++ b/tests/mocks.pl @@ -44,7 +44,7 @@ sub setup_redis_mock { }, "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg": { "isnew": "false", - "pagecount": 0, + "pagecount": 200, "progress": 34, "tags": "character:segata, female:very cool too", "title": "Saturn Backup Cartridge - American Manual", @@ -52,7 +52,7 @@ sub setup_redis_mock { }, "e4c422fd10943dc169e3489a38cdbf57101a5f7e": { "isnew": "true", - "pagecount": 0, + "pagecount": 10, "progress": 0, "tags": "parody: jojo's bizarre adventure", "title": "Rohan Kishibe goes to Gucci", @@ -76,7 +76,7 @@ sub setup_redis_mock { }, "28697b96f0ac5858be2614ed10ca47742c9522fd": { "isnew": "false", - "pagecount": 0, + "pagecount": 1, "progress": 0, "tags": "parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color, male:very cool too", "title": "Fate GO MEMO", diff --git a/tests/modules.t b/tests/modules.t index 6807da35b..8c40a6ad2 100644 --- a/tests/modules.t +++ b/tests/modules.t @@ -3,64 +3,44 @@ use warnings; use utf8; use Cwd; -use Mojo::Base 'Mojolicious'; use Test::More; -use Test::Mojo; -use Test::MockObject; - -sub test_module { - my $module_name = shift; - - eval "require $module_name"; - if ($@) { - warn "Could not load module: $@\n"; - return 0; - } - - return 1; -} - -# Mock Redis -my $cwd = getcwd; -require $cwd . "/tests/mocks.pl"; -setup_redis_mock(); my @modules = ( - "Shinobu", "LANraragi", - "LANraragi::Utils::Archive", "LANraragi::Utils::Database", - "LANraragi::Utils::Generic", "LANraragi::Utils::Plugins", - "LANraragi::Utils::Routing", "LANraragi::Utils::TempFolder", - "LANraragi::Utils::Logging", "LANraragi::Utils::Minion", - "LANraragi::Utils::Tags", "LANraragi::Controller::Api::Archive", - "LANraragi::Controller::Api::Search", "LANraragi::Controller::Api::Category", - "LANraragi::Controller::Api::Database", "LANraragi::Controller::Api::Shinobu", - "LANraragi::Controller::Api::Minion", "LANraragi::Controller::Api::Other", - "LANraragi::Controller::Backup", "LANraragi::Controller::Batch", - "LANraragi::Controller::Config", "LANraragi::Controller::Edit", - "LANraragi::Controller::Index", "LANraragi::Controller::Logging", - "LANraragi::Controller::Login", "LANraragi::Controller::Plugins", - "LANraragi::Controller::Reader", "LANraragi::Controller::Stats", - "LANraragi::Controller::Upload", "LANraragi::Controller::Category", - "LANraragi::Model::Archive", "LANraragi::Model::Backup", - "LANraragi::Model::Config", "LANraragi::Model::Plugins", - "LANraragi::Model::Reader", "LANraragi::Model::Search", - "LANraragi::Model::Stats", "LANraragi::Model::Category", - "LANraragi::Model::Upload", "LANraragi::Plugin::Metadata::Chaika", - "LANraragi::Plugin::Metadata::CopyTags", "LANraragi::Plugin::Metadata::DateAdded", - "LANraragi::Plugin::Metadata::EHentai", "LANraragi::Plugin::Metadata::Eze", - "LANraragi::Plugin::Metadata::Hdoujin", "LANraragi::Plugin::Metadata::Koromo", - "LANraragi::Plugin::Metadata::MEMS", "LANraragi::Plugin::Metadata::nHentai", - "LANraragi::Plugin::Metadata::RegexParse", "LANraragi::Plugin::Metadata::Fakku", - "LANraragi::Plugin::Login::EHentai", "LANraragi::Plugin::Login::Fakku", - "LANraragi::Plugin::Scripts::SourceFinder", "LANraragi::Plugin::Scripts::FolderToCat", - "LANraragi::Plugin::Download::EHentai", "LANraragi::Plugin::Download::Chaika", - "LANraragi::Plugin::Scripts::nHentaiSourceConverter", "LANraragi::Plugin::Scripts::BlacklistMigrate", - "LANraragi::Plugin::Metadata::Hitomi" + "Shinobu", "LANraragi", + "LANraragi::Utils::Archive", "LANraragi::Utils::Database", + "LANraragi::Utils::Generic", "LANraragi::Utils::Plugins", + "LANraragi::Utils::Routing", "LANraragi::Utils::TempFolder", + "LANraragi::Utils::Logging", "LANraragi::Utils::Minion", + "LANraragi::Utils::Tags", "LANraragi::Controller::Api::Archive", + "LANraragi::Controller::Api::Search", "LANraragi::Controller::Api::Category", + "LANraragi::Controller::Api::Database", "LANraragi::Controller::Api::Shinobu", + "LANraragi::Controller::Api::Minion", "LANraragi::Controller::Api::Other", + "LANraragi::Controller::Backup", "LANraragi::Controller::Batch", + "LANraragi::Controller::Config", "LANraragi::Controller::Edit", + "LANraragi::Controller::Index", "LANraragi::Controller::Logging", + "LANraragi::Controller::Login", "LANraragi::Controller::Plugins", + "LANraragi::Controller::Reader", "LANraragi::Controller::Stats", + "LANraragi::Controller::Upload", "LANraragi::Controller::Category", + "LANraragi::Model::Archive", "LANraragi::Model::Backup", + "LANraragi::Model::Config", "LANraragi::Model::Plugins", + "LANraragi::Model::Reader", "LANraragi::Model::Search", + "LANraragi::Model::Stats", "LANraragi::Model::Category", + "LANraragi::Model::Upload", "LANraragi::Model::Opds", + "LANraragi::Plugin::Metadata::Chaika", "LANraragi::Plugin::Metadata::CopyTags", + "LANraragi::Plugin::Metadata::DateAdded", "LANraragi::Plugin::Metadata::EHentai", + "LANraragi::Plugin::Metadata::Eze", "LANraragi::Plugin::Metadata::Hdoujin", + "LANraragi::Plugin::Metadata::Koromo", "LANraragi::Plugin::Metadata::MEMS", + "LANraragi::Plugin::Metadata::nHentai", "LANraragi::Plugin::Metadata::RegexParse", + "LANraragi::Plugin::Metadata::Fakku", "LANraragi::Plugin::Login::EHentai", + "LANraragi::Plugin::Login::Fakku", "LANraragi::Plugin::Scripts::SourceFinder", + "LANraragi::Plugin::Scripts::FolderToCat", "LANraragi::Plugin::Download::EHentai", + "LANraragi::Plugin::Download::Chaika", "LANraragi::Plugin::Scripts::nHentaiSourceConverter", + "LANraragi::Plugin::Scripts::BlacklistMigrate", "LANraragi::Plugin::Metadata::Hitomi" ); # Test all modules load properly foreach my $module_name (@modules) { - ok( test_module($module_name), $module_name ); + require_ok($module_name); } done_testing(); diff --git a/tests/opds.t b/tests/opds.t index c44177e00..ce5365ad2 100644 --- a/tests/opds.t +++ b/tests/opds.t @@ -5,17 +5,17 @@ use Cwd; use Mojo::Base 'Mojolicious'; -use Test::More tests => 1; +use Test::More; use Test::Mojo; use Test::MockObject; -use Mojo::JSON qw(decode_json encode_json); use Data::Dumper; use Template; use Mojo::File; use LANraragi::Model::Config; -use LANraragi::Model::Archive; +use LANraragi::Model::Opds; +use LANraragi::Model::Stats; # Mock Redis my $cwd = getcwd; @@ -23,6 +23,9 @@ my $SAMPLES = "$cwd/tests/samples"; require $cwd . "/tests/mocks.pl"; setup_redis_mock(); +# Build search hashes +LANraragi::Model::Stats::build_stat_hashes(); + # Mock basic mojo stuff used by the opds call my $mojo = Test::MockObject->new(); $mojo->mock( 'LRR_CONF', sub { LANraragi::Model::Config:: } ); @@ -54,5 +57,11 @@ $mojo->mock( my $expected_opds = ( Mojo::File->new("$SAMPLES/opds/opds_sample.xml")->slurp ); # Generate a new OPDS Catalog and compare it against our sample -my $opds_result = LANraragi::Model::Archive::generate_opds_catalog($mojo); +my $opds_result = LANraragi::Model::Opds::generate_opds_catalog($mojo); + +# Compare without whitespace +$opds_result =~ s/\s//g; +$expected_opds =~ s/\s//g; is( $opds_result, $expected_opds, "OPDS API Test" ); + +done_testing(); diff --git a/tests/samples/opds/opds_sample.xml b/tests/samples/opds/opds_sample.xml index 12cdf4b21..3bf1ace34 100644 --- a/tests/samples/opds/opds_sample.xml +++ b/tests/samples/opds/opds_sample.xml @@ -1,195 +1,223 @@ - - - urn:lrr:0 - - - - - - LANraragi - 2010-01-10T10:03:10Z - Welcome to this Library running LANraragi! - /favicon.ico - - 9.9.9 - http://github.org/Difegue/LANraragi - - - - - Fate GO MEMO - urn:lrr:28697b96f0ac5858be2614ed10ca47742c9522fd - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - wada rco - - - - wadamemo - - - - - parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color, male:very cool too - - - - - - - - - - - - Fate GO MEMO 2 - urn:lrr:2810d5e0a8d027ecefebca6237031a0fa7b91eb3 - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - wada rco - - - - wadamemo - - - - - parody:fate grand order, character:abigail williams, character:artoria pendragon alter, character:asterios, character:ereshkigal, character:gilgamesh, character:hans christian andersen, character:hassan of serenity, character:hector, character:helena blavatsky, character:irisviel von einzbern, character:jeanne alter, character:jeanne darc, character:kiara sessyoin, character:kiyohime, character:lancer, character:martha, character:minamoto no raikou, character:mochizuki chiyome, character:mordred pendragon, character:nitocris, character:oda nobunaga, character:osakabehime, character:penthesilea, character:queen of sheba, character:rin tosaka, character:saber, character:sakata kintoki, character:scheherazade, character:sherlock holmes, character:suzuka gozen, character:tamamo no mae, character:ushiwakamaru, character:waver velvet, character:xuanzang, character:zhuge liang, group:wadamemo, artist:wada rco, artbook, full color - - - - - - - - - - - - Ghost in the Shell 1.5 - Human-Error Processor vol01ch01 - urn:lrr:4857fd2e7c00db8b0af0337b94055d8445118630 - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - shirow masamune - - - - - - - - - artist:shirow masamune - - - - - - - - - - - - Rohan Kishibe goes to Gucci - urn:lrr:e4c422fd10943dc169e3489a38cdbf57101a5f7e - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - - - - - - - - - - parody: jojo's bizarre adventure - - - - - - - - - - - - Saturn Backup Cartridge - American Manual - urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebg - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - - - - - - - - - - character:segata, female:very cool too - - - - - - - - - - - - Saturn Backup Cartridge - Japanese Manual - urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebf - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - - - - - - - - - - character:segata sanshiro, male:very cool - - - - - - - - - - - - + xmlns:opds="http://opds-spec.org/2010/catalog" + xmlns:pse="http://vaemendis.net/opds-pse/ns"> + + urn:lrr:0 + + + + + LANraragi + 2010-01-10T10:03:10Z + Welcome to this Library running LANraragi! + /favicon.ico + + 9.9.9 + http://github.org/Difegue/LANraragi + + + + + + + + + + + + + Fate GO MEMO + urn:lrr:28697b96f0ac5858be2614ed10ca47742c9522fd + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + wada rco + + + + wadamemo + + + + + parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color, male:very cool too + + + + + + + + + + + + Fate GO MEMO 2 + urn:lrr:2810d5e0a8d027ecefebca6237031a0fa7b91eb3 + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + wada rco + + + + wadamemo + + + + + parody:fate grand order, character:abigail williams, character:artoria pendragon alter, character:asterios, character:ereshkigal, character:gilgamesh, character:hans christian andersen, character:hassan of serenity, character:hector, character:helena blavatsky, character:irisviel von einzbern, character:jeanne alter, character:jeanne darc, character:kiara sessyoin, character:kiyohime, character:lancer, character:martha, character:minamoto no raikou, character:mochizuki chiyome, character:mordred pendragon, character:nitocris, character:oda nobunaga, character:osakabehime, character:penthesilea, character:queen of sheba, character:rin tosaka, character:saber, character:sakata kintoki, character:scheherazade, character:sherlock holmes, character:suzuka gozen, character:tamamo no mae, character:ushiwakamaru, character:waver velvet, character:xuanzang, character:zhuge liang, group:wadamemo, artist:wada rco, artbook, full color + + + + + + + + + + + + Ghost in the Shell 1.5 - Human-Error Processor vol01ch01 + urn:lrr:4857fd2e7c00db8b0af0337b94055d8445118630 + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + shirow masamune + + + + + + + + + artist:shirow masamune + + + + + + + + + + + + Rohan Kishibe goes to Gucci + urn:lrr:e4c422fd10943dc169e3489a38cdbf57101a5f7e + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + + + + + + + + + + parody: jojo's bizarre adventure + + + + + + + + + + + + Saturn Backup Cartridge - American Manual + urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebg + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + + + + + + + + + + character:segata, female:very cool too + + + + + + + + + + + + Saturn Backup Cartridge - Japanese Manual + urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebf + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + + + + + + + + + + character:segata sanshiro, male:very cool + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/Documentation/advanced-usage/external-readers.md b/tools/Documentation/advanced-usage/external-readers.md index 11e422b33..f5922eece 100644 --- a/tools/Documentation/advanced-usage/external-readers.md +++ b/tools/Documentation/advanced-usage/external-readers.md @@ -52,7 +52,13 @@ You can download it [here.](https://github.com/inorichi/tachiyomi-extensions/blo ![Example OPDS reader](<../.screenshots/opds.jpg>) Some readers can leverage the [OPDS Catalog](https://opds.io) exposed by LANraragi to visualize and read the available archives. -Those programs can't exploit all of LRR's features(Search, Database backup, Streaming images), but they might have reading features you won't find in the current dedicated clients. +Those programs can't exploit all of LRR's features(Search, Database backup), but they might have reading features you won't find in the current dedicated clients. + +If your OPDS reader supports [Page Streaming Extensions](https://anansi-project.github.io/docs/opds-pse/intro), LANraragi is compatible with it and will serve individual pages. + +{% hint style="info" %} +LRR supports OPDS PSE 1.1, which means that if you have server-side progress tracking enabled, you can pick up where you stopped reading from any OPDS client. Syndication! +{% endhint %} The URL for the OPDS Catalog is `[YOUR_LANRARAGI_URL]/api/opds`. You can use [the Demo](https://lrr.tvc-16.science/api/opds) as an example. @@ -73,3 +79,4 @@ The following readers haven't been tested but should work: * [**TiReader (iOS)**](http://tireader.com) * [**Chunky Reader (iOS)**](http://chunkyreader.com) +* [**Panels (iOS)**](https://panels.app/) From 854fb2200bd0ff22b8d399e916f3c46b86853c23 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 01:05:01 +0100 Subject: [PATCH 37/45] woops --- tests/modules.t | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/modules.t b/tests/modules.t index 8c40a6ad2..35898abd9 100644 --- a/tests/modules.t +++ b/tests/modules.t @@ -5,6 +5,11 @@ use Cwd; use Test::More; +# Mock Redis +my $cwd = getcwd; +require $cwd . "/tests/mocks.pl"; +setup_redis_mock(); + my @modules = ( "Shinobu", "LANraragi", "LANraragi::Utils::Archive", "LANraragi::Utils::Database", From 7f93999243b652f3bbd55fcf061baf5f71f74ae2 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 02:10:06 +0100 Subject: [PATCH 38/45] Update OPDS API documentation, use only woff2 for FA --- README.md | 2 +- .../miscellaneous-other-api.md | 396 ++++++++++-------- tools/install.pl | 2 - 3 files changed, 227 insertions(+), 173 deletions(-) diff --git a/README.md b/README.md index 5980091b2..baf999e84 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Open source server for archival of comics/manga, running on Mojolicious + Redis. * Read your archives in dedicated reader software using the built-in OPDS Catalog (now with PSE support!) -* Use the Client API to interact with LANraragi from other programs +* Use the Client API to interact with LANraragi from other programs (Available for [many platforms!](https://sugoi.gitbook.io/lanraragi/v/dev/advanced-usage/external-readers)) * Two different user interfaces : compact archive list with thumbnails-on-hover, or thumbnail view. diff --git a/tools/Documentation/api-documentation/miscellaneous-other-api.md b/tools/Documentation/api-documentation/miscellaneous-other-api.md index 29b5b0cf1..d219166b4 100644 --- a/tools/Documentation/api-documentation/miscellaneous-other-api.md +++ b/tools/Documentation/api-documentation/miscellaneous-other-api.md @@ -32,184 +32,240 @@ Returns some basic information about the LRR instance this server is running. {% swagger baseUrl="http://lrr.tvc-16.science" path="/api/opds" method="get" summary="Get the OPDS Catalog" %} {% swagger-description %} -Get the Archive Index as an OPDS 1.2 Catalog. +Get the Archive Index as an OPDS 1.2 Catalog with PSE 1.1 compatibility. {% endswagger-description %} -{% swagger-parameter name="id" type="string" required="false" in="query" %} -ID of an archive. Passing this will show only one - -`` +{% swagger-parameter name="category" type="string" required="false" in="query" %} +Category ID. If passed, the OPDS catalog will be filtered to only show archives from this category. +{% endswagger-parameter %} - for the given ID in the result, instead of all the archives. +{% swagger-parameter name="id" type="string" required="false" in="path" %} +ID of an archive. Passing this will show only one `` for the given ID in the result, instead of all the archives. {% endswagger-parameter %} {% swagger-response status="200" description="" %} ```markup - - - urn:lrr:0 - - - - - - LANraragi Demo - 2010-01-10T10:03:10Z - LANraragi Demo, running in Docker @ TVC-16 - /favicon.ico - - 0.6.6 - http://github.org/Difegue/LANraragi - - - - - Ghost in the Shell 1.5 - Human-Error Processor vol01ch01 - urn:lrr:4857fd2e7c00db8b0af0337b94055d8445118630 - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - shirow masamune - - - - - - - - - artist:shirow masamune - - - - - - - - - - - - Fate GO MEMO 2 - urn:lrr:2810d5e0a8d027ecefebca6237031a0fa7b91eb3 - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - wada rco - - - - wadamemo - - - - - parody:fate grand order, character:abigail williams, character:artoria pendragon alter, character:asterios, character:ereshkigal, character:gilgamesh, character:hans christian andersen, character:hassan of serenity, character:hector, character:helena blavatsky, character:irisviel von einzbern, character:jeanne alter, character:jeanne darc, character:kiara sessyoin, character:kiyohime, character:lancer, character:martha, character:minamoto no raikou, character:mochizuki chiyome, character:mordred pendragon, character:nitocris, character:oda nobunaga, character:osakabehime, character:penthesilea, character:queen of sheba, character:rin tosaka, character:saber, character:sakata kintoki, character:scheherazade, character:sherlock holmes, character:suzuka gozen, character:tamamo no mae, character:ushiwakamaru, character:waver velvet, character:xuanzang, character:zhuge liang, group:wadamemo, artist:wada rco, artbook, full color - - - - - - - - - - - - Saturn Backup Cartridge - Japanese Manual - urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebf - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - - - - - - - - - - character:segata sanshiro - - - - - - - - - - - - Rohan Kishibe goes to Gucci - urn:lrr:e4c422fd10943dc169e3489a38cdbf57101a5f7e - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - - - - - - - - - - parody: jojo's bizarre adventure - - - - - - - - - - - - Fate GO MEMO - urn:lrr:28697b96f0ac5858be2614ed10ca47742c9522fd - 2010-01-10T10:01:11Z - 2010-01-10T10:01:11Z - - wada rco - - - - wadamemo - - - - - parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color - - - - - - - - - - + + + urn:lrr:0 + + + + + LANraragi + 2010-01-10T10:03:10Z + Welcome to this Library running LANraragi! + /favicon.ico + + 9.9.9 + http://github.org/Difegue/LANraragi + + + + + + + + + + + + + Fate GO MEMO + urn:lrr:28697b96f0ac5858be2614ed10ca47742c9522fd + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + wada rco + + + + wadamemo + + + + + parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color, male:very cool too + + + + + + + + + + + + Fate GO MEMO 2 + urn:lrr:2810d5e0a8d027ecefebca6237031a0fa7b91eb3 + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + wada rco + + + + wadamemo + + + + + parody:fate grand order, character:abigail williams, character:artoria pendragon alter, character:asterios, character:ereshkigal, character:gilgamesh, character:hans christian andersen, character:hassan of serenity, character:hector, character:helena blavatsky, character:irisviel von einzbern, character:jeanne alter, character:jeanne darc, character:kiara sessyoin, character:kiyohime, character:lancer, character:martha, character:minamoto no raikou, character:mochizuki chiyome, character:mordred pendragon, character:nitocris, character:oda nobunaga, character:osakabehime, character:penthesilea, character:queen of sheba, character:rin tosaka, character:saber, character:sakata kintoki, character:scheherazade, character:sherlock holmes, character:suzuka gozen, character:tamamo no mae, character:ushiwakamaru, character:waver velvet, character:xuanzang, character:zhuge liang, group:wadamemo, artist:wada rco, artbook, full color + + + + + + + + + + + + Ghost in the Shell 1.5 - Human-Error Processor vol01ch01 + urn:lrr:4857fd2e7c00db8b0af0337b94055d8445118630 + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + shirow masamune + + + + + + + + + artist:shirow masamune + + + + + + + + + + + + Rohan Kishibe goes to Gucci + urn:lrr:e4c422fd10943dc169e3489a38cdbf57101a5f7e + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + + + + + + + + + + parody: jojo's bizarre adventure + + + + + + + + + + + + Saturn Backup Cartridge - American Manual + urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebg + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + + + + + + + + + + character:segata, female:very cool too + + + + + + + + + + + + Saturn Backup Cartridge - Japanese Manual + urn:lrr:e69e43e1355267f7d32a4f9b7f2fe108d2401ebf + 2010-01-10T10:01:11Z + 2010-01-10T10:01:11Z + + + + + + + + + + + character:segata sanshiro, male:very cool + + + + + + + + + + ``` diff --git a/tools/install.pl b/tools/install.pl index 76aff8d32..5e573217b 100755 --- a/tools/install.pl +++ b/tools/install.pl @@ -35,9 +35,7 @@ ); my @vendor_woff = ( - "/\@fortawesome/fontawesome-free/webfonts/fa-solid-900.woff", "/\@fortawesome/fontawesome-free/webfonts/fa-solid-900.woff2", - "/\@fortawesome/fontawesome-free/webfonts/fa-regular-400.woff", "/\@fortawesome/fontawesome-free/webfonts/fa-regular-400.woff2", "/open-sans-fontface/fonts/Regular/OpenSans-Regular.woff", "/open-sans-fontface/fonts/Bold/OpenSans-Bold.woff", From 2c04489652100a8a35073a5c5f2129ff3e59eee4 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 02:12:07 +0100 Subject: [PATCH 39/45] Fix brew --- tools/build/homebrew/Lanraragi.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build/homebrew/Lanraragi.rb b/tools/build/homebrew/Lanraragi.rb index c11ef54fa..0bfa9b2b7 100644 --- a/tools/build/homebrew/Lanraragi.rb +++ b/tools/build/homebrew/Lanraragi.rb @@ -31,7 +31,7 @@ class Lanraragi < Formula on_macos do resource "libarchive-headers" do url "https://opensource.apple.com/tarballs/libarchive/libarchive-83.100.2.tar.gz" - sha256 "e54049be1b1d4f674f33488fdbcf5bb9f9390db5cc17a5b34cbeeb5f752b207a" + sha256 "a0228f75792f881bc927196f8b794d0263a019aab741765e54550f75271258aa" end end From bc13a71e26536172b34e7f4af374d4782442428a Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 20:48:13 +0100 Subject: [PATCH 40/45] Allow the API key to be set as a parameter again (undocumented, for OPDS only) i hate everything about this --- lib/LANraragi/Controller/Login.pm | 6 +- lib/LANraragi/Model/Opds.pm | 16 +++- templates/opds.html.tt2 | 20 ++--- templates/opds_entry.html.tt2 | 14 ++-- tests/samples/opds/opds_sample.xml | 82 +++++++++---------- .../advanced-usage/external-readers.md | 4 +- 6 files changed, 78 insertions(+), 64 deletions(-) diff --git a/lib/LANraragi/Controller/Login.pm b/lib/LANraragi/Controller/Login.pm index b4a4a76c9..ad9200d22 100644 --- a/lib/LANraragi/Controller/Login.pm +++ b/lib/LANraragi/Controller/Login.pm @@ -58,12 +58,16 @@ sub logged_in_api { # The API key is in the Authentication header. my $expected_key = $self->LRR_CONF->get_apikey; + my $expected_header = "Bearer " . encode_base64( $expected_key, "" ); my $auth_header = $self->req->headers->authorization || ""; - my $expected_header = "Bearer " . encode_base64( $expected_key, "" ); + + # It can also be passed as a parameter. (Undocumented, mostly just meant for OPDS) + my $param_key = $self->req->param('key') || ''; return 1 if ( $expected_key ne "" && $auth_header eq $expected_header ) + || ( $param_key ne "" && $param_key eq $expected_key ) || $self->session('is_logged') || $self->LRR_CONF->enable_pass == 0; $self->render( diff --git a/lib/LANraragi/Model/Opds.pm b/lib/LANraragi/Model/Opds.pm index c1a760f0f..9083237a0 100644 --- a/lib/LANraragi/Model/Opds.pm +++ b/lib/LANraragi/Model/Opds.pm @@ -15,9 +15,12 @@ use LANraragi::Model::Search; sub generate_opds_catalog { - my $mojo = shift; + my $mojo = shift; my $cat_id = $mojo->req->param('category') || ""; - my @cats = LANraragi::Model::Category->get_category_list; + + # If the user authentified to this via an API key, we need to carry it over to the OPDS links. + my $api_key = $mojo->req->param('key') ? "key=" . $mojo->req->param('key') : ""; + my @cats = LANraragi::Model::Category->get_category_list; # Use the search engine to get the list of archives to show in the catalog. my ( $total, $filtered, @keys ) = LANraragi::Model::Search::do_search( "", $cat_id, -1, "title", 0, 0, 0 ); @@ -52,7 +55,8 @@ sub generate_opds_catalog { nocat => $cat_id eq "", title => $mojo->LRR_CONF->get_htmltitle, motd => $mojo->LRR_CONF->get_motd, - version => $mojo->LRR_VERSION + version => $mojo->LRR_VERSION, + api_key => $api_key ); } @@ -60,6 +64,9 @@ sub generate_opds_item { my ( $mojo, $id ) = @_; + # If the user authentified to this via an API key, we need to carry it over to the OPDS links. + my $api_key = $mojo->req->param('key') ? "key=" . $mojo->req->param('key') : ""; + # Detailed pages just return a single entry instead of all the archives. my $arcdata = get_opds_data($id); @@ -68,7 +75,8 @@ sub generate_opds_item { arc => $arcdata, title => $mojo->LRR_CONF->get_htmltitle, motd => $mojo->LRR_CONF->get_motd, - version => $mojo->LRR_VERSION + version => $mojo->LRR_VERSION, + api_key => $api_key ); } diff --git a/templates/opds.html.tt2 b/templates/opds.html.tt2 index 5592726ca..f2e67d9ed 100644 --- a/templates/opds.html.tt2 +++ b/templates/opds.html.tt2 @@ -6,8 +6,8 @@ urn:lrr:0 - - + + [% title %] 2010-01-10T10:03:10Z @@ -19,14 +19,14 @@ [% FOREACH cat IN catlist %] @@ -52,17 +52,17 @@ [% END %] [% arc.tags %] - - - + - - + href="/api/opds/[% arc.arcid %]/pse?page={pageNumber}&[% api_key %]" pse:count="[% arc.pagecount %]" [% IF arc.progress %] pse:lastRead="[% arc.progress %]" [% END %]/> + [% END %] diff --git a/templates/opds_entry.html.tt2 b/templates/opds_entry.html.tt2 index 9c60c326c..37de00c1d 100644 --- a/templates/opds_entry.html.tt2 +++ b/templates/opds_entry.html.tt2 @@ -7,8 +7,8 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:schema="http://schema.org/"> - - + + [% arc.title %] urn:lrr:[% arc.arcid %] @@ -28,12 +28,12 @@ [% END %] [% arc.tags %] - - - + + - + href="/api/opds/[% arc.arcid %]/pse?page={pageNumber}&[% api_key %]" pse:count="[% arc.pagecount %]" [% IF arc.progress %] pse:lastRead="[% arc.progress %]" [% END %]/> + \ No newline at end of file diff --git a/tests/samples/opds/opds_sample.xml b/tests/samples/opds/opds_sample.xml index 3bf1ace34..d18e18743 100644 --- a/tests/samples/opds/opds_sample.xml +++ b/tests/samples/opds/opds_sample.xml @@ -6,8 +6,8 @@ urn:lrr:0 - - + + LANraragi 2010-01-10T10:03:10Z @@ -19,20 +19,20 @@ @@ -56,17 +56,17 @@ parody:fate grand order, group:wadamemo, artist:wada rco, artbook, full color, male:very cool too - - - + - - + href="/api/opds/28697b96f0ac5858be2614ed10ca47742c9522fd/pse?page={pageNumber}&" pse:count="1" /> + @@ -86,17 +86,17 @@ parody:fate grand order, character:abigail williams, character:artoria pendragon alter, character:asterios, character:ereshkigal, character:gilgamesh, character:hans christian andersen, character:hassan of serenity, character:hector, character:helena blavatsky, character:irisviel von einzbern, character:jeanne alter, character:jeanne darc, character:kiara sessyoin, character:kiyohime, character:lancer, character:martha, character:minamoto no raikou, character:mochizuki chiyome, character:mordred pendragon, character:nitocris, character:oda nobunaga, character:osakabehime, character:penthesilea, character:queen of sheba, character:rin tosaka, character:saber, character:sakata kintoki, character:scheherazade, character:sherlock holmes, character:suzuka gozen, character:tamamo no mae, character:ushiwakamaru, character:waver velvet, character:xuanzang, character:zhuge liang, group:wadamemo, artist:wada rco, artbook, full color - - - + - - + href="/api/opds/2810d5e0a8d027ecefebca6237031a0fa7b91eb3/pse?page={pageNumber}&" pse:count="34" pse:lastRead="34" /> + @@ -116,17 +116,17 @@ artist:shirow masamune - - - + - - + href="/api/opds/4857fd2e7c00db8b0af0337b94055d8445118630/pse?page={pageNumber}&" pse:count="34" pse:lastRead="34" /> + @@ -146,17 +146,17 @@ parody: jojo's bizarre adventure - - - + - - + href="/api/opds/e4c422fd10943dc169e3489a38cdbf57101a5f7e/pse?page={pageNumber}&" pse:count="10" /> + @@ -176,17 +176,17 @@ character:segata, female:very cool too - - - + - - + href="/api/opds/e69e43e1355267f7d32a4f9b7f2fe108d2401ebg/pse?page={pageNumber}&" pse:count="200" pse:lastRead="34" /> + @@ -206,17 +206,17 @@ character:segata sanshiro, male:very cool - - - + - - + href="/api/opds/e69e43e1355267f7d32a4f9b7f2fe108d2401ebf/pse?page={pageNumber}&" pse:count="2" pse:lastRead="10" /> + diff --git a/tools/Documentation/advanced-usage/external-readers.md b/tools/Documentation/advanced-usage/external-readers.md index f5922eece..0704b0452 100644 --- a/tools/Documentation/advanced-usage/external-readers.md +++ b/tools/Documentation/advanced-usage/external-readers.md @@ -65,7 +65,9 @@ You can use [the Demo](https://lrr.tvc-16.science/api/opds) as an example. Refer to your reader's documentation to figure out where to put this URL. {% hint style="warning" %} -If you have No-Fun Mode enabled, remember that you'll need to add the API Key to this URL for the catalog to be available to your reader application. +If you have No-Fun Mode enabled, remember that you'll need to add the API Key to this URL for the catalog to be available to your reader application. +You can either use the `Bearer` token approach like with regular API calls, or add `?key=[API_KEY]` as a parameter to the URL. + {% endhint %} The following readers have been tested with the OPDS Catalog: From 6d8873d8e2b86c10f5d7a46fe27e6ddf39f94c5f Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 20:48:34 +0100 Subject: [PATCH 41/45] Fix random search using a different name for the new filter --- lib/LANraragi/Controller/Api/Search.pm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index aab4f5779..987651fa4 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -94,13 +94,13 @@ sub get_random_archives { my $filter = $req->param('filter'); my $category = $req->param('category') || ""; - my $isnew = $req->param('isnew') || ""; - my $untaggedonly = $req->param('untaggedonly') || ""; + my $newfilter = $req->param('newonly') || "false"; + my $untaggedf = $req->param('untaggedonly') || "false"; my $random_count = $req->param('count') || 5; # Use the search engine to get IDs matching the filter/category selection, with start=-1 to get all data my ( $total, $filtered, @ids ) = - LANraragi::Model::Search::do_search( $filter, $category, -1, "title", 0, $isnew, $untaggedonly ); + LANraragi::Model::Search::do_search( $filter, $category, -1, "title", 0, $newfilter eq "true", $untaggedf eq "true" ); my @random_ids; $random_count = min( $random_count, scalar(@ids) ); From 3050d9717fa0b49587f229e90b02ae4e98a8e618 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 21:00:51 +0100 Subject: [PATCH 42/45] Remove workaround code for empty array intersection since it doesn't work that way anymore --- lib/LANraragi/Model/Search.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index 54d1de933..877f44bf8 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -258,9 +258,9 @@ sub intersect_arrays { my ( $array1, $array2, $isneg ) = @_; - # Special case: If array1 is empty, just return array2 as we don't have anything to intersect yet + # If array1 is empty, just return an empty array or the second array if $isneg is true if ( scalar @$array1 == 0 ) { - return @$array2; + return $isneg ? @$array2 : (); } # If array2 is empty, die since this sub shouldn't even be used in that case From 2e0f88f7cdb5ea4cc71d9aedad885e368b5444f5 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 23:03:53 +0100 Subject: [PATCH 43/45] Fix add_timestamp_tag still using an old function --- lib/LANraragi/Utils/Database.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index 3a9e2fc53..bd963c553 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -96,7 +96,7 @@ sub add_timestamp_tag { $date = time(); } - add_tags( $id, "date_added:$date" ); + set_tags( $id, "date_added:$date", 1 ); } } From 76acac13a9ec54a7d47ae1adfc8e5eaa0e7b65f7 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 23:26:29 +0100 Subject: [PATCH 44/45] Fix deleting an ID not cleaning up the indexes --- lib/LANraragi/Utils/Database.pm | 39 +++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index bd963c553..6a8b8efe7 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -13,7 +13,7 @@ use Cwd; use Unicode::Normalize; use LANraragi::Model::Plugins; -use LANraragi::Utils::Generic qw(flat remove_spaces remove_newlines); +use LANraragi::Utils::Generic qw(flat remove_spaces remove_newlines trim_url); use LANraragi::Utils::Tags qw(unflat_tagrules tags_rules_to_array restore_CRLF); use LANraragi::Utils::Archive qw(get_filelist); use LANraragi::Utils::Logging qw(get_logger); @@ -204,10 +204,26 @@ sub delete_archive { my $id = $_[0]; my $redis = LANraragi::Model::Config->get_redis; my $filename = $redis->hget( $id, "file" ); + my $oldtags = $redis->hget( $id, "tags" ); + $oldtags = redis_decode($oldtags); + + my $oldtitle = lc( redis_decode( $redis->hget( $id, "title" ) ) ); + remove_spaces($oldtitle); + remove_newlines($oldtitle); + $oldtitle = redis_encode($oldtitle); $redis->del($id); $redis->quit(); + # Remove matching data from the search indexes + my $redis_search = LANraragi::Model::Config->get_redis_search; + $redis_search->zrem( "LRR_TITLES", "$oldtitle\0$id" ); + $redis_search->srem( "LRR_NEW", $id ); + $redis_search->srem( "LRR_UNTAGGED", $id ); + $redis_search->quit(); + + update_indexes( $id, $oldtags, "" ); + if ( -e $filename ) { unlink $filename; @@ -414,8 +430,16 @@ sub update_indexes { foreach my $tag (@oldtags) { - # Tag is lowercased here to avoid redundancy/dupes - $redis->srem( "INDEX_" . redis_encode( lc($tag) ), $id ); + if ( $tag =~ /source:(.*)/i ) { + my $url = $1; + trim_url($url); + $redis->hdel( "LRR_URLMAP", $url ); + } else { + + # Tag is lowercased here to avoid redundancy/dupes + $redis->srem( "INDEX_" . redis_encode( lc($tag) ), $id ); + } + } foreach my $tag (@newtags) { @@ -423,7 +447,14 @@ sub update_indexes { # The following are basic and therefore don't count as "tagged" $has_tags = 1 unless $tag =~ /(artist|parody|series|language|event|group|date_added|timestamp):.*/; - $redis->sadd( "INDEX_" . redis_encode( lc($tag) ), $id ); + # If the tag is a source: tag, add it to the URL index + if ( $tag =~ /source:(.*)/i ) { + my $url = $1; + trim_url($url); + $redis->hset( "LRR_URLMAP", $url, $id ); + } else { + $redis->sadd( "INDEX_" . redis_encode( lc($tag) ), $id ); + } } # Add or remove the ID from the untagged list From 1a8e152631441a91103dc7c976c774302f0c1570 Mon Sep 17 00:00:00 2001 From: Difegue Date: Mon, 19 Dec 2022 23:29:26 +0100 Subject: [PATCH 45/45] Fix leftover use; causing Model::Plugins to be unable to import some Utils functions welp --- lib/LANraragi/Utils/Database.pm | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/LANraragi/Utils/Database.pm b/lib/LANraragi/Utils/Database.pm index 6a8b8efe7..830ad52a1 100644 --- a/lib/LANraragi/Utils/Database.pm +++ b/lib/LANraragi/Utils/Database.pm @@ -12,7 +12,6 @@ use Redis; use Cwd; use Unicode::Normalize; -use LANraragi::Model::Plugins; use LANraragi::Utils::Generic qw(flat remove_spaces remove_newlines trim_url); use LANraragi::Utils::Tags qw(unflat_tagrules tags_rules_to_array restore_CRLF); use LANraragi::Utils::Archive qw(get_filelist);