misc

library/createdb/computers_add.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 
 from library import usage
-from library.mediadb import db_playlists
+from library.mediadb import db_media, db_playlists
 from library.utils import arggroups, argparse_utils, consts, db_utils, objects, remote_processes
 from library.utils.log_utils import log
 
@@ -102,4 +102,7 @@ def computer_add(args, hostnames):
 def computers_add():
     args = parse_args()
 
+    db_playlists.create(args)
+    db_media.create(args)
+
     computer_add(args, args.hostnames)

library/createdb/fs_add.py

@@ -296,6 +296,8 @@ def fs_add(args=None) -> None:
         sys.argv = ["lb", *args]
 
     args = parse_args(SC.fs_add, usage.fs_add)
+    db_playlists.create(args)
+    db_media.create(args)
     extractor(args, args.paths)
 
 

library/createdb/links_add.py

@@ -370,9 +370,9 @@ def links_update() -> None:
             new_media = extractor(args_env, playlist["path"])
 
             if new_media > 0:
-                db_playlists.decrease_update_delay(args, playlist["path"])
+                db_playlists.update_more_frequently(args, playlist["path"])
             else:
-                db_playlists.increase_update_delay(args, playlist["path"])
+                db_playlists.update_less_frequently(args, playlist["path"])
 
             if playlist_count > 3:
                 time.sleep(random.uniform(0.05, 2))

library/createdb/reddit_add.py

@@ -364,8 +364,8 @@ def reddit_update(args=None) -> None:
             elif extractor_key == "reddit_praw_redditor":
                 redditor_new(args, {"path": path, "name": name})
 
-            db_playlists.decrease_update_delay(args, playlist["path"])
+            db_playlists.update_more_frequently(args, playlist["path"])
         except skip_errors as e:
-            db_playlists.increase_update_delay(args, playlist["path"])
+            db_playlists.update_less_frequently(args, playlist["path"])
             log.error("[%s] skipping: %s", name, e)
             continue

library/createdb/tabs_add.py

@@ -84,6 +84,8 @@ def tabs_add(args=None) -> None:
     args = parse_args()
     paths = list(gen_paths(args))
 
+    db_media.create(args)
+
     tabs = iterables.list_dict_filter_bool([consolidate_url(args, path) for path in get_new_paths(args, paths)])
     for tab in tabs:
         db_media.add(args, tab)

library/createdb/torrents_add.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 
 from library import usage
-from library.mediadb import db_playlists
+from library.mediadb import db_media, db_playlists
 from library.utils import arg_utils, arggroups, argparse_utils, consts, db_utils, iterables, nums, objects, printing
 from library.utils.file_utils import trash
 from library.utils.log_utils import log
@@ -85,6 +85,9 @@ def extract_metadata(path):
 def torrents_add():
     args = parse_args()
 
+    db_playlists.create(args)
+    db_media.create(args)
+
     scanned_set = set(arg_utils.gen_paths(args, default_exts=(".torrent",)))
 
     known_hashes = set()

library/createdb/tube_add.py

@@ -37,6 +37,9 @@ def tube_add(args=None) -> None:
     args = parse_args(SC.tube_add, usage=usage.tube_add)
     paths = arg_utils.gen_paths(args)
 
+    db_playlists.create(args)
+    db_media.create(args)
+
     if args.insert_only:
         args.db["media"].insert_all(
             [
@@ -89,6 +92,7 @@ def tube_update(args=None) -> None:
         sys.argv = ["tubeupdate", *args]
 
     args = parse_args(SC.tube_update, usage=usage.tube_update)
+
     tube_playlists = db_playlists.get_all(
         args,
         sql_filters=["AND extractor_key NOT IN ('Local', 'reddit_praw_redditor', 'reddit_praw_subreddit')"],

library/createdb/tube_backend.py

@@ -130,7 +130,7 @@ def run(self, info) -> tuple[list, dict]:  # pylint: disable=arguments-renamed
 
                 entry = objects.dumbcopy(info)
                 if entry:
-                    if db_playlists.media_exists(args, playlist_path, webpath) and not args.ignore_errors:
+                    if db_playlists.media_exists(args, webpath, playlist_path) and not args.ignore_errors:
                         raise ExistingPlaylistVideoReached
 
                     if not info.get("playlist_id") or webpath == playlist_path:
@@ -172,9 +172,9 @@ def run(self, info) -> tuple[list, dict]:  # pylint: disable=arguments-renamed
 
         if args.action == consts.SC.tube_update:
             if added_media_count > count_before_extract:
-                db_playlists.decrease_update_delay(args, playlist_path)
+                db_playlists.update_more_frequently(args, playlist_path)
             else:
-                db_playlists.increase_update_delay(args, playlist_path)
+                db_playlists.update_less_frequently(args, playlist_path)
 
 
 def yt_subs_config(args):

library/createdb/web_add.py

@@ -184,16 +184,14 @@ def spider(args, paths: list):
 
                 if link in traversed_paths or link in paths:
                     continue
-                if web.is_index(link):
-                    if web.is_subpath(path, link):
-                        paths.append(link)
-                    continue
 
                 if db_media.exists(args, link):
                     known_paths.add(link)
-                    continue
                 else:
                     new_paths[link] = objects.merge_dict_values_str(new_paths.get(link) or {}, link_dict)
+
+                if web.is_subpath(path, link) and web.is_html(link):
+                    paths.append(link)
         else:  # not HTML page
             if path in traversed_paths or path in paths:
                 pass
@@ -263,6 +261,9 @@ def web_add(args=None) -> None:
 
     args = parse_args(consts.SC.web_add, usage=usage.web_add)
 
+    db_playlists.create(args)
+    db_media.create(args)
+
     if args.insert_only:
         media_new = set()
         media_known = set()
@@ -300,7 +301,7 @@ def web_update(args=None) -> None:
 
     web_playlists = db_playlists.get_all(
         args,
-        sql_filters="extractor_key = 'WebFolder'",
+        sql_filters=["AND extractor_key = 'WebFolder'"],
         order_by="""length(path)-length(REPLACE(path, '/', '')) desc
         , random()
         """,
@@ -320,9 +321,9 @@ def web_update(args=None) -> None:
             new_media = spider(args_env, [playlist["path"]])
 
             if new_media > 0:
-                db_playlists.decrease_update_delay(args, playlist["path"])
+                db_playlists.update_more_frequently(args, playlist["path"])
             else:
-                db_playlists.increase_update_delay(args, playlist["path"])
+                db_playlists.update_less_frequently(args, playlist["path"])
 
             if playlist_count > 3:
                 time.sleep(random.uniform(0.05, 2))

library/mediadb/db_history.py

@@ -21,9 +21,15 @@ def exists(args, media_id) -> bool:
 def create(args):
     args.db.create_table(
         "history",
-        {"media_id": int, "time_played": int, "playhead": int, "done": int},
+        {
+            "media_id": int,
+            "time_played": int,
+            "playhead": int,
+            "done": bool,
+        },
         pk="id",
         if_not_exists=True,
+        strict=True,
     )
 
 

library/mediadb/db_media.py

@@ -1,6 +1,7 @@
 import argparse, os, sqlite3
 from collections.abc import Collection
 from pathlib import Path
+from typing import Counter
 
 from library.createdb import fs_add_metadata
 from library.createdb.subtitle import clean_up_temp_dirs
@@ -9,6 +10,31 @@
 from library.utils.log_utils import log
 
 
+def create(args):
+    args.db.create_table(
+        "media",
+        {
+            "id": int,
+            "playlists_id": int,
+            "time_created": int,
+            "time_modified": int,
+            "time_deleted": int,
+            "time_uploaded": int,
+            "time_downloaded": int,
+            "size": int,
+            "duration": int,
+            # "fps": float,
+            "path": str,
+            # "title": str,
+            # "webpath": str,
+            # "error": str,
+        },
+        pk="id",
+        if_not_exists=True,
+        strict=True,
+    )
+
+
 def exists(args, path) -> bool:
     m_columns = db_utils.columns(args, "media")
     try:
@@ -73,13 +99,16 @@ def consolidate(v: dict) -> dict | None:
     cv["time_modified"] = 0  # this should be 0 if the file has never been downloaded
     cv["time_deleted"] = 0
     cv["time_downloaded"] = 0
-    language = v.pop("language", None)
+    cv["language"] = iterables.safe_unpack(
+        v.pop("language", None),
+        v.pop("lang_code", None),
+    )
     cv["tags"] = strings.combine(
-        "language:" + language if language else None,
         v.pop("description", None),
         v.pop("caption", None),
         v.pop("content", None),
         v.pop("categories", None),
+        v.pop("catname", None),
         v.pop("genre", None),
         v.pop("tags", None),
         v.pop("labels", None),
@@ -121,6 +150,7 @@ def consolidate(v: dict) -> dict | None:
         account.pop("id", None),
         v.pop("name", None),
         v.pop("author", None),
+        v.pop("author_info", None),
         v.pop("post_author", None),
         v.pop("blog_name", None),
         v.pop("uuid", None),
@@ -541,39 +571,56 @@ def get_playlist_media(args, playlist_paths) -> list[dict]:
     return media
 
 
-def get_next_dir_media(args, folder):
+def get_next_dir_media(args, folder, limit=1):
     if args.play_in_order:
-        media = get_dir_media(args, [folder], limit=100)
+        media = get_dir_media(args, [folder], limit=limit * 100)
         media = natsort_media(args, media)
-        m = media[0:1]
+        m = media[0:limit]
     else:
-        m = get_dir_media(args, [folder], limit=1)[0:1]
+        m = get_dir_media(args, [folder], limit=limit)[0:limit]
     return m
 
 
 def get_sibling_media(args, media):
     if args.fetch_siblings in ("all", "always"):
         dirs = {str(Path(d["path"]).parent) + os.sep for d in media}
         media = get_dir_media(args, dirs)
+
     elif args.fetch_siblings == "each":
-        parents = {str(Path(d["path"]).parent) + os.sep for d in media}
+        parent_counts = Counter(str(Path(d["path"]).parent) + os.sep for d in media)
         media = []
-        for parent in parents:
-            media.extend(get_next_dir_media(args, parent))
+        for parent, count in parent_counts.items():
+            media.extend(
+                get_next_dir_media(
+                    args, parent, limit=min(args.fetch_siblings_max, count) if args.fetch_siblings_max > 0 else count
+                )
+            )
+
     elif args.fetch_siblings == "if-audiobook":
         new_media = []
-        seen = set()
+        seen_parents = {}
         for d in media:
             if "audiobook" in d["path"].lower():
                 parent = str(Path(d["path"]).parent) + os.sep
-                if parent not in seen:
-                    seen.add(parent)
-                    new_media.extend(get_next_dir_media(args, parent))
+                if parent not in seen_parents:
+                    seen_parents[parent] = sum(1 for m in media if str(Path(m["path"]).parent) + os.sep == parent)
+                    count = seen_parents[parent]
+                    new_media.extend(
+                        get_next_dir_media(
+                            args,
+                            parent,
+                            limit=min(args.fetch_siblings_max, count) if args.fetch_siblings_max > 0 else count,
+                        )
+                    )
             else:
                 new_media.append(d)
         media = new_media
 
-    # TODO: all-if>10, each-if=10 --folder-counts functionality could be replicated here
+    elif args.fetch_siblings.isdigit():
+        parents = {str(Path(d["path"]).parent) + os.sep for d in media}
+        media = []
+        for parent in parents:
+            media.extend(get_next_dir_media(args, parent, limit=int(args.fetch_siblings)))
 
     return media