diff --git a/buku b/buku index 21a52e93..41ef30a6 100755 --- a/buku +++ b/buku @@ -44,19 +44,13 @@ import webbrowser from enum import Enum from itertools import chain from subprocess import DEVNULL, PIPE, Popen -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, NamedTuple import urllib3 from bs4 import BeautifulSoup from urllib3.exceptions import LocationParseError from urllib3.util import Retry, make_headers, parse_url -# note catch ModuleNotFoundError instead Exception -# when python3.5 not supported -try: - import readline -except Exception: - import pyreadline as readline # type: ignore try: from mypy_extensions import TypedDict except ImportError: @@ -95,6 +89,24 @@ COLORMAP = {k: '\x1b[%sm' % v for k, v in { 'x': '0', 'X': '1', 'y': '7', 'Y': '7;1', 'z': '2', }.items()} +# DB flagset values +[FLAG_NONE, FLAG_IMMUTABLE] = [0x00, 0x01] + +FIELD_FILTER = { + 1: ('id', 'url'), + 2: ('id', 'url', 'tags'), + 3: ('id', 'title'), + 4: ('id', 'url', 'title', 'tags'), + 5: ('id', 'title', 'tags'), + 10: ('url',), + 20: ('url', 'tags'), + 30: ('title',), + 40: ('url', 'title', 'tags'), + 50: ('title', 'tags'), +} +ALL_FIELDS = ('id', 'url', 'title', 'desc', 'tags') +JSON_FIELDS = {'id': 'index', 'url': 'uri', 'desc': 'description'} + USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:101.0) Gecko/20100101 Firefox/101.0' MYHEADERS = None # Default dictionary of headers MYPROXY = None # Default proxy @@ -362,9 +374,28 @@ class BukuCrypt: sys.exit(1) -BookmarkVar = Tuple[int, str, Optional[str], str, str, int] -# example: -# (1, 'http://example.com', 'example title', ',tags1,', 'randomdesc', 0)) +class BookmarkVar(NamedTuple): + """Bookmark data named tuple""" + id: int + url: str + title: Optional[str] = None + tags_raw: str = '' + desc: str = '' + flags: int = FLAG_NONE + + @property + def immutable(self) -> bool: + return bool(self.flags & FLAG_IMMUTABLE) + + @property + def tags(self) -> str: + return self.tags_raw[1:-1] + + @property + def taglist(self) -> List[str]: + return [x for x in self.tags_raw.split(',') if x] + +bookmark_vars = lambda xs: (BookmarkVar(*x) for x in xs) class BukuDb: @@ -385,8 +416,8 @@ class BukuDb: """ def __init__( - self, json: Optional[str] = None, field_filter: Optional[int] = 0, chatty: Optional[bool] = False, - dbfile: Optional[str] = None, colorize: Optional[bool] = True) -> None: + self, json: Optional[str] = None, field_filter: int = 0, chatty: bool = False, + dbfile: Optional[str] = None, colorize: bool = True) -> None: """Database initialization API. Parameters @@ -395,11 +426,11 @@ class BukuDb: Empty string if results should be printed in JSON format to stdout. Nonempty string if results should be printed in JSON format to file. The string has to be a valid path. None if the results should be printed as human-readable plaintext. - field_filter : int, optional + field_filter : int Indicates format for displaying bookmarks. Default is 0. - chatty : bool, optional + chatty : bool Sets the verbosity of the APIs. Default is False. - colorize : bool, optional + colorize : bool Indicates whether color should be used in output. Default is True. """ @@ -439,7 +470,7 @@ class BukuDb: return os.path.join(data_home, 'buku') @staticmethod - def initdb(dbfile: Optional[str] = None, chatty: Optional[bool] = False) -> Tuple[sqlite3.Connection, sqlite3.Cursor]: + def initdb(dbfile: Optional[str] = None, chatty: bool = False) -> Tuple[sqlite3.Connection, sqlite3.Cursor]: """Initialize the database connection. Create DB file and/or bookmarks table if they don't exist. @@ -514,7 +545,7 @@ class BukuDb: def _fetch(self, query: str, *args) -> List[BookmarkVar]: self.cur.execute(query, args) - return self.cur.fetchall() + return [BookmarkVar(*x) for x in self.cur.fetchall()] def _fetch_first(self, query: str, *args) -> Optional[BookmarkVar]: rows = self._fetch(query + ' LIMIT 1', *args) @@ -562,7 +593,7 @@ class BukuDb: """ row = self._fetch_first('SELECT * FROM bookmarks WHERE url = ?', url) - return row and row[0] + return row and row.id def get_max_id(self) -> int: """Fetch the ID of the last record. @@ -582,9 +613,9 @@ class BukuDb: title_in: Optional[str] = None, tags_in: Optional[str] = None, desc: Optional[str] = None, - immutable: Optional[int] = 0, - delay_commit: Optional[bool] = False, - fetch: Optional[bool] = True) -> int: + immutable: bool = False, + delay_commit: bool = False, + fetch: bool = True) -> int: """Add a new bookmark. Parameters @@ -598,13 +629,12 @@ class BukuDb: Must start and end with comma. Default is None. desc : str, optional Description of the bookmark. Default is None. - immutable : int, optional - Indicates whether to disable title fetch from web. - Default is 0. - delay_commit : bool, optional + immutable : bool + Indicates whether to disable title fetch from web. Default is False. + delay_commit : bool True if record should not be committed to the DB, leaving commit responsibility to caller. Default is False. - fetch : bool, optional + fetch : bool Fetch page from web and parse for data Returns @@ -614,7 +644,7 @@ class BukuDb: """ # Return error for empty URL - if not url or url == '': + if not url: LOGERR('Invalid URL') return None @@ -650,9 +680,9 @@ class BukuDb: desc = '' if pdesc is None else pdesc try: - flagset = 0 - if immutable == 1: - flagset |= immutable + flagset = FLAG_NONE + if immutable: + flagset |= FLAG_IMMUTABLE qry = 'INSERT INTO bookmarks(URL, metadata, tags, desc, flags) VALUES (?, ?, ?, ?, ?)' self.cur.execute(qry, (url, ptitle, tags_in, desc, flagset)) @@ -674,7 +704,7 @@ class BukuDb: DB index of the record. 0 indicates all records. tags_in : str Comma-separated tags to add manually. - delay_commit : bool, optional + delay_commit : bool True if record should not be committed to the DB, leaving commit responsibility to caller. Default is False. @@ -722,10 +752,10 @@ class BukuDb: DB index of bookmark record. 0 indicates all records. tags_in : str Comma-separated tags to delete manually. - delay_commit : bool, optional + delay_commit : bool True if record should not be committed to the DB, leaving commit responsibility to caller. Default is False. - chatty: bool, optional + chatty: bool Skip confirmation when set to False. Returns @@ -792,7 +822,7 @@ class BukuDb: title_in: Optional[str] = None, tags_in: Optional[str] = None, desc: Optional[str] = None, - immutable: Optional[int] = -1, + immutable: Optional[bool] = None, threads: int = 4) -> bool: """Update an existing record at index. @@ -813,9 +843,9 @@ class BukuDb: Prefix with '-,' to delete from current tags. desc : str, optional Description of bookmark. - immutable : int, optional - Disable title fetch from web if 1. Default is -1. - threads : int, optional + immutable : bool, optional + Disable title fetch from web if True. Default is None (no change). + threads : int Number of threads to use to refresh full DB. Default is 4. Returns @@ -871,15 +901,13 @@ class BukuDb: to_update = True # Update immutable flag if passed as argument - if immutable != -1: - flagset = 1 - if immutable == 1: + if immutable is not None: + if immutable: query += ' flags = flags | ?,' - elif immutable == 0: + arguments += (FLAG_IMMUTABLE,) + else: query += ' flags = flags & ?,' - flagset = ~flagset - - arguments += (flagset,) + arguments += (~FLAG_IMMUTABLE,) to_update = True # Update title @@ -1115,7 +1143,7 @@ class BukuDb: self.conn.commit() return True - def edit_update_rec(self, index, immutable=-1): + def edit_update_rec(self, index, immutable=None): """Edit in editor and update a record. Parameters @@ -1123,8 +1151,8 @@ class BukuDb: index : int DB index of the record. Last record, if index is -1. - immutable : int, optional - Diable title fetch from web if 1. Default is -1. + immutable : bool, optional + Diable title fetch from web if True. Default is None (no change). Returns ------- @@ -1151,14 +1179,13 @@ class BukuDb: # If reading from DB, show empty title and desc as empty lines. We have to convert because # even in case of add with a blank title or desc, '' is used as initializer to show '-'. - result = edit_rec(editor, rec[1], rec[2] if rec[2] != '' else None, - rec[3], rec[4] if rec[4] != '' else None) + result = edit_rec(editor, rec.url, rec.title or None, rec.tags_raw, rec.desc or None) if result is not None: url, title, tags, desc = result return self.update_rec(index, url, title, tags, desc, immutable) - if immutable != -1: - return self.update_rec(index, immutable) + if immutable is not None: + return self.update_rec(index, immutable=immutable) return False @@ -1203,9 +1230,9 @@ class BukuDb: def searchdb( self, keywords: List[str], - all_keywords: Optional[bool] = False, - deep: Optional[bool] = False, - regex: Optional[bool] = False + all_keywords: bool = False, + deep: bool = False, + regex: bool = False ) -> List[BookmarkVar]: """Search DB for entries where tags, URL, or title fields match keywords. @@ -1213,12 +1240,12 @@ class BukuDb: ---------- keywords : list of str Keywords to search. - all_keywords : bool, optional + all_keywords : bool True to return records matching ALL keywords. False (default value) to return records matching ANY keyword. - deep : bool, optional + deep : bool True to search for matching substrings. Default is False. - regex : bool, optional + regex : bool Match a regular expression if True. Default is False. Returns @@ -1395,12 +1422,12 @@ class BukuDb: ---------- keywords : list of str Keywords to search. - all_keywords : bool, optional + all_keywords : bool True to return records matching ALL keywords. False to return records matching ANY keyword. - deep : bool, optional + deep : bool True to search for matching substrings. - regex : bool, optional + regex : bool Match a regular expression if True. stag : str String of tags to search for. @@ -1428,7 +1455,7 @@ class BukuDb: List of search results without : list of str Keywords to search. - deep : bool, optional + deep : bool True to search for matching substrings. Returns @@ -1447,7 +1474,7 @@ class BukuDb: ---------- index : int DB index of deleted entry. - delay_commit : bool, optional + delay_commit : bool True if record should not be committed to the DB, leaving commit responsibility to caller. Default is False. """ @@ -1465,12 +1492,12 @@ class BukuDb: if max_id > index: results = self._fetch(query1, max_id) for row in results: - self.cur.execute(query2, (row[0],)) - self.cur.execute(query3, (index, row[1], row[2], row[3], row[4], row[5])) + self.cur.execute(query2, (row.id,)) + self.cur.execute(query3, (index, row.url, row.title, row.tags_raw, row.desc, row.flags)) if not delay_commit: self.conn.commit() if self.chatty: - print('Index %d moved to %d' % (row[0], index)) + print('Index %d moved to %d' % (row.id, index)) def delete_rec( self, @@ -1486,14 +1513,14 @@ class BukuDb: ---------- index : int, optional DB index of deleted entry. - low : int, optional + low : int Actual lower index of range. - high : int, optional + high : int Actual higher index of range. - is_range : bool, optional + is_range : bool A range is passed using low and high arguments. An index is ignored if is_range is True. - delay_commit : bool, optional + delay_commit : bool True if record should not be committed to the DB, leaving commit responsibility to caller. Default is False. @@ -1681,7 +1708,7 @@ class BukuDb: Parameters ---------- - delay_commit : bool, optional + delay_commit : bool True if record should not be committed to the DB, leaving commit responsibility to caller. Default is False. @@ -1731,13 +1758,13 @@ class BukuDb: Parameters ----------- - index : int, optional + index : int DB index of record to print. 0 prints all records. - low : int, optional + low : int Actual lower index of range. - high : int, optional + high : int Actual higher index of range. - is_range : bool, optional + is_range : bool A range is passed using low and high arguments. An index is ignored if is_range is True. @@ -1932,7 +1959,7 @@ class BukuDb: return parse_tags(tags) - def replace_tag(self, orig: str, new: Optional[List[str]] = None) -> bool: + def replace_tag(self, orig: str, new: List[str] = []) -> bool: """Replace original tag by new tags in all records. Remove original tag if new tag is empty. @@ -1950,11 +1977,8 @@ class BukuDb: True on success, False on failure. """ - newtags = DELIM - orig = delim_wrap(orig) - if new is not None: - newtags = parse_tags(new) + newtags = parse_tags(new) if new else DELIM if orig == newtags: print('Tags are same.') @@ -2226,7 +2250,7 @@ class BukuDb: outdb = BukuDb(dbfile=filepath) qry = 'INSERT INTO bookmarks(URL, metadata, tags, desc, flags) VALUES (?, ?, ?, ?, ?)' for row in resultset: - outdb.cur.execute(qry, (row[1], row[2], row[3], row[4], row[5])) + outdb.cur.execute(qry, (row.url, row.title, row.tags_raw, row.desc, row.flags)) count += 1 outdb.conn.commit() outdb.close() @@ -2381,10 +2405,7 @@ class BukuDb: tags = parse_tags(formatted_tags) # get the title - if row[2]: - title = row[2] - else: - title = '' + title = row[2] or '' self.add_rec(url, title, tags, None, 0, True, False) try: @@ -2569,7 +2590,7 @@ class BukuDb: ---------- filepath : str Path to file to import. - tacit : bool, optional + tacit : bool If True, no questions asked and folder names are automatically imported as tags from bookmarks HTML. If True, automatic timestamp tag is NOT added. @@ -2709,8 +2730,8 @@ n: don't add parent folder as tag resultset = indb_cur.fetchall() if resultset: - for row in resultset: - self.add_rec(row[1], row[2], row[3], row[4], row[5], True, False) + for row in bookmark_vars(resultset): + self.add_rec(row.url, row.title, row.tags_raw, row.desc, row.flags, True, False) self.conn.commit() @@ -2726,7 +2747,7 @@ n: don't add parent folder as tag self, index: Optional[int] = None, url: Optional[str] = None, - shorten: Optional[bool] = True) -> Optional[str]: + shorten: bool = True) -> Optional[str]: """Shorten a URL using Google URL shortener. Parameters @@ -2735,7 +2756,7 @@ n: don't add parent folder as tag DB index of the bookmark with the URL to shorten. Default is None. url : str, optional (if index is provided) URL to shorten. - shorten : bool, optional + shorten : bool True to shorten, False to expand. Default is False. Returns @@ -2894,7 +2915,7 @@ n: don't add parent folder as tag Parameters ---------- - exitval : int, optional + exitval : int Program exit value. """ @@ -2917,7 +2938,7 @@ class ExtendedArgumentParser(argparse.ArgumentParser): Parameters ---------- - file : file, optional + file : file File to write program info to. Default is sys.stdout. """ if sys.platform == 'win32' and file == sys.stdout: @@ -2941,7 +2962,7 @@ Webpage: https://github.com/jarun/buku Parameters ---------- - file : file, optional + file : file File to write program info to. Default is sys.stdout. """ file.write(''' @@ -3001,7 +3022,7 @@ PROMPT KEYS: Parameters ---------- - file : file, optional + file : file File to write program info to. Default is sys.stdout. """ super().print_help(file) @@ -3046,30 +3067,24 @@ def convert_bookmark_set( import html assert export_type in ['markdown', 'html', 'org', 'xbel'] # compatibility - resultset = bookmark_set + resultset = bookmark_vars(bookmark_set) count = 0 out = '' if export_type == 'markdown': for row in resultset: - if not row[2] or row[2] is None: - out += '- [Untitled](' + row[1] + ')' - else: - out += '- [' + row[2] + '](' + row[1] + ')' + out += '- [' + (row.title or 'Untitled') + '](' + row.url + ')' - if row[3] != DELIM: - out += ' \n'.format(row[3][1:-1]) + if row.tags: + out += ' \n'.format(row.tags) else: out += '\n' count += 1 elif export_type == 'org': for row in resultset: - if not row[2]: - out += '* [[{}][Untitled]]'.format(row[1]) - else: - out += '* [[{}][{}]]'.format(row[1], row[2]) - out += convert_tags_to_org_mode_tags(row[3]) + out += '* [[{}][{}]]'.format(row.url, row.title or 'Untitled') + out += convert_tags_to_org_mode_tags(row.tags_raw) count += 1 elif export_type == 'xbel': timestamp = str(int(time.time())) @@ -3081,11 +3096,11 @@ def convert_bookmark_set( '\n') for row in resultset: - out += '

\n'.format(timestamp)) for row in resultset: - out += '

Tuple[Optional[str], Optional[str], Optional[str], int, int]: """Handle server connection and redirections. @@ -3978,7 +3993,7 @@ def parse_tags(keywords=[]): Parameters ---------- - keywords : list, optional + keywords : list List of tags to parse. Default is empty list. Returns @@ -3994,7 +4009,7 @@ def parse_tags(keywords=[]): if keywords is None: return None - if not keywords or len(keywords) < 1 or not keywords[0]: + if not keywords or not keywords[0]: return DELIM tags = DELIM @@ -4114,7 +4129,7 @@ def edit_at_prompt(obj, nav, suggest=False): A valid instance of BukuDb class. nav : str Navigation command argument passed at prompt by user. - suggest : bool, optional + suggest : bool If True, suggest similar tags on new bookmark addition. """ @@ -4166,15 +4181,15 @@ def prompt(obj, results, noninteractive=False, deep=False, listtags=False, sugge A valid instance of BukuDb class. results : list Search result set from a DB query. - noninteractive : bool, optional + noninteractive : bool If True, does not seek user input. Shows all results. Default is False. - deep : bool, optional + deep : bool Use deep search. Default is False. - listtags : bool, optional + listtags : bool If True, list all tags. - suggest : bool, optional + suggest : bool If True, suggest similar tags on edit and add bookmark. - num : int, optional + num : int Number of results to show per page. Default is 10. """ @@ -4481,53 +4496,29 @@ def print_rec_with_filter(records, field_filter=0): records : list or sqlite3.Cursor object List of bookmark records to print field_filter : int - Integer indicating which fields to print. + Integer indicating which fields to print. Default is 0 ("all fields"). """ try: - if field_filter == 0: + records = bookmark_vars(records) + fields = FIELD_FILTER.get(field_filter) + if fields: + pattern = '\t'.join('%s' for k in fields) for row in records: - try: - columns, _ = os.get_terminal_size() - except OSError: - columns = 0 - print_single_rec(row, columns=columns) - elif field_filter == 1: - for row in records: - print('%s\t%s' % (row[0], row[1])) - elif field_filter == 2: - for row in records: - print('%s\t%s\t%s' % (row[0], row[1], row[3][1:-1])) - elif field_filter == 3: - for row in records: - print('%s\t%s' % (row[0], row[2])) - elif field_filter == 4: - for row in records: - print('%s\t%s\t%s\t%s' % (row[0], row[1], row[2], row[3][1:-1])) - elif field_filter == 5: - for row in records: - print('%s\t%s\t%s' % (row[0], row[2], row[3][1:-1])) - elif field_filter == 10: - for row in records: - print(row[1]) - elif field_filter == 20: - for row in records: - print('%s\t%s' % (row[1], row[3][1:-1])) - elif field_filter == 30: - for row in records: - print(row[2]) - elif field_filter == 40: - for row in records: - print('%s\t%s\t%s' % (row[1], row[2], row[3][1:-1])) - elif field_filter == 50: + print(pattern % tuple(getattr(row, k) for k in fields)) + else: + try: + columns, _ = os.get_terminal_size() + except OSError: + columns = 0 for row in records: - print('%s\t%s' % (row[2], row[3][1:-1])) + print_single_rec(row, columns=columns) except BrokenPipeError: sys.stdout = os.fdopen(1) sys.exit(1) -def print_single_rec(row: BookmarkVar, idx: Optional[int]=0, columns: Optional[int]=0): # NOQA +def print_single_rec(row: BookmarkVar, idx: int=0, columns: int=0): # NOQA """Print a single DB record. Handles both search results and individual record. @@ -4536,35 +4527,36 @@ def print_single_rec(row: BookmarkVar, idx: Optional[int]=0, columns: Optional[i ---------- row : tuple Tuple representing bookmark record data. - idx : int, optional + idx : int Search result index. If 0, print with DB index. Default is 0. - columns : int, optional + columns : int Number of columns to wrap comments to. Default is 0. """ str_list = [] + row = BookmarkVar(*row) # ensuring named tuple # Start with index and title if idx != 0: - id_title_res = ID_STR % (idx, row[2] if row[2] else 'Untitled', row[0]) + id_title_res = ID_STR % (idx, row.title or 'Untitled', row.id) else: - id_title_res = ID_DB_STR % (row[0], row[2] if row[2] else 'Untitled') + id_title_res = ID_DB_STR % (row.id, row.title or 'Untitled') # Indicate if record is immutable - if row[5] & 1: - id_title_res = MUTE_STR % (id_title_res) + if row.immutable: + id_title_res = MUTE_STR % (id_title_res,) else: id_title_res += '\n' try: print(id_title_res, end='') - print(URL_STR % (row[1]), end='') + print(URL_STR % (row.url,), end='') if columns == 0: - if row[4]: - print(DESC_STR % (row[4]), end='') - if row[3] != DELIM: - print(TAG_STR % (row[3][1:-1]), end='') + if row.desc: + print(DESC_STR % (row.desc,), end='') + if row.tags: + print(TAG_STR % (row.tags,), end='') print() return @@ -4572,7 +4564,7 @@ def print_single_rec(row: BookmarkVar, idx: Optional[int]=0, columns: Optional[i ln_num = 1 fillwidth = columns - INDENT - for line in textwrap.wrap(row[4].replace('\n', ''), width=fillwidth): + for line in textwrap.wrap(row.desc.replace('\n', ''), width=fillwidth): if ln_num == 1: print(DESC_STR % line, end='') ln_num += 1 @@ -4580,7 +4572,7 @@ def print_single_rec(row: BookmarkVar, idx: Optional[int]=0, columns: Optional[i print(DESC_WRAP % (' ' * INDENT, line)) ln_num = 1 - for line in textwrap.wrap(row[3][1:-1].replace('\n', ''), width=fillwidth): + for line in textwrap.wrap(row.tags.replace('\n', ''), width=fillwidth): if ln_num == 1: print(TAG_STR % line, end='') ln_num += 1 @@ -4590,11 +4582,11 @@ def print_single_rec(row: BookmarkVar, idx: Optional[int]=0, columns: Optional[i except UnicodeEncodeError: str_list = [] str_list.append(id_title_res) - str_list.append(URL_STR % (row[1])) - if row[4]: - str_list.append(DESC_STR % (row[4])) - if row[3] != DELIM: - str_list.append(TAG_STR % (row[3][1:-1])) + str_list.append(URL_STR % (row.url,)) + if row.desc: + str_list.append(DESC_STR % (row.desc,)) + if row.tags: + str_list.append(TAG_STR % (row.tags,)) sys.stdout.buffer.write((''.join(str_list) + '\n').encode('utf-8')) except BrokenPipeError: sys.stdout = os.fdopen(1) @@ -4627,10 +4619,10 @@ def format_json(resultset, single_record=False, field_filter=0): ---------- resultset : list Search results from DB query. - single_record : bool, optional + single_record : bool If True, indicates only one record. Default is False. - field_filter : int, optional - Indicates format for displaying bookmarks. Default is 0. + field_filter : int + Indicates format for displaying bookmarks. Default is 0 ("all fields"). Returns ------- @@ -4638,47 +4630,11 @@ def format_json(resultset, single_record=False, field_filter=0): Record(s) in JSON format. """ + resultset = bookmark_vars(resultset) + fields = [(k, JSON_FIELDS.get(k, k)) for k in FIELD_FILTER.get(field_filter, ALL_FIELDS)] + marks = [{field: getattr(row, k) for k, field in fields} for row in resultset] if single_record: - marks = {} - for row in resultset: - if field_filter == 1: - marks['uri'] = row[1] - elif field_filter == 2: - marks['uri'] = row[1] - marks['tags'] = row[3][1:-1] - elif field_filter == 3: - marks['title'] = row[2] - elif field_filter == 4: - marks['uri'] = row[1] - marks['tags'] = row[3][1:-1] - marks['title'] = row[2] - else: - marks['index'] = row[0] - marks['uri'] = row[1] - marks['title'] = row[2] - marks['description'] = row[4] - marks['tags'] = row[3][1:-1] - else: - marks = [] - for row in resultset: - if field_filter == 1: - record = {'uri': row[1]} - elif field_filter == 2: - record = {'uri': row[1], 'tags': row[3][1:-1]} - elif field_filter == 3: - record = {'title': row[2]} - elif field_filter == 4: - record = {'uri': row[1], 'title': row[2], 'tags': row[3][1:-1]} - else: - record = { - 'index': row[0], - 'uri': row[1], - 'title': row[2], - 'description': row[4], - 'tags': row[3][1:-1] - } - - marks.append(record) + marks = marks[-1] if marks else {} return json.dumps(marks, sort_keys=True, indent=4) @@ -4690,10 +4646,10 @@ def print_json_safe(resultset, single_record=False, field_filter=0): ---------- resultset : list Search results from DB query. - single_record : bool, optional + single_record : bool If True, indicates only one record. Default is False. - field_filter : int, optional - Indicates format for displaying bookmarks. Default is 0. + field_filter : int + Indicates format for displaying bookmarks. Default is 0 ("all fields"). Returns ------- @@ -5324,6 +5280,11 @@ def monkeypatch_textwrap_for_cjk(): def main(): """Main.""" global ID_STR, ID_DB_STR, MUTE_STR, URL_STR, DESC_STR, DESC_WRAP, TAG_STR, TAG_WRAP, PROMPTMSG + # readline should not be loaded when buku is used as a library + try: + import readline + except ImportError: + import pyreadline3 as readline # type: ignore title_in = None tags_in = None @@ -5410,7 +5371,9 @@ POSITIONAL ARGUMENTS: addarg('--tag', nargs='*', help=hide) addarg('--title', nargs='*', help=hide) addarg('-c', '--comment', nargs='*', help=hide) - addarg('--immutable', type=int, default=-1, choices={0, 1}, help=hide) + addarg('--immutable', type=int, choices={0, 1}, help=hide) + _bool = lambda x: x if x is None else bool(x) + _immutable = lambda args: _bool(args.immutable) # -------------------- # SEARCH OPTIONS GROUP @@ -5641,7 +5604,7 @@ POSITIONAL ARGUMENTS: bdb.close_quit(1) if is_int(args.write): - if not bdb.edit_update_rec(int(args.write), args.immutable): + if not bdb.edit_update_rec(int(args.write), _immutable(args)): bdb.close_quit(1) elif args.add is None: # Edit and add a new bookmark @@ -5661,7 +5624,7 @@ POSITIONAL ARGUMENTS: url, title_in, tags, desc_in = result if args.suggest: tags = bdb.suggest_similar_tag(tags) - bdb.add_rec(url, title_in, tags, desc_in, args.immutable) + bdb.add_rec(url, title_in, tags, desc_in, _immutable(args)) # Add record if args.add is not None: @@ -5699,7 +5662,7 @@ POSITIONAL ARGUMENTS: if edit_aborted is False: if args.suggest: tags = bdb.suggest_similar_tag(tags) - bdb.add_rec(url, title_in, tags, desc_in, args.immutable) + bdb.add_rec(url, title_in, tags, desc_in, _immutable(args)) # Search record search_results = None @@ -5886,7 +5849,7 @@ POSITIONAL ARGUMENTS: if not args.update: # Update all records only if search was not opted if not search_opted: - bdb.update_rec(0, url_in, title_in, tags, desc_in, args.immutable, args.threads) + bdb.update_rec(0, url_in, title_in, tags, desc_in, _immutable(args), args.threads) elif search_results and search_results is not None and update_search_results: if not args.tacit: print('Updated results:\n') @@ -5900,7 +5863,7 @@ POSITIONAL ARGUMENTS: title_in, tags, desc_in, - args.immutable, + _immutable(args), args.threads ) @@ -5918,7 +5881,7 @@ POSITIONAL ARGUMENTS: title_in, tags, desc_in, - args.immutable, + _immutable(args), args.threads ) elif '-' in idx: @@ -5935,7 +5898,7 @@ POSITIONAL ARGUMENTS: title_in, tags, desc_in, - args.immutable, + _immutable(args), args.threads ) else: @@ -5946,7 +5909,7 @@ POSITIONAL ARGUMENTS: title_in, tags, desc_in, - args.immutable, + _immutable(args), args.threads ) if INTERRUPTED: diff --git a/bukuserver/api.py b/bukuserver/api.py index 5ab984ab..03a06244 100644 --- a/bukuserver/api.py +++ b/bukuserver/api.py @@ -111,13 +111,13 @@ def get(self, rec_id: Union[int, None]): result = {'bookmarks': []} # type: Dict[str, Any] for bookmark in all_bookmarks: result_bookmark = { - 'url': bookmark[1], - 'title': bookmark[2], - 'tags': [x for x in bookmark[3].split(',') if x], - 'description': bookmark[4] + 'url': bookmark.url, + 'title': bookmark.title, + 'tags': bookmark.taglist, + 'description': bookmark.desc } if not request.path.startswith('/api/'): - result_bookmark['id'] = bookmark[0] + result_bookmark['id'] = bookmark.id result['bookmarks'].append(result_bookmark) res = jsonify(result) else: @@ -127,10 +127,10 @@ def get(self, rec_id: Union[int, None]): res = response_bad() else: res = jsonify({ - 'url': bookmark[1], - 'title': bookmark[2], - 'tags': [x for x in bookmark[3].split(',') if x], - 'description': bookmark[4] + 'url': bookmark.url, + 'title': bookmark.title, + 'tags': bookmark.taglist, + 'description': bookmark.desc, }) return res @@ -178,10 +178,10 @@ def get(self, starting_id: int, ending_id: int): for i in range(starting_id, ending_id + 1, 1): bookmark = bukudb.get_rec_by_id(i) result['bookmarks'][i] = { - 'url': bookmark[1], - 'title': bookmark[2], - 'tags': [x for x in bookmark[3].split(',') if x], - 'description': bookmark[4] + 'url': bookmark.url, + 'title': bookmark.title, + 'tags': bookmark.taglist, + 'description': bookmark.desc, } return jsonify(result) @@ -236,11 +236,11 @@ def get(self): res = None for bookmark in bukudb.searchdb(keywords, all_keywords, deep, regex): result_bookmark = { - 'id': bookmark[0], - 'url': bookmark[1], - 'title': bookmark[2], - 'tags': list(filter(lambda x: x, bookmark[3].split(','))), - 'description': bookmark[4] + 'id': bookmark.id, + 'url': bookmark.url, + 'title': bookmark.title, + 'tags': bookmark.taglist, + 'description': bookmark.desc, } result['bookmarks'].append(result_bookmark) current_app.logger.debug('total bookmarks:{}'.format(len(result['bookmarks']))) @@ -266,7 +266,7 @@ def delete(self): bukudb = getattr(flask.g, 'bukudb', get_bukudb()) res = None for bookmark in bukudb.searchdb(keywords, all_keywords, deep, regex): - if not bukudb.delete_rec(bookmark[0]): + if not bukudb.delete_rec(bookmark.id): res = response_bad() return res or response_ok() diff --git a/bukuserver/views.py b/bukuserver/views.py index 8ae288f8..c2c9cd17 100644 --- a/bukuserver/views.py +++ b/bukuserver/views.py @@ -259,15 +259,7 @@ def get_list(self, page, sort_field, sort_desc, _, filters, page_size=None): for bookmark in bookmarks: bm_sns = types.SimpleNamespace(id=None, url=None, title=None, tags=None, description=None) for field in list(BookmarkField): - if field == BookmarkField.TAGS: - value = bookmark[field.value] - if value.startswith(","): - value = value[1:] - if value.endswith(","): - value = value[:-1] - setattr(bm_sns, field.name.lower(), value) - else: - setattr(bm_sns, field.name.lower(), bookmark[field.value]) + setattr(bm_sns, field.name.lower(), format_value(field, bookmark)) data.append(bm_sns) return count, data @@ -277,17 +269,8 @@ def get_one(self, id): return None bm_sns = types.SimpleNamespace(id=None, url=None, title=None, tags=None, description=None) for field in list(BookmarkField): - if field == BookmarkField.TAGS and bookmark[field.value].startswith(","): - value = bookmark[field.value] - if value.startswith(","): - value = value[1:] - if value.endswith(","): - value = value[:-1] - setattr(bm_sns, field.name.lower(), value.replace(',', ', ')) - else: - setattr(bm_sns, field.name.lower(), bookmark[field.value]) - if field == BookmarkField.URL: - session['netloc'] = urlparse(bookmark[field.value]).netloc + setattr(bm_sns, field.name.lower(), format_value(field, bookmark, spacing=' ')) + session['netloc'] = urlparse(bookmark.url).netloc return bm_sns def get_pk_value(self, model): @@ -739,3 +722,7 @@ def page_of(items, size, idx): def filter_key(flt, idx=''): return 'flt' + str(idx) + '_' + BookmarkModelView._filter_arg(flt) + +def format_value(field, bookmark, spacing=''): + s = bookmark[field.value] + return s if field != BookmarkField.TAGS else s.strip(',').replace(',', ','+spacing) diff --git a/requirements.txt b/requirements.txt index a1b6165c..1ff0a36c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ cryptography>=1.2.3 html5lib>=1.0.1 setuptools urllib3>=1.23 -pyreadline; sys_platform == 'windows' +pyreadline3; sys_platform == 'win32' diff --git a/setup.py b/setup.py index a74d50bb..1f8620e8 100644 --- a/setup.py +++ b/setup.py @@ -54,8 +54,8 @@ 'certifi', 'cryptography>=1.2.3', 'html5lib>=1.0.1', - 'pyreadline; sys_platform == \'windows\'', 'urllib3>=1.23', + 'pyreadline3; sys_platform == \'win32\'', ] setup( diff --git a/tests/cassettes/test_buku/test_network_handler_with_url[http---www.vim.org-scripts-script.php-script_id=4641-exp_res7].yaml b/tests/cassettes/test_buku/test_network_handler_with_url[http---www.vim.org-scripts-script.php~-exp_res7].yaml similarity index 100% rename from tests/cassettes/test_buku/test_network_handler_with_url[http---www.vim.org-scripts-script.php-script_id=4641-exp_res7].yaml rename to tests/cassettes/test_buku/test_network_handler_with_url[http---www.vim.org-scripts-script.php~-exp_res7].yaml diff --git a/tests/cassettes/test_buku/test_network_handler_with_url[https---www.google.ru-search-newwindow=1&safe=off&q=xkbcomp+alt+gr&oq=xkbcomp+alt+gr&gs_l=serp.3..33i21.28976559.28977886.0.28978017.6.6.0.0.0.0.167.668.0j5.5.0....0...1c.1.64.serp..1.2.311.06cSKPTLo18-exp_res6].yaml b/tests/cassettes/test_buku/test_network_handler_with_url[https---www.google.ru-search~-exp_res6].yaml similarity index 100% rename from tests/cassettes/test_buku/test_network_handler_with_url[https---www.google.ru-search-newwindow=1&safe=off&q=xkbcomp+alt+gr&oq=xkbcomp+alt+gr&gs_l=serp.3..33i21.28976559.28977886.0.28978017.6.6.0.0.0.0.167.668.0j5.5.0....0...1c.1.64.serp..1.2.311.06cSKPTLo18-exp_res6].yaml rename to tests/cassettes/test_buku/test_network_handler_with_url[https---www.google.ru-search~-exp_res6].yaml diff --git a/tests/test_buku.py b/tests/test_buku.py index 2641a10e..c1c341dd 100644 --- a/tests/test_buku.py +++ b/tests/test_buku.py @@ -11,7 +11,11 @@ import pytest -from buku import DELIM, is_int, prep_tag_search +from buku import DELIM, FIELD_FILTER, ALL_FIELDS, is_int, prep_tag_search, print_rec_with_filter + +def strip_query(s): + if isinstance(s, str) and '?' in s: + return s.split('?')[0] + '~' only_python_3_5 = pytest.mark.skipif(sys.version_info < (3, 5), reason="requires Python 3.5 or later") @@ -133,86 +137,25 @@ def test_parse_tags_no_args(): assert buku.parse_tags() == DELIM -@pytest.mark.parametrize( - "records, field_filter, exp_res", - [ - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 1, - ["1\thttp://url1.com", "2\thttp://url2.com"], - ], - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 2, - ["1\thttp://url1.com\ttag1", "2\thttp://url2.com\ttag1,tag2"], - ], - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 3, - ["1\ttitle1", "2\ttitle2"], - ], - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 4, - [ - "1\thttp://url1.com\ttitle1\ttag1", - "2\thttp://url2.com\ttitle2\ttag1,tag2", - ], - ], - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 10, - ["http://url1.com", "http://url2.com"], - ], - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 20, - ["http://url1.com\ttag1", "http://url2.com\ttag1,tag2"], - ], - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 30, - ["title1", "title2"], - ], - [ - [ - (1, "http://url1.com", "title1", ",tag1,"), - (2, "http://url2.com", "title2", ",tag1,tag2,"), - ], - 40, - ["http://url1.com\ttitle1\ttag1", "http://url2.com\ttitle2\ttag1,tag2"], - ], - ], -) -def test_print_rec_with_filter(records, field_filter, exp_res): - """test func.""" - with mock.patch("buku.print", create=True) as m_print: - import buku - - buku.print_rec_with_filter(records, field_filter) - for res in exp_res: - m_print.assert_any_call(res) +@pytest.mark.parametrize("field_filter, exp_res", [ + (0, ["1. title1\n > http://url1.com\n + desc1\n # tag1\n", + "2. title2\n > http://url2.com\n + desc2\n # tag1,tag2\n"]), + (1, ["1\thttp://url1.com", "2\thttp://url2.com"]), + (2, ["1\thttp://url1.com\ttag1", "2\thttp://url2.com\ttag1,tag2"]), + (3, ["1\ttitle1", "2\ttitle2"]), + (4, ["1\thttp://url1.com\ttitle1\ttag1", "2\thttp://url2.com\ttitle2\ttag1,tag2"]), + (5, ["1\ttitle1\ttag1", "2\ttitle2\ttag1,tag2"]), + (10, ["http://url1.com", "http://url2.com"]), + (20, ["http://url1.com\ttag1", "http://url2.com\ttag1,tag2"]), + (30, ["title1", "title2"]), + (40, ["http://url1.com\ttitle1\ttag1", "http://url2.com\ttitle2\ttag1,tag2"]), + (50, ["title1\ttag1", "title2\ttag1,tag2"]), +]) +def test_print_rec_with_filter(capfd, field_filter, exp_res): + records = [(1, "http://url1.com", "title1", ",tag1,", "desc1"), + (2, "http://url2.com", "title2", ",tag1,tag2,", "desc2")] + print_rec_with_filter(records, field_filter) + assert capfd.readouterr().out == ''.join(f'{s}\n' for s in exp_res) @pytest.mark.parametrize( @@ -263,24 +206,22 @@ def test_edit_at_prompt(nav, is_editor_valid_retval, edit_rec_retval): obj.add_rec(*edit_rec_retval) -@pytest.mark.parametrize("field_filter, single_record", product(list(range(4)), [True, False])) +@pytest.mark.parametrize('single_record', [True, False]) +@pytest.mark.parametrize('field_filter', [0, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50]) def test_format_json(field_filter, single_record): - """test func.""" - resultset = [["row{}".format(x) for x in range(5)]] - if field_filter == 1: - marks = {"uri": "row1"} - elif field_filter == 2: - marks = {"uri": "row1", "tags": "row3"[1:-1]} - elif field_filter == 3: - marks = {"title": "row2"} - else: - marks = { - "index": "row0", - "uri": "row1", - "title": "row2", - "description": "row4", - "tags": "row3"[1:-1], - } + resultset = [[f'' for x in range(5)]] + fields = FIELD_FILTER.get(field_filter, ALL_FIELDS) + marks = {} + if 'id' in fields: + marks['index'] = '' + if 'url' in fields: + marks['uri'] = '' + if 'title' in fields: + marks['title'] = '' + if 'tags' in fields: + marks['tags'] = 'row3' + if 'desc' in fields: + marks['description'] = '' if not single_record: marks = [marks] @@ -591,6 +532,7 @@ def test_sigint_handler(capsys): ), ], ], + ids=strip_query, ) def test_network_handler_with_url(url, exp_res): """test func.""" diff --git a/tests/test_bukuDb.py b/tests/test_bukuDb.py index c79f125d..15c79349 100644 --- a/tests/test_bukuDb.py +++ b/tests/test_bukuDb.py @@ -1084,34 +1084,34 @@ def test_add_rec_add_invalid_url(caplog, url): @pytest.mark.parametrize( "kwargs, exp_arg", [ - [{"url": "example.com"}, ("example.com", "Example Domain", ",", "", 0)], + [{"url": "example.com"}, ("example.com", "Example Domain", ",", "", False)], [ {"url": "http://example.com"}, - ("http://example.com", "Example Domain", ",", "", 0), + ("http://example.com", "Example Domain", ",", "", False), ], [ - {"url": "http://example.com", "immutable": 1}, - ("http://example.com", "Example Domain", ",", "", 1), + {"url": "http://example.com", "immutable": True}, + ("http://example.com", "Example Domain", ",", "", True), ], [ {"url": "http://example.com", "desc": "randomdesc"}, - ("http://example.com", "Example Domain", ",", "randomdesc", 0), + ("http://example.com", "Example Domain", ",", "randomdesc", False), ], [ {"url": "http://example.com", "title_in": "randomtitle"}, - ("http://example.com", "randomtitle", ",", "", 0), + ("http://example.com", "randomtitle", ",", "", False), ], [ {"url": "http://example.com", "tags_in": "tag1"}, - ("http://example.com", "Example Domain", ",tag1,", "", 0), + ("http://example.com", "Example Domain", ",tag1,", "", False), ], [ {"url": "http://example.com", "tags_in": ",tag1"}, - ("http://example.com", "Example Domain", ",tag1,", "", 0), + ("http://example.com", "Example Domain", ",tag1,", "", False), ], [ {"url": "http://example.com", "tags_in": ",tag1,"}, - ("http://example.com", "Example Domain", ",tag1,", "", 0), + ("http://example.com", "Example Domain", ",tag1,", "", False), ], ], )