Merge branch 'dev'

CHANGELOG.md

@@ -2,6 +2,14 @@ CHANGELOG
 =========
 
 
+5.1, 2022-05-01
+---------------
+- added contacts HTML export;
+- refresh contacts list during and after live sync;
+- fixed live sync updating messages needlessly in Python3;
+- fixed error on filtering current chat by text (issue #107).
+
+
 5.0, 2022-04-02
 ---------------
 - added contacts-tab;

setup.py

@@ -8,69 +8,77 @@
 
 @author      Erki Suurjaak
 @created     10.12.2014
-@modified    26.03.2022
+@modified    29.04.2022
 ------------------------------------------------------------------------------
 """
 import os
+import re
 import sys
+
 import setuptools
 
 ROOTPATH  = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(ROOTPATH, "src"))
 
 from skyperious import conf
 
+
+PACKAGE = conf.Title.lower()
+REPOSITORY = "https://github.com/suurjaak/Skyperious"
+
+
+def readfile(path):
+    """Returns contents of path, relative to current file."""
+    root = os.path.dirname(os.path.abspath(__file__))
+    with open(os.path.join(root, path)) as f: return f.read()
+
+def get_description():
+    """Returns package description from README."""
+    LINK_RGX = r"\[([^\]]+)\]\(([^\)]+)\)"  # 1: content in [], 2: content in ()
+    KEEP = ("ftp://", "http://", "https://", "www.")
+    # Unwrap page anchor links like [Page link](#page-link) as "Page link",
+    # make package file links like [LICENSE.md](LICENSE.md) point to repository
+    repl = lambda m: m.group(1) if m.group(2).startswith("#") else \
+                     m.group(0) if any(map(m.group(2).startswith, KEEP)) else \
+                     "[%s](%s/blob/master/%s)" % (m.group(1), REPOSITORY, m.group(2))
+    return re.sub(LINK_RGX, repl, readfile("README.md"))
+
+
 setuptools.setup(
-    name=conf.Title,
-    version=conf.Version,
-    description="Skype chat history tool",
-    url="https://github.com/suurjaak/Skyperious",
-
-    author="Erki Suurjaak",
-    author_email="erki@lap.ee",
-    license="MIT",
-    platforms=["any"],
-    keywords="skype sqlite merge export",
-
-    install_requires=["appdirs", "beautifulsoup4", "ijson", "pyparsing", "Pillow",
-                      "six", "SkPy", "wxPython>=4.0", "XlsxWriter"],
-    entry_points={"gui_scripts": ["skyperious = skyperious.main:run"]},
-
-    package_dir={"": "src"},
-    packages=[conf.Title.lower()],
-    include_package_data=True, # Use MANIFEST.in for data files
-    classifiers=[
+    name                 = PACKAGE,
+    version              = conf.Version,
+    description          = "Skype chat history tool",
+    url                  = REPOSITORY,
+
+    author               = "Erki Suurjaak",
+    author_email         = "erki@lap.ee",
+    license              = "MIT",
+    platforms            = ["any"],
+    keywords             = "skype sqlite merge export",
+
+    install_requires     = ["appdirs", "beautifulsoup4", "ijson", "pyparsing", "Pillow",
+                            "six", "SkPy", "wxPython>=4.0", "XlsxWriter"],
+    entry_points         = {"gui_scripts": ["{0} = {0}.main:run".format(PACKAGE)]},
+
+    package_dir          = {"": "src"},
+    packages             = [PACKAGE],
+    include_package_data = True, # Use MANIFEST.in for data files
+    classifiers          = [
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: End Users/Desktop",
         "Operating System :: Microsoft :: Windows",
         "Operating System :: Unix",
         "Operating System :: MacOS",
         "Topic :: Communications :: Chat",
         "Topic :: Database",
-        "Topic :: Utilities",
         "Topic :: Desktop Environment",
+        "Topic :: Utilities",
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
     ],
 
-    long_description_content_type="text/markdown",
-    long_description=
-"""Skyperious is a Skype chat history tool, written in Python.
-
-You can open Skype SQLite databases and work with their contents:
-
-- import messages from Skype online service and Skype export archives
-- search across all messages and contacts
-- read chat history in full, see chat statistics and word clouds
-- export chats as HTML, text or spreadsheet
-- view any database table and export their data, fix database corruption
-- change, add or delete data in any table
-- execute direct SQL queries
-
-and
-
-- synchronize messages in two Skype databases, merging their differences
-""",
+    long_description_content_type = "text/markdown",
+    long_description = get_description(),
 )

skyperious.bat

@@ -1 +1 @@
-@cd src && start pythonw -m skyperious %*
+@cd /D %~dp0\src && start pythonw -m skyperious %*

skyperious.sh

@@ -1,2 +1,3 @@
 #!/bin/bash
-cd src && exec >/dev/null 2>&1 python3 -m skyperious "$@"
+cd $(dirname "${BASH_SOURCE[0]}")/src
+exec >/dev/null 2>&1 python3 -m skyperious "$@"

src/skyperious/conf.py

@@ -10,7 +10,7 @@
 
 @author      Erki Suurjaak
 @created     26.11.2011
-@modified    02.04.2022
+@modified    01.05.2022
 ------------------------------------------------------------------------------
 """
 try: from ConfigParser import RawConfigParser                 # Py2
@@ -24,8 +24,8 @@
 
 """Program title, version number and version date."""
 Title = "Skyperious"
-Version = "5.0"
-VersionDate = "02.04.2022"
+Version = "5.1.dev10"
+VersionDate = "01.05.2022"
 
 if getattr(sys, "frozen", False):
     # Running as a pyinstaller executable

src/skyperious/export.py

@@ -8,7 +8,7 @@
 
 @author      Erki Suurjaak
 @created     13.01.2012
-@modified    02.04.2022
+@modified    29.04.2022
 ------------------------------------------------------------------------------
 """
 import codecs
@@ -74,8 +74,8 @@
                   "%sCSV spreadsheet (*.csv)|*.csv" % XLSX_WILDCARD)
 QUERY_EXTS = ["html", "xlsx", "csv"] if xlsxwriter else ["html", "csv"]
 
-CONTACT_WILDCARD = "%sCSV spreadsheet (*.csv)|*.csv" % XLSX_WILDCARD
-CONTACT_EXTS = ["xlsx", "csv"] if xlsxwriter else ["csv"]
+CONTACT_WILDCARD = "HTML document (*.html)|*.html|%sCSV spreadsheet (*.csv)|*.csv" % XLSX_WILDCARD
+CONTACT_EXTS = ["html", "xlsx", "csv"] if xlsxwriter else ["html","csv"]
 
 IMAGE_EXTS = ["bmp", "jpg", "png"]
 IMAGE_WILDCARD = "|".join("%s image (*.%s)|*.%s" % (x.upper(), x, x) for x in IMAGE_EXTS)
@@ -363,16 +363,25 @@ def export_chat_csv(chat, filename, db, messages, opts=None):
 
 def export_contacts(contacts, filename, format, db):
     """
-    Exports the contacts to a spreadsheet file.
+    Exports the contacts to an HTML or spreadsheet file.
 
-    @param   contacts      list of chat dicts, as returned from SkypeDatabase
+    @param   contacts      list of contacts dicts, as returned from SkypeDatabase,
+                           supplemented with statistics
     @param   filename      full path and filename of resulting file
-    @param   format        export format (xlsx|csv)
+    @param   format        export format (html|xlsx|csv)
     @param   db            SkypeDatabase instance
     """
+    is_html = ("html" == format)
     is_csv  = ("csv"  == format)
     is_xlsx = ("xlsx" == format)
 
+    if is_html:
+        namespace = {"contacts": contacts, "db": db}
+        with open(filename, "wb") as f:
+            template = step.Template(templates.EXPORT_CONTACTS_HTML, escape=True)
+            template.stream(f, namespace)
+        return
+
     writer = None
     colnames, collabels = zip(*((k, "%s (%s)" % (v, k))
                                 for k, v in skypedata.CONTACT_FIELD_TITLES.items()))
@@ -385,7 +394,7 @@ def export_contacts(contacts, filename, format, db):
         writer.writerow(*([collabels, "bold"] if is_xlsx else [collabels]))
         writer.set_header(False) if is_xlsx else 0
         for c in contacts:
-            writer.writerow(["" if c.get(n) is None else c[n] for n in colnames])
+            writer.writerow([skypedata.format_contact_field(c, n) or "" for n in colnames])
         writer.close()
         writer = None
     finally:

src/skyperious/gui.py

@@ -8,7 +8,7 @@
 
 @author      Erki Suurjaak
 @created     26.11.2011
-@modified    02.04.2022
+@modified    01.05.2022
 ------------------------------------------------------------------------------
 """
 import ast
@@ -3603,12 +3603,7 @@ def after(result):
                 else:
                     plabel = u"Synchronizing %s" % result["table"]
                     if "messages" == result["table"] and result.get("chat"):
-                        chat = next((c for c in self.chats if c["identity"] == result["chat"]), None)
-                        if not chat:
-                            cc = self.db.get_conversations(chatidentities=[result["chat"]], reload=True, log=False)
-                            self.chats.extend(cc)
-                            if cc: chat = cc[0]
-
+                        chat  = next((c for c in self.chats if c["identity"] == result["chat"]), None)
                         title = chat["title_long_lc"] if chat else result["chat"]
                         if len(title) > 35:
                             title = title[:35] + ".."
@@ -3641,9 +3636,20 @@ def after(result):
                                 ]))
 
                             self.chats = self.db.get_conversations(reload=True, log=False)
+                            self.contacts = self.db.get_contacts(reload=True)
+
                             def after2():
-                                if self: self.db.get_conversations_stats(self.chats)
-                                if self: self.list_chats.Populate(self.chats)
+                                """Populate statistics for all chats after completion."""
+                                if not self: return
+                                self.db.get_conversations_stats(self.chats)
+                                self.list_chats.Populate(self.chats)
+                                self.db.get_contacts_stats(self.contacts, self.chats, log=False)
+                                self.list_contacts.Populate(self.contacts)
+                                if self.contact:
+                                    self.contact = next((x for x in self.contacts
+                                                         if x["identity"] == self.contact["identity"]), None)
+                                self.load_contact(self.contact)
+
                             wx.CallAfter(after2)
 
                         if "messages" == result["table"]:
@@ -3657,9 +3663,9 @@ def after2():
                                 self.list_chats_sync.AppendRow(row)
                                 self.list_chats_sync.ResetColumnWidths()
                                 chat["message_count" ] = (chat["message_count"] or 0) + result["new"]
-                                chat["first_message_datetime"]  = min(chat["first_message_datetime"] or result["first"], result["first"])
-                                chat["last_message_datetime"]   = max(chat["last_message_datetime"]  or result["first"], result["last"])
-                                chat["last_activity_datetime"]  = max(chat["last_activity_datetime"] or result["last"],  result["last"])
+                                chat["first_message_datetime"] = min(chat["first_message_datetime"] or result["first"], result["first"])
+                                chat["last_message_datetime"]  = max(chat["last_message_datetime"]  or result["first"], result["last"])
+                                chat["last_activity_datetime"] = max(chat["last_activity_datetime"] or result["last"],  result["last"])
                                 for k in "first_message", "last_message", "last_activity":
                                     chat[k + "_timestamp"] = util.datetime_to_epoch(chat[k + "_datetime"])
                                 self.list_chats.Populate(self.chats)
@@ -3671,6 +3677,25 @@ def after2():
                                 if result["updated"]: slabel += ", %s updated" % result["updated"]
                                 slabel += "."
 
+                    elif not result.get("start"):
+                        if "chats" == result["table"] and result.get("chat"):
+                            # Inserted or updated a chat: load new contacts, if any
+                            plabel = ""
+
+                            chat = next((c for c in self.chats if c["identity"] == result["chat"]), None)
+                            if not chat:
+                                cc = self.db.get_conversations(chatidentities=[result["chat"]], reload=True, log=False)
+                                self.chats.extend(cc)
+                                if cc: chat = cc[0]
+                            if chat:
+                                ids = [x["identity"] for x in chat["participants"]]
+                                newids = set(ids) - set(x["identity"] for x in self.contacts)
+                                newcontacts = list(filter(bool, map(self.db.get_contact, newids)))
+                                if newcontacts:
+                                    self.contacts.extend(newcontacts)
+                                    self.list_contacts.Populate(self.contacts)
+                                    self.load_contact(self.contact)
+
                 if plabel:
                     self.label_sync_progress.Label = plabel
                 if slabel:
@@ -4317,11 +4342,15 @@ def on_export_contacts(self, event):
         dialog.Wildcard = export.CONTACT_WILDCARD
         if wx.ID_OK != dialog.ShowModal(): return
 
+        busy = controls.BusyPanel(self, "Exporting contacts.")
         filepath = controls.get_dialog_path(dialog)
         format = export.CONTACT_EXTS[dialog.FilterIndex]
         guibase.status("Exporting %s.", filepath, log=True)
-        export.export_contacts(self.contacts, filepath, format, self.db)
-        util.start_file(filepath)
+        try:
+            export.export_contacts(self.contacts, filepath, format, self.db)
+            util.start_file(filepath)
+        finally:
+            busy.Close()
 
 
     def on_export_chats(self, chats, do_all=False, do_singlefile=False, do_timerange=False, event=None):
@@ -5770,31 +5799,31 @@ def load_contact(self, contact):
                 if clist.GetItemMappedData(i) == self.contact:
                     clist.SetItemFont(i, clist.Font)
             self.contact = None
-        if not contact:
             return
 
         # Update contact list colours and scroll to the opened contact
         if contact != self.contact:
             logger.info("Opening contact %s.", contact["name"])
-            clist.Freeze()
-            scrollpos = clist.GetScrollPos(wx.VERTICAL)
-            index_selected = -1
-            for i in range(clist.ItemCount):
-                if clist.GetItemMappedData(i) == self.contact:
-                    clist.SetItemFont(i, clist.Font)
-                elif clist.GetItemMappedData(i) == contact:
-                    index_selected = i
-                    f = clist.Font; f.SetWeight(wx.FONTWEIGHT_BOLD)
-                    clist.SetItemFont(i, f)
-            if index_selected >= 0:
-                delta = index_selected - scrollpos
-                if delta < 0 or abs(delta) >= clist.CountPerPage:
-                    nudge = -clist.CountPerPage // 2
-                    clist.ScrollLines(delta + nudge)
-            clist.Thaw()
-            wx.YieldIfNeeded() # Allow display to refresh
-            self.label_contact.Label = "&Contact %(name)s (%(identity)s):" % contact
-            self.label_contact.Parent.Layout()
+        clist.Freeze()
+        scrollpos = clist.GetScrollPos(wx.VERTICAL)
+        index_selected = -1
+        for i in range(clist.ItemCount):
+            myid = clist.GetItemMappedData(i)["id"]
+            if myid == contact["id"]:
+                index_selected = i
+                f = clist.Font; f.SetWeight(wx.FONTWEIGHT_BOLD)
+                clist.SetItemFont(i, f)
+            elif self.contact and myid == self.contact["id"]:
+                clist.SetItemFont(i, clist.Font)
+        if index_selected >= 0:
+            delta = index_selected - scrollpos
+            if delta < 0 or abs(delta) >= clist.CountPerPage:
+                nudge = -clist.CountPerPage // 2
+                clist.ScrollLines(delta + nudge)
+        clist.Thaw()
+        wx.YieldIfNeeded() # Allow display to refresh
+        self.label_contact.Label = "&Contact %(name)s (%(identity)s):" % contact
+        self.label_contact.Parent.Layout()
 
         titlemap = {x["id"]: x["title_long"] for x in self.chats}
         for data in contact.get("conversations", []):
@@ -8160,9 +8189,9 @@ def _append_text(self, text, style="default", rgx_highlight=None):
                                 in highlighted style
         """
         text = text or ""
-        if isinstance(text, six.text_type):
-            text = text.encode("utf-8")
         text_parts = rgx_highlight.split(text) if rgx_highlight else [text]
+        if isinstance(text, six.text_type):
+            text_parts = [x.encode("utf-8") for x in text_parts]
         bold = "bold%s" % style if "bold%s" % style in self._styles else style
         len_self = self.GetTextLength()
         self.STC.AppendText(text)