Merge branch 'develop' of github.com:bugre/mtools into fix/bugre-#764…

…_JSONSerialize_crash_w_bytes

* 'develop' of github.com:bugre/mtools:
  Add VSCode config to .gitignore
  Bump dev version
  Update for 1.6.1 release
  Fix Flake8 complaints
  Fix rueckstiess#765: mloginfo --clients: more robust parsing of client metadata (rueckstiess#766)
  Fix rueckstiess#761: mtools should use python3 in shebangs
  Fix rueckstiess#698: Add rounding option for mloginfo --queries (rueckstiess#758)
  More specific match for checkpoint duration log line
  Fix rueckstiess#258: Add timezone to mloginfo summary

.gitignore

@@ -29,5 +29,9 @@ pip-log.txt
 # Doc build files
 _build
 
+<<<<<<< HEAD
 # Visual Studio Code (vscode) config
+=======
+# VSCode config
+>>>>>>> 333c0194e98e70e4f6a008f851f40f85089aed18
 .vscode

CHANGES.rst

@@ -2,6 +2,18 @@
 Changes to mtools
 =================
 
+version 1.6.1
+~~~~~~~~~~~~~
+ * mtools should use python3 in shebangs (#761)
+ * mloginfo: add timezone to mloginfo summary (#258)
+ * mloginfo --clients: more robust parsing of client metadata (#765)
+ * mloginfo --queries: fix missing patterns for remove command (#742)
+ * mloginfo --queries: add rounding option (#698)
+ * mplotqueries: fix error parsing checkpoint log line (#757)
+
+Thanks to @stennie, @kallimachos, @kevinadi, @niccottrell, and @p-mongo
+for contributions to this release.
+
 version 1.6.0
 ~~~~~~~~~~~~~
 Now supporting Python 3.6+ (only)

doc/mloginfo.rst

@@ -13,13 +13,21 @@ Usage
 
 .. code-block:: bash
 
-   mloginfo [-h] [--version] logfile
-            [--verbose]
-            [--queries] [--restarts] [--distinct] [--connections] [--rsstate]
+   mloginfo [-h] logfile
             [--clients]
+            [--connections]
             [--cursors]
+            [--distinct]
+            [--queries]
+               [--rounding {0,1,2,3,4}]
+               [--sort {namespace,pattern,count,min,max,mean,95%,sum}]
+            [--restarts]
+            [--rsstate]
             [--storagestats]
-            [--transactions] [--tsort {duration}]
+            [--transactions]
+               [--tsort {duration}]
+            [--verbose]
+            [--version]
 
 General Parameters
 ~~~~~~~~~~~~~~~~~~
@@ -130,6 +138,24 @@ In addition to the default information, this command will also output the
    serverside.auth_sessions   update        {"session_endtime": 1, "session_userid": 1}        1         244         244          244          0.2          244             False
    serverside.game_level      find          {"_id": 1}                                         1         104         104          104          0.1          104             None
 
+
+``--rounding``
+^^^^^^^^^^^^^^
+
+This option adjusts the rounding for calculated statistics like mean and
+95%-ile.
+
+For example:
+
+.. code-block:: bash
+
+   mloginfo mongod.log --queries --rounding 2
+
+This option has no effect unless ``--queries`` is also specified.
+
+Valid rounding values are from 0 to 4 decimal places. The default value is 1.
+
+
 ``--sort``
 ^^^^^^^^^^
 
@@ -143,6 +169,11 @@ example:
 
 This option has no effect unless ``--queries`` is also specified.
 
+Valid sort options are: ``namespace``, ``pattern``, ``count``, ``min``,
+``max``, ``mean``, ``95%``, and ``sum``.
+
+The default sort option is ``sum``.
+
 Restarts (``--restarts``)
 -------------------------
 

mtools/__init__.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Mtools init."""
 
 __author__ = 'thomas@rueckstiess.net'

mtools/mgenerate/mgenerate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 mgenerate is no longer included with mtools. Please use
 `mgeneratejs <https://www.npmjs.com/package/mgeneratejs>`__ instead.

mtools/mlaunch/mlaunch.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 

mtools/mlogfilter/mlogfilter.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import inspect
 import re

mtools/mloginfo/mloginfo.py

@@ -1,5 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
+import datetime
 import inspect
 import sys
 
@@ -64,8 +65,15 @@ def run(self, arguments=None):
             print("      start: %s" % (start_time))
             print("        end: %s" % (end_time))
 
-            # TODO: add timezone if iso8601 format
             print("date format: %s" % self.logfile.datetime_format)
+
+            # self.logfile.timezone is a dateutil.tzinfo object
+            tzdt = datetime.datetime.now(self.logfile.timezone)
+            if (tzdt.tzname()):
+                timezone = tzdt.tzname()
+            else:
+                timezone = f"UTC {tzdt.strftime('%z')}"
+            print("   timezone: %s" % timezone)
             print("     length: %s" % len(self.logfile))
             print("     binary: %s" % (self.logfile.binary or "unknown"))
 

mtools/mloginfo/sections/clients_section.py

@@ -54,24 +54,25 @@ def run(self):
                 ip_formatted = str(ip)
 
                 if ip_formatted != "127.0.0.1":
-                    # driver metadata is not strict JSON, parsing string
-                    # then adding required double quotes to keys
-                    driver_data_raw = ("{" + line.split("{", 1)[1].split("}")[0] + "}}")
-                    driver_data = (re.sub(r"(\w+): ", r'"\1":', driver_data_raw))
-                    driver_data_json = json.loads(driver_data)
-
-                    driver = driver_data_json["driver"]["name"]
-                    version = driver_data_json["driver"]["version"]
-                    dv_formatted = str(driver) + ":" + str(version)
-                    if dv_formatted not in driver_info:
-                        driver_info[dv_formatted] = [ip_formatted]
-                    elif dv_formatted in driver_info:
-                        if ip_formatted in driver_info.get(dv_formatted):
-                            continue
-                        else:
-                            driver_info[dv_formatted].append(ip_formatted)
+                    client_metadata = logevent.client_metadata
+                    try:
+                        driver = client_metadata["driver"]["name"]
+                        version = client_metadata["driver"]["version"]
+                        dv_formatted = str(driver) + ":" + str(version)
+                        if dv_formatted not in driver_info:
+                            driver_info[dv_formatted] = [ip_formatted]
+                        elif dv_formatted in driver_info:
+                            if ip_formatted in driver_info.get(dv_formatted):
+                                continue
+                            else:
+                                driver_info[dv_formatted].append(ip_formatted)
+                    except Exception:
+                        next
 
         print('%-15s - Unique connections' % 'Driver:Version ')
-        for key, value in sorted(driver_info.items()):
-            print("%-15s : " % (key) + str(value))
+        if len(driver_info):
+            for key, value in sorted(driver_info.items()):
+                print("%-15s : " % (key) + str(value))
+        else:
+            print("No client metadata found")
         print('')

mtools/mloginfo/sections/query_section.py

@@ -43,6 +43,13 @@ def __init__(self, mloginfo):
                                                                    'mean',
                                                                    '95%',
                                                                    'sum'])
+        helptext = 'Number of decimal places for rounding of calculated stats'
+        self.mloginfo.argparser_sectiongroup.add_argument('--rounding',
+                                                          action='store',
+                                                          type=int,
+                                                          default=1,
+                                                          choices=range(0, 5),
+                                                          help=helptext)
 
     @property
     def active(self):
@@ -54,6 +61,7 @@ def run(self):
         grouping = Grouping(group_by=lambda x: (x.namespace, x.operation,
                                                 x.pattern, x.allowDiskUse))
         logfile = self.mloginfo.logfile
+        rounding = self.mloginfo.args['rounding']
 
         if logfile.start and logfile.end:
             progress_start = self.mloginfo._datetime_to_epoch(logfile.start)
@@ -113,14 +121,13 @@ def run(self):
             stats['count'] = len(group_events_all)
             stats['min'] = min(group_events) if group_events else 0
             stats['max'] = max(group_events) if group_events else 0
-            stats['mean'] = 0
             if np:
-                stats['95%'] = (np.percentile(group_events, 95)
+                stats['95%'] = (round(np.percentile(group_events, 95), rounding)
                                 if group_events else 0)
             else:
                 stats['95%'] = 'n/a'
             stats['sum'] = sum(group_events) if group_events else 0
-            stats['mean'] = (stats['sum'] / stats['count']
+            stats['mean'] = (round(stats['sum'] / stats['count'], rounding)
                              if group_events else 0)
 
             if self.mloginfo.args['verbose']:

mtools/mlogvis/mlogvis.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys

mtools/mplotqueries/mplotqueries.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from six.moves import cPickle
 import glob

mtools/util/__init__.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Make OrderedDict available: collections (py2.7) or ordereddict (py2.6)."""
 
 try:

mtools/util/cmdlinetool.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Command line tool utility."""
 
 import argparse

mtools/util/grouping.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Utility for grouping items and working with grouped items."""
 
 import re

mtools/util/hci.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Utility for working with datetimes."""
 
 import re

mtools/util/input_source.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Input source utility."""
 
 

mtools/util/log2code.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 

mtools/util/logcodeline.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 from collections import defaultdict
 

mtools/util/logevent.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 import json
 import re
@@ -119,7 +119,6 @@ def _reset(self):
         self._counters_calculated = False
         self._allowDiskUse = None
 
-
         self._bytesRead = None
         self._bytesWritten = None
         self._timeReadingMicros = None
@@ -151,6 +150,9 @@ def _reset(self):
         self._component = None
         self.merge_marker_str = ''
 
+        self._client_metadata_calculated = False
+        self._client_metadata = None
+
     def set_line_str(self, line_str):
         """
         Set line_str.
@@ -215,7 +217,7 @@ def duration(self):
                 if matchobj:
                     self._duration = int(matchobj.group(1))
             # SERVER-16176 - Logging of slow checkpoints
-            elif "Checkpoint" in self.line_str:
+            elif "Checkpoint took" in self.line_str:
                 matchobj = re.search("Checkpoint took ([\d]+) seconds to complete", self.line_str)
                 if matchobj:
                     self._duration = int(matchobj.group(1)) * 1000
@@ -938,6 +940,29 @@ def _extract_level(self):
                 self._level = False
                 self._component = False
 
+    @property
+    def client_metadata(self):
+        """Return client metadata."""
+        if not self._client_metadata_calculated:
+            self._client_metadata_calculated = True
+
+            line_str = self.line_str
+            if (line_str and line_str.find('client metadata')):
+                try:
+                    metadata_pos = line_str.find("{")
+                    if metadata_pos == -1:
+                        return
+                    else:
+                        metadata = line_str[metadata_pos:]
+                        # Make valid JSON by wrapping field names in quotes
+                        metadata, _ = re.subn(r'([{,])\s*([^,{\s\'"]+)\s*:',
+                                              ' \\1 "\\2" : ', metadata)
+                        self._client_metadata = json.loads(metadata)
+                except ValueError:
+                    self._client_metadata = None
+
+        return self._client_metadata
+
     def parse_all(self):
         """
         Trigger extraction of all information.

mtools/util/logfile.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 

mtools/util/parse_sourcecode.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from six.moves import cPickle
 import os

mtools/util/pattern.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 import json
 import re

mtools/util/presplit.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 

mtools/util/print_table.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 def print_table(rows, override_headers=None, uppercase_headers=True):
     """All rows need to be a list of dictionaries, all with the same keys."""

mtools/util/profile_collection.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 
 from dateutil.tz import tzutc
 from pymongo import ASCENDING, DESCENDING

mtools/version.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Mtools version."""
 
-__version__ = '1.6.1-dev'
+__version__ = '1.6.2-dev'

setup.py

@@ -1,4 +1,4 @@
-#!/bin/python
+#!/usr/bin/env python3
 """Setup file for mtools."""
 
 import platform