improvements

.gitignore

@@ -8,3 +8,4 @@
 /tmp/
 key.json
 data/
+logfile.log

.rubocop.yml

@@ -1,3 +1,5 @@
+inherit_from: .rubocop_todo.yml
+
 Style/StringLiterals:
   Enabled: true
   EnforcedStyle: double_quotes

.rubocop_todo.yml

@@ -0,0 +1,36 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2024-05-25 12:30:22 UTC using RuboCop version 1.64.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 4
+# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 58
+
+# Offense count: 2
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+# AllowedMethods: refine
+Metrics/BlockLength:
+  Max: 28
+
+# Offense count: 1
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 188
+
+# Offense count: 6
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+Metrics/MethodLength:
+  Max: 61
+
+# Offense count: 1
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/mvg/data_handler.rb'

Gemfile

@@ -7,4 +7,5 @@ gemspec
 
 gem "rake", "~> 13.0"
 
-gem "rubocop", "~> 0.80"
+gem "httpclient", git: "https://github.com/nahi/httpclient"
+gem "mutex_m"

Gemfile.lock

@@ -1,3 +1,9 @@
+GIT
+  remote: https://github.com/nahi/httpclient
+  revision: d57cc6d5ffee1b566b5c189fe6dc8cc89570b812
+  specs:
+    httpclient (2.8.3)
+
 PATH
   remote: .
   specs:
@@ -7,7 +13,9 @@ PATH
       oj
       ruby-zstds (~> 1.3)
       thor
+      tty-logger
       tty-progressbar
+      tty-prompt
       typhoeus (~> 1.4)
 
 GEM
@@ -58,12 +66,14 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    httpclient (2.8.3)
+    json (2.7.2)
     jwt (2.8.1)
       base64
+    language_server-protocol (3.17.0.3)
     mini_mime (1.1.5)
     minitar (0.9)
     multi_json (1.15.0)
+    mutex_m (0.2.0)
     net-http (0.4.1)
       uri
     oj (3.16.3)
@@ -73,8 +83,10 @@ GEM
     parser (3.3.1.0)
       ast (~> 2.4.1)
       racc
+    pastel (0.8.0)
+      tty-color (~> 0.5)
     public_suffix (5.0.5)
-    racc (1.7.3)
+    racc (1.8.0)
     rainbow (3.1.1)
     rake (13.2.1)
     regexp_parser (2.9.2)
@@ -85,15 +97,17 @@ GEM
     retriable (3.1.2)
     rexml (3.2.8)
       strscan (>= 3.0.9)
-    rubocop (0.93.1)
+    rubocop (1.64.0)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 2.7.1.5)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8)
-      rexml
-      rubocop-ast (>= 0.6.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
+      unicode-display_width (>= 2.4.0, < 3.0)
     rubocop-ast (1.31.3)
       parser (>= 3.3.1.0)
     ruby-progressbar (1.13.0)
@@ -108,27 +122,40 @@ GEM
     strscan (3.1.0)
     thor (1.3.1)
     trailblazer-option (0.1.2)
+    tty-color (0.6.0)
     tty-cursor (0.7.1)
+    tty-logger (0.6.0)
+      pastel (~> 0.8)
     tty-progressbar (0.18.2)
       strings-ansi (~> 0.2)
       tty-cursor (~> 0.7)
       tty-screen (~> 0.8)
       unicode-display_width (>= 1.6, < 3.0)
+    tty-prompt (0.23.1)
+      pastel (~> 0.8)
+      tty-reader (~> 0.8)
+    tty-reader (0.9.0)
+      tty-cursor (~> 0.7)
+      tty-screen (~> 0.8)
+      wisper (~> 2.0)
     tty-screen (0.8.2)
     typhoeus (1.4.1)
       ethon (>= 0.9.0)
     uber (0.1.0)
-    unicode-display_width (1.8.0)
+    unicode-display_width (2.5.0)
     uri (0.13.0)
+    wisper (2.0.1)
 
 PLATFORMS
   ruby
   x86_64-linux
 
 DEPENDENCIES
+  httpclient!
+  mutex_m
   mvg-analyser!
   rake (~> 13.0)
-  rubocop (~> 0.80)
+  rubocop (~> 1.64)
 
 BUNDLED WITH
    2.5.9

exe/mvg-analyser

@@ -1,8 +1,8 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'thor'
-require_relative '../lib/mvg/analyser'
-require_relative '../lib/mvg/cli'
-require_relative '../lib/mvg/data_handler'
-MVG::Cli.start(ARGV)
+require "thor"
+require_relative "../lib/mvg/analyser"
+require_relative "../lib/mvg/cli"
+require_relative "../lib/mvg/data_handler"
+MVG::Cli.start(ARGV)

export.log

@@ -14,4 +14,86 @@
 20240229.tar.zst
 20240301.tar.zst
 20240302.tar.zst
-20240303.tar.zst
+20240303.tar.zst
+20240304.tar.zst
+20240305.tar.zst
+20240306.tar.zst
+20240307.tar.zst
+20240308.tar.zst
+20240309.tar.zst
+20240310.tar.zst
+20240311.tar.zst
+20240312.tar.zst
+20240313.tar.zst
+20240314.tar.zst
+20240315.tar.zst
+20240316.tar.zst
+20240317.tar.zst
+20240318.tar.zst
+20240319.tar.zst
+20240320.tar.zst
+20240321.tar.zst
+20240322.tar.zst
+20240323.tar.zst
+20240324.tar.zst
+20240325.tar.zst
+20240326.tar.zst
+20240327.tar.zst
+20240328.tar.zst
+20240329.tar.zst
+20240330.tar.zst
+20240331.tar.zst
+20240401.tar.zst
+20240402.tar.zst
+20240403.tar.zst
+20240404.tar.zst
+20240405.tar.zst
+20240406.tar.zst
+20240407.tar.zst
+20240408.tar.zst
+20240409.tar.zst
+20240410.tar.zst
+20240411.tar.zst
+20240412.tar.zst
+20240413.tar.zst
+20240414.tar.zst
+20240415.tar.zst
+20240416.tar.zst
+20240417.tar.zst
+20240418.tar.zst
+20240419.tar.zst
+20240420.tar.zst
+20240421.tar.zst
+20240422.tar.zst
+20240423.tar.zst
+20240424.tar.zst
+20240425.tar.zst
+20240426.tar.zst
+20240427.tar.zst
+20240428.tar.zst
+20240429.tar.zst
+20240430.tar.zst
+20240501.tar.zst
+20240502.tar.zst
+20240503.tar.zst
+20240504.tar.zst
+20240505.tar.zst
+20240506.tar.zst
+20240507.tar.zst
+20240508.tar.zst
+20240509.tar.zst
+20240510.tar.zst
+20240511.tar.zst
+20240512.tar.zst
+20240513.tar.zst
+20240514.tar.zst
+20240515.tar.zst
+20240516.tar.zst
+20240517.tar.zst
+20240518.tar.zst
+20240519.tar.zst
+20240520.tar.zst
+20240521.tar.zst
+20240522.tar.zst
+20240523.tar.zst
+20240524.tar.zst

lib/mvg/cli.rb

@@ -1,25 +1,24 @@
 # frozen_string_literal: true
 
-
 module MVG
   ###
   # Provides command line interface
   class Cli < Thor
-    desc 'version', 'display the stan version'
+    desc "version", "display the stan version"
     def version
       puts("MVG Scraper version #{MVG::Analyser::VERSION}")
     end
 
-    desc 'export FILE', 'exports the given compressed file into sqlite'
+    desc "export FILE", "exports the given compressed file into sqlite"
     def export(file)
       analyser = MVG::DataHandler.new
       analyser.export_bq(file)
     end
-    
-    desc 'full', 'download and export full upstream data'
+
+    desc "full", "download and export full upstream data"
     def full
       analyser = MVG::DataHandler.new
       analyser.full
     end
   end
-end
+end