From f0d86a0d597e1e47b5214299425ce4a975a2eab8 Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Mon, 4 Mar 2024 21:37:30 -0800
Subject: [PATCH 01/10] Remove debug code.

---
 lib/ipstats.cpp | 48 ------------------------------------------------
 lib/ipstats.h   |  6 ------
 2 files changed, 54 deletions(-)

diff --git a/lib/ipstats.cpp b/lib/ipstats.cpp
index f7eb047..022daa7 100644
--- a/lib/ipstats.cpp
+++ b/lib/ipstats.cpp
@@ -129,11 +129,6 @@ IPStatsRecord::IPStatsRecord() :
     memset(name_parts, 0, sizeof(name_parts));
     memset(rr_types, 0, sizeof(rr_types));
     memset(locales, 0, sizeof(locales));
-    /* Debug variables */
-    tld_length = 0;
-    sld_length = 0;
-    memset(TLD, 0, 64);
-    memset(SLD, 0, 64);
 }
 
 IPStatsRecord::~IPStatsRecord()
@@ -475,10 +470,6 @@ const size_t nb_TLD_subset = sizeof(TLD_subset) / sizeof(const char*);
 
 void IPStatsRecord::SetTLD(size_t tld_length, uint8_t* tld)
 {
-#if 1
-    this->tld_length = tld_length;
-    memcpy(this->TLD, tld, tld_length);
-#endif
     IPStatsRecord::SetXLD(tld_length, tld, TLD_subset, nb_TLD_subset, this->tld_counts, &this->tld_hyperlog);
 }
 
@@ -491,44 +482,6 @@ const size_t nb_SLD_subset = sizeof(SLD_subset) / sizeof(const char*);
 void IPStatsRecord::SetSLD(size_t sld_length, uint8_t* sld)
 {
     IPStatsRecord::SetXLD(sld_length, sld, SLD_subset, nb_SLD_subset, this->sld_counts, &this->sld_hyperlog);
-#if 1
-    this->sld_length = sld_length;
-    memcpy(this->SLD, sld, sld_length);
-#endif
-}
-
-void IPStatsRecord::DebugPrint(FILE* F)
-{
-#if 1
-    uint8_t test_ip[4] = { 10, 9, 1, 109 };
-    if (memcmp(this->ip_addr, test_ip, 4) == 0) {
-        uint64_t tld_hash = HyperLogLog::Fnv64(TLD, tld_length);
-        uint64_t sld_hash = HyperLogLog::Fnv64(SLD, sld_length);
-        fprintf(F, "IP:%d.%d.%d.%d,", this->ip_addr[0], this->ip_addr[1], this->ip_addr[2], this->ip_addr[3]);
-        fprintf(F, "TLD[%zu, 0x% " PRIx64 "] = .", tld_length, tld_hash);
-        for (size_t i = 0; i < tld_length; i++) {
-            int c = TLD[i];
-            if (c >= 32 && c < 127) {
-                fprintf(F, "%c.", c);
-            }
-            else {
-                fprintf(F, "\\%x.", c);
-            }
-        }
-
-        fprintf(F, ", SLD[%zu, 0x% " PRIx64 "] = .", sld_length, sld_hash);
-        for (size_t i = 0; i < sld_length; i++) {
-            int c = SLD[i];
-            if (c >= 32 && c < 127) {
-                fprintf(F, "%c.", c);
-            }
-            else {
-                fprintf(F, "\\%x.", c);
-            }
-        }
-        fprintf(F, "\n");
-    }
-#endif
 }
 
 IPStats::IPStats()
@@ -692,7 +645,6 @@ void IPStats::SubmitCborPacket(cdns* cdns_ctx, size_t packet_id)
         bool stored = false;
         this->ip_records.InsertOrAdd(&ipsr, true, &stored);
     }
-    ipsr.DebugPrint(stdout);
 }
 
 bool IPStats::IsRegisteredTLD(uint8_t* x, size_t l)
diff --git a/lib/ipstats.h b/lib/ipstats.h
index 582953a..4034a6c 100644
--- a/lib/ipstats.h
+++ b/lib/ipstats.h
@@ -130,16 +130,11 @@ class IPStatsRecord
 
     bool WriteRecord(FILE* F);
     static IPStatsRecord* ParseLine(char const* line);
-    void DebugPrint(FILE* F);
 private:
     bool WriteIP(FILE* F);
     static void SetXLD(size_t xld_length, uint8_t * xld, const char ** XLD_subset, size_t nb_XLD_subset, uint64_t * xld_counts, HyperLogLog * xld_hyperlog);
     void SetTLD(size_t tld_length, uint8_t* tld);
     void SetSLD(size_t sld_length, uint8_t* sld);
-    size_t tld_length;
-    size_t sld_length;
-    uint8_t TLD[64];
-    uint8_t SLD[64];
 };
 
 class IPStats
@@ -166,7 +161,6 @@ class IPStats
 private:
     BinHash<IPStatsRecord> ip_records;
     void SubmitCborPacket(cdns* cdns_ctx, size_t packet_id);
-    void LoadRegisteredTLD_from_memory();
     static bool IPAddressIsLower(IPStatsRecord * x, IPStatsRecord * y);
     BinHash<TldAsKey> registeredTld;
     DnsStats dnsstats;

From ed03890e5f0d46327b065e3ae612cb52bee0b97a Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Tue, 5 Mar 2024 15:43:56 -0800
Subject: [PATCH 02/10] Prepare loading various files.

---
 lib/ipstats.cpp      |  5 ++++-
 lib/ipstats.h        |  2 +-
 lib/ithiutil.cpp     | 29 +++++++++++++++++++++++++++++
 lib/ithiutil.h       |  2 ++
 src/ithitools.cpp    |  8 ++++----
 test/IPStatsTest.cpp |  2 +-
 6 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/lib/ipstats.cpp b/lib/ipstats.cpp
index 022daa7..54dad4e 100644
--- a/lib/ipstats.cpp
+++ b/lib/ipstats.cpp
@@ -492,13 +492,16 @@ IPStats::~IPStats()
 {
 }
 
-bool IPStats::LoadCborFiles(size_t nb_files, char const** fileNames)
+bool IPStats::LoadInputFiles(size_t nb_files, char const** fileNames)
 {
     bool ret = true;
 
     for (size_t i = 0; ret && i < nb_files; i++)
     {
+        /* If ends with ".cbor", load as cbor file */
         ret = LoadCborFile(fileNames[i]);
+        /* If ends with ".cbor.xz", load as compressed cbor file */
+        /* If ends with ".csv", load as csv file */
     }
 
     return ret;
diff --git a/lib/ipstats.h b/lib/ipstats.h
index 4034a6c..3406e89 100644
--- a/lib/ipstats.h
+++ b/lib/ipstats.h
@@ -144,7 +144,7 @@ class IPStats
     ~IPStats();
 
     /* For the command line tools */
-    bool LoadCborFiles(size_t nb_files, char const** fileNames);
+    bool LoadInputFiles(size_t nb_files, char const** fileNames);
     bool LoadCborFile(char const* fileNames);
 
     bool SaveToCsv(char const* file_name);
diff --git a/lib/ithiutil.cpp b/lib/ithiutil.cpp
index 68ed418..5b47f3f 100644
--- a/lib/ithiutil.cpp
+++ b/lib/ithiutil.cpp
@@ -130,6 +130,35 @@ FILE* ithi_gzip_compress_open(char const* file_name, int* last_err)
     return F;
 }
 
+FILE* ithi_xzcat_decompress_open(char const* file_name, int* last_err)
+{
+    FILE * F = NULL;
+    char const* xzcat_command = NULL;
+    char command[512];
+    int n_char = 0;
+
+#ifdef _WINDOWS
+    /* Running on windows requires that 7z.exe is installed */
+    xzcat_command = "7z.exe e -so";
+#else
+    xzcat_command = "zxcat -k";
+#endif
+
+#ifdef _WINDOWS
+    n_char = sprintf_s(command, sizeof(command), "%s %s", xzcat_command, file_name);
+#else 
+    n_char = sprintf(command, "%s %s", xzcat_command, file_name);
+#endif
+    if (n_char <= 0) {
+        *last_err = -1;
+    }
+    else {
+        F = ithi_pipe_open(command, false, last_err);
+    }
+
+    return F;
+}
+
 void ithi_pipe_close(FILE* F)
 {
 #ifdef _WINDOWS
diff --git a/lib/ithiutil.h b/lib/ithiutil.h
index b003387..3d448d9 100644
--- a/lib/ithiutil.h
+++ b/lib/ithiutil.h
@@ -35,6 +35,8 @@ FILE* ithi_pipe_open(char const* command, bool pipe_write, int* last_err);
 
 FILE* ithi_gzip_compress_open(char const* file_name, int* last_err);
 
+FILE* ithi_xzcat_decompress_open(char const* file_name, int* last_err);
+
 void ithi_pipe_close(FILE* F);
 
 size_t ithi_copy_to_safe_text(char* text, size_t text_max, uint8_t* x_in, size_t l_in);
diff --git a/src/ithitools.cpp b/src/ithitools.cpp
index fd46a39..f8a2915 100644
--- a/src/ithitools.cpp
+++ b/src/ithitools.cpp
@@ -642,18 +642,18 @@ int main(int argc, char ** argv)
 
         if (optind >= argc)
         {
-            fprintf(stderr, "No capture file to analyze!\n");
+            fprintf(stderr, "No file to load!\n");
             exit_code = usage();
         }
         else {
-            if (!ipstats.LoadCborFiles((size_t)argc - optind, (char const**)(argv + optind)))
+            if (!ipstats.LoadInputFiles((size_t)argc - optind, (char const**)(argv + optind)))
             {
-                fprintf(stderr, "Cannot process the CBOR input files.\n");
+                fprintf(stderr, "Cannot process the input files.\n");
                 exit_code = -1;
             }
             else
             {
-                printf("CBOR Capture processing succeeded, %d records.\n", ipstats.GetCount());
+                printf("File processing succeeded, %d records.\n", ipstats.GetCount());
                 if (!ipstats.SaveToCsv(ip_stats_csv)) {
                     fprintf(stderr, "Cannot save to csv file: %s.\n", ip_stats_csv);
                     exit_code = usage();
diff --git a/test/IPStatsTest.cpp b/test/IPStatsTest.cpp
index 8304f84..c5e9061 100644
--- a/test/IPStatsTest.cpp
+++ b/test/IPStatsTest.cpp
@@ -50,7 +50,7 @@ bool IPStatsTest::DoTest()
 {
     IPStats ipstats;
     char const * list[1] = { ipstats_test_input };
-    bool ret = ipstats.LoadCborFiles(1, list);
+    bool ret = ipstats.LoadInputFiles(1, list);
 
     if (!ret){
         TEST_LOG("Cannot process the CBOR input file: %s\n", list[0]);

From 9cb2bc7e685a0b8834c6a0c8ba086bef08c05c0d Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Tue, 5 Mar 2024 22:50:18 -0800
Subject: [PATCH 03/10] Intermediate commit before debugging.

---
 lib/ipstats.cpp  | 57 +++++++++++++++++++++++++++++++++++++++++-------
 lib/ipstats.h    |  4 +++-
 lib/ithiutil.cpp | 11 ++++++++++
 lib/ithiutil.h   |  2 ++
 4 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/lib/ipstats.cpp b/lib/ipstats.cpp
index 54dad4e..ca6d90f 100644
--- a/lib/ipstats.cpp
+++ b/lib/ipstats.cpp
@@ -11,6 +11,7 @@
 #include <algorithm>
 #include <math.h>
 #include <time.h>
+#include "ithiutil.h"
 
 #ifdef _WINDOWS
 #define WIN32_LEAN_AND_MEAN
@@ -499,8 +500,13 @@ bool IPStats::LoadInputFiles(size_t nb_files, char const** fileNames)
     for (size_t i = 0; ret && i < nb_files; i++)
     {
         /* If ends with ".cbor", load as cbor file */
-        ret = LoadCborFile(fileNames[i]);
+        if (ithi_endswith(fileNames[i], ".cbor")) {
+            ret = LoadCborFile(fileNames[i]);
+        }
         /* If ends with ".cbor.xz", load as compressed cbor file */
+        else if (ithi_endswith(fileNames[i], ".cbor.cx")) {
+            ret = LoadCborCxFile(fileNames[i]);
+        }
         /* If ends with ".csv", load as csv file */
     }
 
@@ -513,15 +519,34 @@ bool IPStats::LoadCborFile(char const* fileName)
     int err;
     bool ret = cdns_ctx.open(fileName);
 
-    while (ret) {
-        if (!cdns_ctx.open_block(&err)) {
-            ret = (err == CBOR_END_OF_ARRAY);
-            break;
-        }
-        for (size_t i = 0; i < cdns_ctx.block.queries.size(); i++) {
-            SubmitCborPacket(&cdns_ctx, i);
+    if (ret) {
+        ret = LoadCdnsRecords(&cdns_ctx, &err);
+    }
+
+    return ret;
+}
+
+bool IPStats::LoadCborCxFile(char const* fileName)
+{
+    cdns cdns_ctx;
+    int err;
+    bool ret = true;
+    FILE* F = ithi_xzcat_decompress_open(fileName, &err);
+
+    if (F == NULL) {
+        fprintf(stderr, "Cannot open pipe for %s, err = 0x%x", fileName, err);
+    }
+    else {
+        bool ret = cdns_ctx.read_entire_file(F);
+
+        if (!ret) {
+            fprintf(stderr, "Cannot read data from %s, err = 0x%x", fileName, err);
+        } else {
+            ret = LoadCdnsRecords(&cdns_ctx, &err);
         }
+        ithi_pipe_close(F);
     }
+
     return ret;
 }
 
@@ -585,6 +610,22 @@ bool IPStats::IPAddressIsLower(IPStatsRecord * x, IPStatsRecord * y)
     return ret;
 }
 
+bool IPStats::LoadCdnsRecords(cdns * cdns_ctx, int * err)
+{
+    bool ret = true;
+
+    while (ret) {
+        if (cdns_ctx->open_block(err)) {
+            ret = (*err == CBOR_END_OF_ARRAY);
+            break;
+        }
+        for (size_t i = 0; i < cdns_ctx->block.queries.size(); i++) {
+            SubmitCborPacket(cdns_ctx, i);
+        }
+    }
+    return ret;
+}
+
 void IPStats::SubmitCborPacket(cdns* cdns_ctx, size_t packet_id)
 {
     /* TODO: add to database. */
diff --git a/lib/ipstats.h b/lib/ipstats.h
index 3406e89..9b5f2f5 100644
--- a/lib/ipstats.h
+++ b/lib/ipstats.h
@@ -145,7 +145,8 @@ class IPStats
 
     /* For the command line tools */
     bool LoadInputFiles(size_t nb_files, char const** fileNames);
-    bool LoadCborFile(char const* fileNames);
+    bool LoadCborFile(char const* fileName);
+    bool LoadCborCxFile(char const* fileName);
 
     bool SaveToCsv(char const* file_name);
 
@@ -160,6 +161,7 @@ class IPStats
 
 private:
     BinHash<IPStatsRecord> ip_records;
+    bool LoadCdnsRecords(cdns * cdns_ctx, int * err);
     void SubmitCborPacket(cdns* cdns_ctx, size_t packet_id);
     static bool IPAddressIsLower(IPStatsRecord * x, IPStatsRecord * y);
     BinHash<TldAsKey> registeredTld;
diff --git a/lib/ithiutil.cpp b/lib/ithiutil.cpp
index 5b47f3f..a3bc245 100644
--- a/lib/ithiutil.cpp
+++ b/lib/ithiutil.cpp
@@ -217,3 +217,14 @@ size_t ithi_copy_to_safe_text(char* text, size_t text_max, uint8_t * x_in, size_
     return (text_length);
 }
 
+bool ithi_endswith(char const* target, char const* suffix)
+{
+    size_t target_length = strlen(target);
+    size_t suffix_length = strlen(suffix);
+    bool ret = suffix_length <= target_length;
+    if (ret) {
+        size_t ix = target_length - suffix_length;
+        ret &= (strcmp(target + ix, suffix) == 0);
+    }
+    return ret;
+}
\ No newline at end of file
diff --git a/lib/ithiutil.h b/lib/ithiutil.h
index 3d448d9..f6215b2 100644
--- a/lib/ithiutil.h
+++ b/lib/ithiutil.h
@@ -41,5 +41,7 @@ void ithi_pipe_close(FILE* F);
 
 size_t ithi_copy_to_safe_text(char* text, size_t text_max, uint8_t* x_in, size_t l_in);
 
+bool ithi_endswith(char const* target, char const* suffix);
+
 #endif
 

From ab226cce228375ae3b71e33bcfa36d80376e4e1e Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Tue, 5 Mar 2024 22:56:08 -0800
Subject: [PATCH 04/10] Fix missing bang.

---
 lib/ipstats.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/ipstats.cpp b/lib/ipstats.cpp
index ca6d90f..d4d37dc 100644
--- a/lib/ipstats.cpp
+++ b/lib/ipstats.cpp
@@ -615,7 +615,7 @@ bool IPStats::LoadCdnsRecords(cdns * cdns_ctx, int * err)
     bool ret = true;
 
     while (ret) {
-        if (cdns_ctx->open_block(err)) {
+        if (!cdns_ctx->open_block(err)) {
             ret = (*err == CBOR_END_OF_ARRAY);
             break;
         }

From bbd91875323c51d09beb241a51678788828c49d7 Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Tue, 5 Mar 2024 23:21:46 -0800
Subject: [PATCH 05/10] Test compressed cbor input

---
 ithiunit/unittest1.cpp   |  8 +++++
 lib/ipstats.cpp          |  2 +-
 test/IPStatsTest.cpp     | 65 +++++++++++++++++++++++++++++++---------
 test/IPStatsTest.h       | 10 +++++++
 test/ithi_test_class.cpp |  5 ++++
 5 files changed, 75 insertions(+), 15 deletions(-)

diff --git a/ithiunit/unittest1.cpp b/ithiunit/unittest1.cpp
index 0aaec72..084ab1b 100644
--- a/ithiunit/unittest1.cpp
+++ b/ithiunit/unittest1.cpp
@@ -380,5 +380,13 @@ namespace ithiunit
             Assert::AreEqual(ret, true);
         }
 
+        TEST_METHOD(IPStatsXZ)
+        {
+            IPStatsXZTest test;
+            bool ret = test.DoTest();
+
+            Assert::AreEqual(ret, true);
+        }
+
 	};
 }
\ No newline at end of file
diff --git a/lib/ipstats.cpp b/lib/ipstats.cpp
index d4d37dc..2eedfe4 100644
--- a/lib/ipstats.cpp
+++ b/lib/ipstats.cpp
@@ -504,7 +504,7 @@ bool IPStats::LoadInputFiles(size_t nb_files, char const** fileNames)
             ret = LoadCborFile(fileNames[i]);
         }
         /* If ends with ".cbor.xz", load as compressed cbor file */
-        else if (ithi_endswith(fileNames[i], ".cbor.cx")) {
+        else if (ithi_endswith(fileNames[i], ".cbor.xz")) {
             ret = LoadCborCxFile(fileNames[i]);
         }
         /* If ends with ".csv", load as csv file */
diff --git a/test/IPStatsTest.cpp b/test/IPStatsTest.cpp
index c5e9061..a49bca6 100644
--- a/test/IPStatsTest.cpp
+++ b/test/IPStatsTest.cpp
@@ -29,45 +29,82 @@
 #ifndef _WINDOWS64
 static char const * ipstats_test_input = "..\\data\\tiny-capture.cbor";
 static char const * ipstats_test_output = "..\\data\\ipstats-tiny-ref.csv";
+static char const * ipstats_xz_test_input = "..\\data\\tiny-capture.cbor.xz";
+static char const * ipstats_xz_test_output = "..\\data\\ipstats-xz-tiny-ref.csv";
 #else
 static char const * ipstats_test_input = "..\\..\\data\\tiny-capture.cbor";
 static char const * ipstats_test_output = "..\\..\\data\\ipstats-tiny-ref.csv";
+static char const * ipstats_xz_test_input = "..\\..\\data\\tiny-capture.cbor.xz";
+static char const * ipstats_xz_test_output = "..\\..\\data\\ipstats-xz-tiny-ref.csv";
 #endif
 #else
 static char const * ipstats_test_input = "data/tiny-capture.cbor";
 static char const * ipstats_test_output = "data/ipstats-tiny-ref.csv";
+static char const * ipstats_xz_test_input = "data/tiny-capture.cbor.xz";
+static char const * ipstats_xz_test_output = "data/ipstats-xz-tiny-ref.csv";
 #endif
 static char const* ip_stats_csv = "tiny-capture-ipstats.csv";
+static char const* ip_stats_xz_csv = "tiny-capture-ipstats-xz.csv";
 
 
-IPStatsTest::IPStatsTest()
-{}
-
-IPStatsTest::~IPStatsTest()
-{}
-
-bool IPStatsTest::DoTest()
+bool IPStatsTestOne(
+    char const * result_file,
+    char const * ref_file,
+    char const** input_files,
+    size_t nb_input_files
+)
 {
     IPStats ipstats;
     char const * list[1] = { ipstats_test_input };
-    bool ret = ipstats.LoadInputFiles(1, list);
+    bool ret = ipstats.LoadInputFiles(nb_input_files, input_files);
 
     if (!ret){
-        TEST_LOG("Cannot process the CBOR input file: %s\n", list[0]);
+        TEST_LOG("Cannot process the input file: %s\n", input_files[0]);
     }
     else
     {
-        ret = ipstats.SaveToCsv(ip_stats_csv);
+        ret = ipstats.SaveToCsv(result_file);
         if (!ret) {
-            TEST_LOG("Cannot save to csv file: %s.\n", ip_stats_csv);
+            TEST_LOG("Cannot save to csv file: %s.\n", result_file);
         }
         else {
-            TEST_LOG("IP Stats have been saved to %s\n", ip_stats_csv);
-            
-            ret = MetricTest::compare_files(ip_stats_csv, ipstats_test_output);
+            TEST_LOG("IP Stats have been saved to %s\n", result_file);
+
+            ret = MetricTest::compare_files(result_file, ref_file);
         }
     }
 
     return ret;
 }
 
+IPStatsTest::IPStatsTest()
+{}
+
+IPStatsTest::~IPStatsTest()
+{}
+
+bool IPStatsTest::DoTest()
+{
+    char const * list[1] = { ipstats_test_input };
+
+    bool ret = IPStatsTestOne(ip_stats_csv, ipstats_test_output, list, 1);
+
+    return ret;
+}
+
+
+IPStatsXZTest::IPStatsXZTest()
+{}
+
+IPStatsXZTest::~IPStatsXZTest()
+{}
+
+bool IPStatsXZTest::DoTest()
+{
+    IPStats ipstats;
+    char const * list[1] = { ipstats_xz_test_input };
+
+    bool ret = IPStatsTestOne(ip_stats_xz_csv, /* TODO: change! */ ipstats_test_output, list, 1);
+
+    return ret;
+}
diff --git a/test/IPStatsTest.h b/test/IPStatsTest.h
index fe309f0..fe54923 100644
--- a/test/IPStatsTest.h
+++ b/test/IPStatsTest.h
@@ -33,5 +33,15 @@ class IPStatsTest : public ithi_test_class
     bool DoTest() override;
 };
 
+class IPStatsXZTest : public ithi_test_class
+{
+public:
+    IPStatsXZTest();
+    ~IPStatsXZTest();
+
+    bool DoTest() override;
+};
+
+
 #endif /* SAVE_TEST_H */
 
diff --git a/test/ithi_test_class.cpp b/test/ithi_test_class.cpp
index a075a9d..9b09e7f 100644
--- a/test/ithi_test_class.cpp
+++ b/test/ithi_test_class.cpp
@@ -75,6 +75,7 @@ enum test_list_enum {
     test_enum_TrailingZeroes,
     test_enum_HyperLogLog,
     test_enum_IPStats,
+    test_enum_IPStatsXZ,
     test_enum_max_number,
 };
 
@@ -178,6 +179,8 @@ char const * ithi_test_class::GetTestName(int number)
         return("HyperLogLog");
     case test_enum_IPStats:
         return("IPStats");
+    case test_enum_IPStatsXZ:
+        return("IPStatsXZ");
     default:
         break;
     }
@@ -330,6 +333,8 @@ ithi_test_class * ithi_test_class::TestByNumber(int number)
     case test_enum_IPStats:
         test = new IPStatsTest();
         break;
+    case test_enum_IPStatsXZ:
+        test = new IPStatsXZTest();
     default:
         break;
     }

From 9aa97df74828ccc9854ce596d23ce64ed5fef75d Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Wed, 6 Mar 2024 12:17:19 -0800
Subject: [PATCH 06/10] Add test of compressed load

---
 cdns                     |  2 +-
 ithiunit/unittest1.cpp   |  8 ++++++++
 test/IPStatsTest.cpp     | 41 +++++++++++++++++++++++++++++++++++++++-
 test/IPStatsTest.h       |  8 ++++++++
 test/ithi_test_class.cpp |  6 ++++++
 5 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/cdns b/cdns
index 6d0f504..5eea536 160000
--- a/cdns
+++ b/cdns
@@ -1 +1 @@
-Subproject commit 6d0f504dbfa4a9412f322b86ac88365def7d0019
+Subproject commit 5eea5364e7390f2ed0b0e6e38e9c0d856d2d816c
diff --git a/ithiunit/unittest1.cpp b/ithiunit/unittest1.cpp
index 084ab1b..da3f6d5 100644
--- a/ithiunit/unittest1.cpp
+++ b/ithiunit/unittest1.cpp
@@ -380,6 +380,14 @@ namespace ithiunit
             Assert::AreEqual(ret, true);
         }
 
+        TEST_METHOD(IPStatsLoad)
+        {
+            IPStatsLoadTest test;
+            bool ret = test.DoTest();
+
+            Assert::AreEqual(ret, true);
+        }
+
         TEST_METHOD(IPStatsXZ)
         {
             IPStatsXZTest test;
diff --git a/test/IPStatsTest.cpp b/test/IPStatsTest.cpp
index a49bca6..2771bbd 100644
--- a/test/IPStatsTest.cpp
+++ b/test/IPStatsTest.cpp
@@ -20,6 +20,7 @@
 */
 
 #include "ithi_test_class.h"
+#include "ithiutil.h"
 #include "ipstats.h"
 #include "MetricTest.h"
 #include "IPStatsTest.h"
@@ -104,7 +105,45 @@ bool IPStatsXZTest::DoTest()
     IPStats ipstats;
     char const * list[1] = { ipstats_xz_test_input };
 
-    bool ret = IPStatsTestOne(ip_stats_xz_csv, /* TODO: change! */ ipstats_test_output, list, 1);
+    bool ret = IPStatsTestOne(ip_stats_xz_csv, /* TODO: change?*/ ipstats_test_output, list, 1);
 
     return ret;
 }
+
+IPStatsLoadTest::IPStatsLoadTest()
+{
+}
+
+IPStatsLoadTest::~IPStatsLoadTest()
+{
+}
+
+bool IPStatsLoadTest::DoTest()
+{
+    bool ret;
+    cdns cdns_ctx;
+    FILE* P = NULL;
+    int err;
+    if (!(ret = cdns_ctx.open(ipstats_test_input))) {
+        fprintf(stderr, "Cannot read file %s\n", ipstats_test_input);
+    }
+    else {
+        cdns cdns_ctx_xz;
+
+        P = ithi_xzcat_decompress_open(ipstats_xz_test_input, &err);
+
+        if (P == NULL) {
+            fprintf(stderr, "Cannot open pipe for %s, err = 0x%x\n", ipstats_xz_test_input, err);
+        }
+        else if (!(ret = cdns_ctx_xz.read_entire_file(P))) {
+            fprintf(stderr, "Cannot decompress file %s\n", ipstats_xz_test_input);
+        }
+        else if (!(ret = (cdns_ctx_xz.buf_read == cdns_ctx.buf_read))) {
+            fprintf(stderr, "Read lengths differ, %zu from xz vs %zu\n", cdns_ctx_xz.buf_read, cdns_ctx.buf_read);
+        }
+        else if (!(ret = (memcmp(cdns_ctx_xz.buf, cdns_ctx.buf, cdns_ctx.buf_read) == 0))) {
+            fprintf(stderr, "Decompressed content differs from file content.\n");
+        }
+    }
+    return ret;
+}
diff --git a/test/IPStatsTest.h b/test/IPStatsTest.h
index fe54923..96b065e 100644
--- a/test/IPStatsTest.h
+++ b/test/IPStatsTest.h
@@ -42,6 +42,14 @@ class IPStatsXZTest : public ithi_test_class
     bool DoTest() override;
 };
 
+class IPStatsLoadTest : public ithi_test_class
+{
+public:
+    IPStatsLoadTest();
+    ~IPStatsLoadTest();
+
+    bool DoTest() override;
+};
 
 #endif /* SAVE_TEST_H */
 
diff --git a/test/ithi_test_class.cpp b/test/ithi_test_class.cpp
index 9b09e7f..e31cdc4 100644
--- a/test/ithi_test_class.cpp
+++ b/test/ithi_test_class.cpp
@@ -75,6 +75,7 @@ enum test_list_enum {
     test_enum_TrailingZeroes,
     test_enum_HyperLogLog,
     test_enum_IPStats,
+    test_enum_IPStatsLoad,
     test_enum_IPStatsXZ,
     test_enum_max_number,
 };
@@ -179,6 +180,8 @@ char const * ithi_test_class::GetTestName(int number)
         return("HyperLogLog");
     case test_enum_IPStats:
         return("IPStats");
+    case test_enum_IPStatsLoad:
+        return("IPStatsLoad");
     case test_enum_IPStatsXZ:
         return("IPStatsXZ");
     default:
@@ -333,6 +336,9 @@ ithi_test_class * ithi_test_class::TestByNumber(int number)
     case test_enum_IPStats:
         test = new IPStatsTest();
         break;
+    case test_enum_IPStatsLoad:
+        test = new IPStatsLoadTest();
+        break;
     case test_enum_IPStatsXZ:
         test = new IPStatsXZTest();
     default:

From 43c8f088e9ba70ff4474f46b9ff6ed4bf9789bec Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Wed, 6 Mar 2024 12:27:30 -0800
Subject: [PATCH 07/10] Remove Guardian

---
 lib/DnsStats.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/DnsStats.cpp b/lib/DnsStats.cpp
index 56112bd..84e0d77 100644
--- a/lib/DnsStats.cpp
+++ b/lib/DnsStats.cpp
@@ -76,7 +76,6 @@ DnsStats::~DnsStats()
 static char const * DefaultRootAddresses[] = {
     "2001:503:ba3e::2:30",
     "198.41.0.4",
-    "2001:500:200::b",
     "192.228.79.201", /* Since 2017-10-24, b.root-servers.net changed from 192.228.79.201 to 199.9.14.201 */
     "199.9.14.201",   /* See https://b.root-servers.org/news/2017/08/09/new-ipv4.html */
     "2001:500:2::c",
@@ -240,7 +239,7 @@ char const * RegisteredTldName[] = {
     "GLASS", "GLE", "GLOBAL", "GLOBO", "GM", "GMAIL", "GMBH", "GMO", "GMX", "GN", "GODADDY",
     "GOLD", "GOLDPOINT", "GOLF", "GOO", "GOODYEAR", "GOOG", "GOOGLE", "GOP",
     "GOT", "GOV", "GP", "GQ", "GR", "GRAINGER", "GRAPHICS", "GRATIS", "GREEN", "GRIPE",
-    "GROCERY", "GROUP", "GS", "GT", "GU", "GUARDIAN", "GUCCI", "GUGE", "GUIDE", "GUITARS",
+    "GROCERY", "GROUP", "GS", "GT", "GU", "GUCCI", "GUGE", "GUIDE", "GUITARS",
     "GURU", "GW", "GY", "HAIR", "HAMBURG", "HANGOUT", "HAUS", "HBO", "HDFC", "HDFCBANK",
     "HEALTH", "HEALTHCARE", "HELP", "HELSINKI", "HERE", "HERMES", "HIPHOP",
     "HISAMITSU", "HITACHI", "HIV", "HK", "HKT", "HM", "HN", "HOCKEY", "HOLDINGS", "HOLIDAY",

From c5604900a35feec34a47ec0cb13dbb666da7dcc4 Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Wed, 6 Mar 2024 12:41:27 -0800
Subject: [PATCH 08/10] Install xz and call xzcat for load

---
 .github/workflows/ci-tests-clang.yml   | 4 ++++
 .github/workflows/ci-tests-macos.yml   | 4 ++++
 .github/workflows/ci-tests-privacy.yml | 4 ++++
 .github/workflows/ci-tests.yml         | 4 ++++
 lib/ithiutil.cpp                       | 2 +-
 5 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-tests-clang.yml b/.github/workflows/ci-tests-clang.yml
index e714951..6a4cbd2 100644
--- a/.github/workflows/ci-tests-clang.yml
+++ b/.github/workflows/ci-tests-clang.yml
@@ -25,6 +25,10 @@ jobs:
           # a pull request then we can checkout the head.
           fetch-depth: 2
           submodules: 'recursive'
+          
+      - name: Install xz
+        run: |
+          sudo apt-get install -y xz
 
       - name: Run cmake
         run: |
diff --git a/.github/workflows/ci-tests-macos.yml b/.github/workflows/ci-tests-macos.yml
index bf1611b..bd662ae 100644
--- a/.github/workflows/ci-tests-macos.yml
+++ b/.github/workflows/ci-tests-macos.yml
@@ -22,6 +22,10 @@ jobs:
           # a pull request then we can checkout the head.
           fetch-depth: 2
           submodules: 'recursive'
+          
+      - name: Install xz
+        run: |
+          brew install xz
 
       - name: Run cmake
         run: |
diff --git a/.github/workflows/ci-tests-privacy.yml b/.github/workflows/ci-tests-privacy.yml
index d7dbe45..94e92a2 100644
--- a/.github/workflows/ci-tests-privacy.yml
+++ b/.github/workflows/ci-tests-privacy.yml
@@ -22,6 +22,10 @@ jobs:
           # a pull request then we can checkout the head.
           fetch-depth: 2
           submodules: 'recursive'
+          
+      - name: Install xz
+        run: |
+          sudo apt-get install -y xz
 
       - name: Run cmake
         run: |
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
index c44919a..e157337 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@@ -30,6 +30,10 @@ jobs:
             cmake .
             make -j 4
 
+      - name: Install xz
+        run: |
+          sudo apt-get install -y xz
+
       - name: Perform Unit Tests
         run: |
             ./ithitest
diff --git a/lib/ithiutil.cpp b/lib/ithiutil.cpp
index a3bc245..86dc6af 100644
--- a/lib/ithiutil.cpp
+++ b/lib/ithiutil.cpp
@@ -141,7 +141,7 @@ FILE* ithi_xzcat_decompress_open(char const* file_name, int* last_err)
     /* Running on windows requires that 7z.exe is installed */
     xzcat_command = "7z.exe e -so";
 #else
-    xzcat_command = "zxcat -k";
+    xzcat_command = "xzcat -k";
 #endif
 
 #ifdef _WINDOWS

From 67a446fdb6c0844ba7dd52cc1bc04c2d094af243 Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Wed, 6 Mar 2024 12:52:51 -0800
Subject: [PATCH 09/10] On ubuntu install xz-utils

---
 .github/workflows/ci-tests-clang.yml   | 2 +-
 .github/workflows/ci-tests-privacy.yml | 2 +-
 .github/workflows/ci-tests.yml         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci-tests-clang.yml b/.github/workflows/ci-tests-clang.yml
index 6a4cbd2..176c3ed 100644
--- a/.github/workflows/ci-tests-clang.yml
+++ b/.github/workflows/ci-tests-clang.yml
@@ -28,7 +28,7 @@ jobs:
           
       - name: Install xz
         run: |
-          sudo apt-get install -y xz
+          sudo apt-get install -y xz-utils
 
       - name: Run cmake
         run: |
diff --git a/.github/workflows/ci-tests-privacy.yml b/.github/workflows/ci-tests-privacy.yml
index 94e92a2..821bff2 100644
--- a/.github/workflows/ci-tests-privacy.yml
+++ b/.github/workflows/ci-tests-privacy.yml
@@ -25,7 +25,7 @@ jobs:
           
       - name: Install xz
         run: |
-          sudo apt-get install -y xz
+          sudo apt-get install -y xz-utils
 
       - name: Run cmake
         run: |
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
index e157337..e8d649c 100644
--- a/.github/workflows/ci-tests.yml
+++ b/.github/workflows/ci-tests.yml
@@ -32,7 +32,7 @@ jobs:
 
       - name: Install xz
         run: |
-          sudo apt-get install -y xz
+          sudo apt-get install -y xz-utils
 
       - name: Perform Unit Tests
         run: |

From 510ac4ea97931cef08ddfdd135a5104d2448ee90 Mon Sep 17 00:00:00 2001
From: Christian Huitema <huitema@huitema.net>
Date: Wed, 6 Mar 2024 13:00:14 -0800
Subject: [PATCH 10/10] Fix scanbuild warnings

---
 lib/ipstats.cpp      | 9 ++++++---
 test/IPStatsTest.cpp | 4 ----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/lib/ipstats.cpp b/lib/ipstats.cpp
index 2eedfe4..5945658 100644
--- a/lib/ipstats.cpp
+++ b/lib/ipstats.cpp
@@ -534,15 +534,18 @@ bool IPStats::LoadCborCxFile(char const* fileName)
     FILE* F = ithi_xzcat_decompress_open(fileName, &err);
 
     if (F == NULL) {
-        fprintf(stderr, "Cannot open pipe for %s, err = 0x%x", fileName, err);
+        fprintf(stderr, "Cannot open pipe for %s, err = 0x%x\n", fileName, err);
     }
     else {
-        bool ret = cdns_ctx.read_entire_file(F);
+        ret = cdns_ctx.read_entire_file(F);
 
         if (!ret) {
-            fprintf(stderr, "Cannot read data from %s, err = 0x%x", fileName, err);
+            fprintf(stderr, "Cannot read data from %s, err = 0x%x\n", fileName, err);
         } else {
             ret = LoadCdnsRecords(&cdns_ctx, &err);
+            if (!ret) {
+                fprintf(stderr, "Cannot load records from %s, err = 0x%x\n", fileName, err);
+            }
         }
         ithi_pipe_close(F);
     }
diff --git a/test/IPStatsTest.cpp b/test/IPStatsTest.cpp
index 2771bbd..0f90c96 100644
--- a/test/IPStatsTest.cpp
+++ b/test/IPStatsTest.cpp
@@ -31,18 +31,15 @@
 static char const * ipstats_test_input = "..\\data\\tiny-capture.cbor";
 static char const * ipstats_test_output = "..\\data\\ipstats-tiny-ref.csv";
 static char const * ipstats_xz_test_input = "..\\data\\tiny-capture.cbor.xz";
-static char const * ipstats_xz_test_output = "..\\data\\ipstats-xz-tiny-ref.csv";
 #else
 static char const * ipstats_test_input = "..\\..\\data\\tiny-capture.cbor";
 static char const * ipstats_test_output = "..\\..\\data\\ipstats-tiny-ref.csv";
 static char const * ipstats_xz_test_input = "..\\..\\data\\tiny-capture.cbor.xz";
-static char const * ipstats_xz_test_output = "..\\..\\data\\ipstats-xz-tiny-ref.csv";
 #endif
 #else
 static char const * ipstats_test_input = "data/tiny-capture.cbor";
 static char const * ipstats_test_output = "data/ipstats-tiny-ref.csv";
 static char const * ipstats_xz_test_input = "data/tiny-capture.cbor.xz";
-static char const * ipstats_xz_test_output = "data/ipstats-xz-tiny-ref.csv";
 #endif
 static char const* ip_stats_csv = "tiny-capture-ipstats.csv";
 static char const* ip_stats_xz_csv = "tiny-capture-ipstats-xz.csv";
@@ -56,7 +53,6 @@ bool IPStatsTestOne(
 )
 {
     IPStats ipstats;
-    char const * list[1] = { ipstats_test_input };
     bool ret = ipstats.LoadInputFiles(nb_input_files, input_files);
 
     if (!ret){