enable more test data.

cmd/root.go

@@ -172,7 +172,6 @@ func initConfig(cmd *cobra.Command) error {
 
 	utd, err := cmd.Flags().GetBool("use-test-data")
 	if err != nil {
-		fmt.Println("error getting use-test-data value:", err)
 		os.Exit(1)
 	}
 

providers/aws/aws.go

@@ -139,8 +139,6 @@ func loadTestData(c *ProviderClient) ([]byte, error) {
 		return nil, fmt.Errorf("error marshalling test data: %w", err)
 	}
 
-	c.Logger.Info("aws match returned from test data", "host", c.Host.String())
-
 	return out, nil
 }
 
@@ -216,17 +214,13 @@ func (c *ProviderClient) FindHost() ([]byte, error) {
 		return nil, fmt.Errorf("error parsing create date: %w", err)
 	}
 
-	// match.ETag = item.Version
-
 	var raw []byte
 
 	raw, err = json.Marshal(match)
 	if err != nil {
 		return nil, fmt.Errorf("error marshalling response: %w", err)
 	}
 
-	// match.Raw = raw
-
 	// TODO: remove before release
 	if os.Getenv("CCI_BACKUP_RESPONSES") == "true" {
 		if err = os.WriteFile(fmt.Sprintf("%s/backups/aws_%s_report.json", config.GetConfigRoot("", config.AppName),

providers/aws/testdata/aws_18_164_52_75_report.json

@@ -0,0 +1,16 @@
+{
+  "Raw": null,
+  "prefix": {
+    "ip_prefix": "18.164.0.0/15",
+    "region": "GLOBAL",
+    "service": "AMAZON"
+  },
+  "ipv6Prefix": {
+    "ipv6_prefix": "",
+    "region": "",
+    "service": ""
+  },
+  "etag": "",
+  "syncToken": "",
+  "createDate": "0001-01-01T00:00:00Z"
+}

providers/criminalip/criminalip.go

@@ -307,7 +307,7 @@ func (c *ProviderClient) GenPortDataForTable(in []PortDataEntry) (GeneratePortDa
 		}
 	}
 
-	return out, fmt.Errorf("error generating port data for table: %w", err)
+	return out, nil
 }
 
 func (c *ProviderClient) CreateTable(data []byte) (*table.Writer, error) {

providers/digitalocean/digitalocean.go

@@ -189,7 +189,7 @@ func (c *ProviderClient) FindHost() ([]byte, error) {
 
 	// return cached report if test data is enabled
 	if c.UseTestData {
-		result, err = loadResultsFile("providers/digitalocean/testdata/digitalocean_18_164_52_75_report.json")
+		result, err = loadResultsFile("providers/digitalocean/testdata/digitalocean_165_232_46_239_report.json")
 		if err != nil {
 			return nil, err
 		}

providers/digitalocean/testdata/digitalocean_165_232_46_239_report.json

@@ -0,0 +1,13 @@
+{
+  "Raw": null,
+  "prefix": {
+    "Network": "165.232.32.0/20",
+    "NetworkText": "165.232.32.0/20",
+    "CountryCode": "GB",
+    "CityCode": "GB-SLG",
+    "CityName": "London",
+    "ZipCode": "SL1 4AX"
+  },
+  "etag": "\"662bc170-e030\"",
+  "last_modified": "2024-04-26T16:00:00+01:00"
+}

providers/ipurl/ipurl.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
+	"os"
 	"sync"
 	"time"
 
@@ -38,6 +39,38 @@ func (c *ProviderClient) GetConfig() *config.Config {
 	return &c.Config
 }
 
+func loadTestData(c *ProviderClient) ([]byte, error) {
+	tdf, err := loadResultsFile("providers/ipurl/testdata/ipurl_5_105_62_0_report.json")
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := json.Marshal(tdf)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling test data: %w", err)
+	}
+
+	return out, nil
+}
+
+func loadResultsFile(path string) (res *HostSearchResult, err error) {
+	jf, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("error opening file: %w", err)
+	}
+
+	defer jf.Close()
+
+	decoder := json.NewDecoder(jf)
+
+	err = decoder.Decode(&res)
+	if err != nil {
+		return res, fmt.Errorf("error decoding json: %w", err)
+	}
+
+	return res, nil
+}
+
 func (c *ProviderClient) FindHost() ([]byte, error) {
 	start := time.Now()
 	defer func() {
@@ -46,6 +79,10 @@ func (c *ProviderClient) FindHost() ([]byte, error) {
 		c.Stats.Mu.Unlock()
 	}()
 
+	if c.UseTestData {
+		return loadTestData(c)
+	}
+
 	pwp := make(map[netip.Prefix][]string)
 
 	err := c.loadProviderDataFromCache(pwp)

providers/ipurl/testdata/ipurl_5_105_62_0_report.json

@@ -0,0 +1 @@
+{"5.105.62.0/24":["https://iplists.firehol.org/files/firehol_level1.netset"]}

providers/ptr/ptr.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"log/slog"
 	"net/netip"
 	"os"
 	"strings"
@@ -202,6 +203,9 @@ func unmarshalResponse(data []byte) (*HostSearchResult, error) {
 	res.Raw = data
 	res.Data = uData
 
+	d, _ := json.MarshalIndent(res, "", "  ")
+	fmt.Println(string(d))
+
 	return &res, nil
 }
 
@@ -229,13 +233,24 @@ func (ssr *HostSearchResult) CreateTable() *table.Writer {
 	return &tw
 }
 
+func loadTestData(l *slog.Logger) (*HostSearchResult, error) {
+	tdf, err := loadResultsFile("providers/ptr/testdata/ptr_8_8_8_8_report.json")
+	if err != nil {
+		return nil, err
+	}
+
+	l.Info("ptr match returned from test data", "host", "8.8.8.8")
+
+	return tdf, nil
+}
+
 func fetchData(c config.Config) (*HostSearchResult, error) {
 	var result *HostSearchResult
 
 	var err error
 
 	if c.UseTestData {
-		result, err = loadResultsFile("providers/ptr/testdata/ptr_google_dns_resp.json")
+		result, err = loadTestData(c.Logger)
 		if err != nil {
 			return nil, fmt.Errorf("error loading ptr test data: %w", err)
 		}

providers/ptr/testdata/ptr_8_8_8_8_report.json

@@ -0,0 +1,34 @@
+{
+  "raw": "eyJwdHIiOlt7IkhkciI6eyJOYW1lIjoiOC44LjguOC5pbi1hZGRyLmFycGEuIiwiUnJ0eXBlIjoxMiwiQ2xhc3MiOjEsIlR0bCI6MzUwNDEsIlJkbGVuZ3RoIjoxMn0sIlB0ciI6ImRucy5nb29nbGUuIn1dLCJtc2ciOnsiSWQiOjAsIlJlc3BvbnNlIjpmYWxzZSwiT3Bjb2RlIjowLCJBdXRob3JpdGF0aXZlIjpmYWxzZSwiVHJ1bmNhdGVkIjpmYWxzZSwiUmVjdXJzaW9uRGVzaXJlZCI6ZmFsc2UsIlJlY3Vyc2lvbkF2YWlsYWJsZSI6ZmFsc2UsIlplcm8iOmZhbHNlLCJBdXRoZW50aWNhdGVkRGF0YSI6ZmFsc2UsIkNoZWNraW5nRGlzYWJsZWQiOmZhbHNlLCJSY29kZSI6MCwiUXVlc3Rpb24iOm51bGwsIkFuc3dlciI6bnVsbCwiTnMiOm51bGwsIkV4dHJhIjpudWxsfX0=",
+  "data": {
+    "ptr": [
+      {
+        "Hdr": {
+          "Name": "8.8.8.8.in-addr.arpa.",
+          "Rrtype": 12,
+          "Class": 1,
+          "Ttl": 35041,
+          "Rdlength": 12
+        },
+        "Ptr": "dns.google."
+      }
+    ],
+    "msg": {
+      "Id": 0,
+      "Response": false,
+      "Opcode": 0,
+      "Authoritative": false,
+      "Truncated": false,
+      "RecursionDesired": false,
+      "RecursionAvailable": false,
+      "Zero": false,
+      "AuthenticatedData": false,
+      "CheckingDisabled": false,
+      "Rcode": 0,
+      "Question": null,
+      "Answer": null,
+      "Ns": null,
+      "Extra": null
+    }
+  }
+}

providers/shodan/shodan.go

@@ -115,6 +115,7 @@ func loadAPIResponse(ctx context.Context, c config.Config, apiKey string) (res *
 	}
 
 	res.Raw = rBody
+
 	if res.Raw == nil {
 		return nil, fmt.Errorf("shodan: %w", providers.ErrNoMatchFound)
 	}
@@ -135,6 +136,13 @@ func unmarshalResponse(data []byte) (*HostSearchResult, error) {
 }
 
 func loadResultsFile(path string) (res *HostSearchResult, err error) {
+	// get raw data
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("error reading shodan file: %w", err)
+	}
+
+	// unmarshal data
 	jf, err := os.Open(path)
 	if err != nil {
 		return nil, fmt.Errorf("error opening shodan file: %w", err)
@@ -149,6 +157,8 @@ func loadResultsFile(path string) (res *HostSearchResult, err error) {
 		return res, fmt.Errorf("error decoding shodan file: %w", err)
 	}
 
+	res.Raw = raw
+
 	return res, nil
 }