Option to show partial search results when limit is triggered

README.md

@@ -200,7 +200,6 @@ Response example for the first query:
 - [ ] Edges groups styling. **TODO** from `search.js`. Implement https://github.com/visjs/vis-network/issues/1229
 - [ ] Data sources access based on user groups permissions
 - [ ] API can return an image instead of JSON
-- [ ] Option to show partial results when limit is triggered
 - [ ] Data source plugins:
   - [ ] RTIR
   - [ ] MS SQL

account.go

@@ -91,6 +91,9 @@ type Options struct {
 	// Will be a part of each SQL query, set to 0 to disable
 	Limit int `bson:"limit"`
 
+	// Whether to show partial search results when limit exceeded
+	ShowLimited bool `bson:"showLimited"`
+
 	// Whether to display queries debug info
 	Debug bool `bson:"debug"`
 }

api.go

@@ -34,6 +34,7 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
 	//   - SQL request
 	uuid := r.FormValue("uuid")
 	format := r.FormValue("format")
+	showLimited := false
 	includeDebug := false
 	sql := r.FormValue("sql")
 
@@ -67,6 +68,11 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Show partial results when limit exceeded
+	if r.FormValue("show_limited") == "true" {
+		showLimited = true
+	}
+
 	// Disable debug info by default
 	if r.FormValue("debug") == "true" {
 		includeDebug = true
@@ -88,7 +94,7 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
 	source := match[1]
 
 	// Query data sources for the new relations
-	response = querySources(source, sql, includeDebug, account.Username)
+	response = querySources(source, sql, showLimited, includeDebug, account.Username)
 
 	if len(response.Stats) != 0 {
 		if response.Error != "" {
@@ -106,7 +112,7 @@ func apiHandler(w http.ResponseWriter, r *http.Request) {
 /*
  * Query all the requested data sources
  */
-func querySources(source, sql string, includeDebug bool, username string) *APIresponse {
+func querySources(source, sql string, showLimited, includeDebug bool, username string) *APIresponse {
 
 	// Response to send back
 	response := &APIresponse{
@@ -173,7 +179,10 @@ func querySources(source, sql string, includeDebug bool, username string) *APIre
 					}
 
 					response.Lock()
-					response.Relations = append(response.Relations, result...)
+
+					if stat == nil || (stat != nil && showLimited) {
+						response.Relations = append(response.Relations, result...)
+					}
 
 					if includeDebug {
 						response.Debug[collector.Conf().Name] = debug

assets/css/profile.css

@@ -22,6 +22,9 @@
     margin-top: 9px;
     position: absolute;
 }
+.ui.toggle.checkbox {
+    margin-top: 9px;
+}
 
 .ui.grid>.row.username {
     padding-bottom: 2rem;

assets/js/charts.js

@@ -54,7 +54,7 @@ class Charts {
         this.header.innerHTML = '<strong>' + data.source.toUpperCase() + '</strong> returns too many results. ';
         if (issql)
             this.header.innerHTML += 'Add filters manually or use the charts (based on limited data) to reduce the amount of returned data. ';
-        this.header.innerHTML += 'Close the charts to see the possible data from the other sources';
+        this.header.innerHTML += 'Close the charts to see the possible limited data';
 
         this.container.style.display = 'block';
 

assets/js/options.js

@@ -37,6 +37,7 @@ class Options {
     save() {
         const options = document.getElementById('stabilization').value + ',' +
                         document.getElementById('limit').value + ',' +
+                        $('.ui.checkbox.show_limited').checkbox('is checked') + ',' +
                         $('.ui.checkbox.debug').checkbox('is checked');
 
         // Send to the server

assets/tmpl/profile.html

@@ -7,10 +7,10 @@
 
         <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1, maximum-scale=1">
 
-        <link href="assets/css/semantic-2.8.6.min.css"                rel="stylesheet" type="text/css" />
-        <link href="assets/css/font-awesome.min.css?v={{ .Version }}" rel="stylesheet" type="text/css" />
-        <link href="assets/css/common.css?v={{ .Version }}"           rel="stylesheet" type="text/css" />
-        <link href="assets/css/profile.css?v={{ .Version }}"          rel="stylesheet" type="text/css" />
+        <link href="assets/css/semantic-2.8.6.min.css"               rel="stylesheet" type="text/css" />
+        <link href="assets/css/fontawesome.min.css?v={{ .Version }}" rel="stylesheet" type="text/css" />
+        <link href="assets/css/common.css?v={{ .Version }}"          rel="stylesheet" type="text/css" />
+        <link href="assets/css/profile.css?v={{ .Version }}"         rel="stylesheet" type="text/css" />
 
         <script type="text/javascript" src="assets/js/jquery-3.4.1.min.js"></script>
         <script type="text/javascript" src="assets/js/semantic-2.8.6.min.js"></script>
@@ -91,6 +91,22 @@ <h3 class="ui header">
                     </div>
                 </div>
 
+                <div class="row short">
+                    <p>Whether to show partial search results when limit exceeded. By default only statistics charts are displayed to update the query when possible. But in some cases you may want to see partial results too.</p>
+                </div>
+
+                <div class="row short">
+                    <div class="column">
+                        <span>Show limited results:</span>
+                    </div>
+                    <div class="column">
+                        <div class="ui toggle checkbox show_limited">
+                            <input type="checkbox" {{ if .Account.Options.ShowLimited }}checked{{ end }}>
+                            <label></label>
+                        </div>
+                    </div>
+                </div>
+
                 <div class="row short">
                     <p>Whether to display queries debug info. During the user interaction many actions happen like SQL to Elasticsearch JSON query conversion, fields name adaptation, etc. Each plugin can save progress information and return to the user. Accessible in a browser's console.</p>
                 </div>

docs/search.md

@@ -11,7 +11,8 @@
 11. [Direct API usage](#direct-api-usage)
 12. [Limit the amount of returned data](#limit-the-amount-of-returned-data)
 13. [Order of returned data](#order-of-returned-data)
-14. [Output format](#output-format)
+14. [Show partial search results](#show-partial-search-results)
+15. [Output format](#output-format)
 
 
 ![datasources](assets/img/datasources.png)
@@ -200,6 +201,17 @@ However, `LIMIT` is not a total amount of graph nodes or edges - it goes to the
 Direct queries also can include `ORDER BY field` or `ORDER BY field DESC`. The default sorting method is alphabetical. However, `DESC` can be used for displaying rows in descending order.
 
 
+## Show partial search results
+
+When search results limit exceeded - partial results can be displayed. By default only statistics charts are displayed to update the query when possible. But in some cases you may want to see partial results too.
+
+Example in API query:
+```sh
+curl -XGET 'https://server/api?uuid=09e545f2-3986-493c-983a-e39d310f695a&show_limited=true&sql=FROM+people+WHERE+age>30'
+```
+... where `show_limited=true` parameter enables or disables partial results.
+
+
 ## Output format
 
 By default JSON is used to represent graph relations data. However, sometimes you may need to display data as a table. In such cases the output formatting feature can be used.`json` or `table` are currently supported.

files/features.yaml

@@ -1,7 +1,6 @@
 [
-    "13.11.2024",
+    "20.12.2024",
 
-    "Saved dashboard can be accessed directly by URL",
-    "Data sources dropdown shows which sources <strong>do not support SQL or datetime range</strong>",
-    "Latest plugins: <strong>ipinfo.io</strong>, <strong>phishtank.org</strong>"
+    "An option to show partial search results when limit exceeded",
+    "Latest plugin: <strong>Shodan</strong>"
 ]

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/mithrandie/csvq-driver v1.7.0
 	github.com/ns3777k/go-shodan/v4 v4.2.0
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/redis/go-redis/v9 v9.5.3
+	github.com/redis/go-redis/v9 v9.7.0
 	github.com/rs/zerolog v1.33.0
 	github.com/umpc/go-sortedmap v0.0.0-20180422175548-64ab94c482f4
 	github.com/yukithm/json2csv v0.1.2

go.sum

@@ -213,8 +213,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU=
-github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=

plugins/src/abuseipdb/abuseipdb.go

@@ -158,7 +158,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 					return nil, nil, debug, err
 				}
 
-				return nil, top, debug, nil
+				return results, top, debug, nil
 			}
 
 			// Update stats

plugins/src/abuseipdb/plugin.go

@@ -9,7 +9,7 @@ import (
  */
 var (
 	Name    = "abuseipdb"
-	Version = "1.0.0"
+	Version = "1.0.1"
 	Plugin  plugin
 )
 

plugins/src/circl_passive_ssl/passive_ssl.go

@@ -158,7 +158,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		// Update stats

plugins/src/circl_passive_ssl/plugin.go

@@ -9,7 +9,7 @@ import (
  */
 var (
 	Name    = "circl_passive_ssl"
-	Version = "1.0.0"
+	Version = "1.0.1"
 	Plugin  plugin
 )
 

plugins/src/elasticsearch.v7/elasticsearch.go

@@ -225,7 +225,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		entry, ok := hit["_source"].(map[string]interface{})

plugins/src/elasticsearch.v7/plugin.go

@@ -10,7 +10,7 @@ import (
  */
 var (
 	Name    = "elasticsearch.v7"
-	Version = "1.0.8"
+	Version = "1.0.9"
 	Plugin  plugin
 )
 

plugins/src/elasticsearch.v8/elasticsearch.go

@@ -225,7 +225,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		entry, ok := hit["_source"].(map[string]interface{})

plugins/src/elasticsearch.v8/plugin.go

@@ -10,7 +10,7 @@ import (
  */
 var (
 	Name    = "elasticsearch.v8"
-	Version = "1.0.1"
+	Version = "1.0.2"
 	Plugin  plugin
 )
 

plugins/src/file/csv/csv.go

@@ -159,7 +159,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		if err := rows.Scan(row...); err != nil {

plugins/src/file/csv/plugin.go

@@ -11,7 +11,7 @@ import (
  */
 var (
 	Name    = "file-csv"
-	Version = "1.0.6"
+	Version = "1.0.7"
 	Plugin  plugin
 )
 

plugins/src/hashlookup/hashlookup.go

@@ -226,7 +226,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, nil, nil
+			return results, top, debug, nil
 		}
 
 		// Update stats

plugins/src/hashlookup/plugin.go

@@ -9,7 +9,7 @@ import (
  */
 var (
 	Name    = "hashlookup"
-	Version = "1.0.0"
+	Version = "1.0.1"
 	Plugin  plugin
 )
 

plugins/src/http/http.go

@@ -117,7 +117,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		// Update stats

plugins/src/http/plugin.go

@@ -9,7 +9,7 @@ import (
  */
 var (
 	Name    = "http"
-	Version = "1.0.4"
+	Version = "1.0.5"
 	Plugin  plugin
 )
 

plugins/src/ipinfo/ipinfo.go

@@ -138,7 +138,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 			return nil, nil, debug, err
 		}
 
-		return nil, top, debug, nil
+		return results, top, debug, nil
 	}
 
 	// Update stats

plugins/src/ipinfo/plugin.go

@@ -9,7 +9,7 @@ import (
  */
 var (
 	Name    = "ipinfo"
-	Version = "1.0.0"
+	Version = "1.0.1"
 	Plugin  plugin
 )
 

plugins/src/misp/misp.go

@@ -119,7 +119,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		// Update stats

plugins/src/misp/plugin.go

@@ -9,7 +9,7 @@ import (
  */
 var (
 	Name    = "misp"
-	Version = "1.0.0"
+	Version = "1.0.1"
 	Plugin  plugin
 )
 

plugins/src/mongodb/mongodb.go

@@ -214,7 +214,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		// Deserialize

plugins/src/mongodb/plugin.go

@@ -10,7 +10,7 @@ import (
  */
 var (
 	Name    = "mongodb"
-	Version = "1.0.5"
+	Version = "1.0.6"
 	Plugin  plugin
 )
 

plugins/src/mysql/mysql.go

@@ -158,7 +158,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		columns := make([]string, len(cols))

plugins/src/mysql/plugin.go

@@ -11,7 +11,7 @@ import (
  */
 var (
 	Name    = "mysql"
-	Version = "1.0.4"
+	Version = "1.0.5"
 	Plugin  plugin
 )
 

plugins/src/pastelyzer/pastelyzer.go

@@ -123,7 +123,7 @@ func (p *plugin) Search(stmt *sqlparser.Select) ([]map[string]interface{}, map[s
 				return nil, nil, debug, err
 			}
 
-			return nil, top, debug, nil
+			return results, top, debug, nil
 		}
 
 		// Update stats