From 05f593ba79b5536b51b74c66f90e7b2fca285304 Mon Sep 17 00:00:00 2001 From: Sergio Giraldo Date: Mon, 31 Jul 2023 20:52:55 +0200 Subject: [PATCH 1/5] front end to have full-text search:: by sergio giraldo @ 20230731T2052CEST, gpg signed --- stuff/links/lang/en_US/search.php | 1 + .../views/app/search/search.blade.php | 23 +++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/stuff/links/lang/en_US/search.php b/stuff/links/lang/en_US/search.php index 5b30960b..1206c07b 100644 --- a/stuff/links/lang/en_US/search.php +++ b/stuff/links/lang/en_US/search.php @@ -8,6 +8,7 @@ return [ 'query' => 'Search for...', 'search_title' => 'Search Title', 'search_description' => 'Search Description', + 'search_fulltext' => 'Search Full-Text', 'private_only' => 'Private Links only', 'broken_links' => 'Broken Links only', 'empty_tags' => 'without Tags', diff --git a/stuff/links/resources/views/app/search/search.blade.php b/stuff/links/resources/views/app/search/search.blade.php index 10a33171..27163332 100644 --- a/stuff/links/resources/views/app/search/search.blade.php +++ b/stuff/links/resources/views/app/search/search.blade.php @@ -49,12 +49,16 @@
- -
+
+ +
+  
@@ -90,6 +94,17 @@ + +
+
+ + +
+
+
-- 2.39.2 (Apple Git-143) From 8ee3d60575cd05ceb663ab704fa1fc38efa65d9c Mon Sep 17 00:00:00 2001 From: Sergio Giraldo Date: Mon, 31 Jul 2023 21:32:31 +0200 Subject: [PATCH 2/5] now it is passing the fulltext parameter to the search engine:: by sergio giraldo @ 20230731T2132CEST, gpg signed --- .../Http/Controllers/App/SearchController.php | 2 ++ .../Http/Controllers/Traits/SearchesLinks.php | 6 ++++++ .../links/app/Providers/AppServiceProvider.php | 18 +++++++++--------- .../views/app/search/search.blade.php | 3 ++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/stuff/links/app/Http/Controllers/App/SearchController.php b/stuff/links/app/Http/Controllers/App/SearchController.php index 86c84059..603f3766 100644 --- a/stuff/links/app/Http/Controllers/App/SearchController.php +++ b/stuff/links/app/Http/Controllers/App/SearchController.php @@ -25,6 +25,7 @@ class SearchController extends Controller 'old_query' => null, 'search_title' => true, 'search_description' => true, + 'search_fulltext' => false, 'private_only' => false, 'broken_only' => false, 'empty_tags' => false, @@ -54,6 +55,7 @@ class SearchController extends Controller 'old_query' => $this->searchQuery, 'search_title' => $this->searchTitle, 'search_description' => $this->searchDescription, + 'search_fulltext' => $this->searchFullText, 'private_only' => $this->searchPrivateOnly, 'broken_only' => $this->searchBrokenOnly, 'only_lists' => $this->searchLists, diff --git a/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php b/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php index 62549e7d..9d6766fc 100644 --- a/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php +++ b/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php @@ -11,6 +11,7 @@ trait SearchesLinks protected $searchQuery; protected $searchTitle; protected $searchDescription; + protected $searchFullText; protected $searchPrivateOnly; protected $searchBrokenOnly; protected $searchLists; @@ -59,6 +60,11 @@ trait SearchesLinks }); } + // Search in the source of the page + if ($this->searchFullText = $request->input('search_fulltext', false)) { + $search->where('check_content', '>', -1); + } + // Show private only if applicable if ($this->searchPrivateOnly = $request->input('private_only', false)) { $search->where('is_private', true); diff --git a/stuff/links/app/Providers/AppServiceProvider.php b/stuff/links/app/Providers/AppServiceProvider.php index fb2aac19..dcace975 100644 --- a/stuff/links/app/Providers/AppServiceProvider.php +++ b/stuff/links/app/Providers/AppServiceProvider.php @@ -21,15 +21,15 @@ class AppServiceProvider extends ServiceProvider Paginator::useBootstrap(); //Log sql statements - // DB::listen(function($query) { - // Log::info( - // $query->sql, - // [ - // 'bindings' => $query->bindings, - // 'time' => $query->time - // ] - // ); - // }); + DB::listen(function($query) { + Log::info( + $query->sql, + [ + 'bindings' => $query->bindings, + 'time' => $query->time + ] + ); + }); } /** diff --git a/stuff/links/resources/views/app/search/search.blade.php b/stuff/links/resources/views/app/search/search.blade.php index 27163332..178d1916 100644 --- a/stuff/links/resources/views/app/search/search.blade.php +++ b/stuff/links/resources/views/app/search/search.blade.php @@ -50,7 +50,8 @@
+ class="form-check-input" + @if($query_settings['search_fulltext']) checked @endif> -- 2.39.2 (Apple Git-143) From 95d60f9013abf93d761b621994f8f82b435af206 Mon Sep 17 00:00:00 2001 From: Sergio Giraldo Date: Mon, 31 Jul 2023 21:44:05 +0200 Subject: [PATCH 3/5] what is the query i need to execute:: by sergio giraldo @ 20230731T2144CEST, gpg signed --- stuff/links/app/Http/Controllers/Traits/SearchesLinks.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php b/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php index 9d6766fc..a7a9e8be 100644 --- a/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php +++ b/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php @@ -63,6 +63,11 @@ trait SearchesLinks // Search in the source of the page if ($this->searchFullText = $request->input('search_fulltext', false)) { $search->where('check_content', '>', -1); + + //TODO + // SELECT * FROM `links` + // LEFT JOIN `link_text` on links.id = link_text.link_id + // WHERE (title like '%luigi%' or content like '%luigi%') } // Show private only if applicable -- 2.39.2 (Apple Git-143) From 22a73989c0151213373aa458593d2db9010343e2 Mon Sep 17 00:00:00 2001 From: Sergio Giraldo Date: Tue, 1 Aug 2023 12:19:51 +0200 Subject: [PATCH 4/5] ignoring pages:: by sergio giraldo @ 20230801T1219CEST, gpg signed --- stuff/links/import/page-source.php | 73 +++++++++++++++++++----------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/stuff/links/import/page-source.php b/stuff/links/import/page-source.php index 6e11f1df..553338eb 100644 --- a/stuff/links/import/page-source.php +++ b/stuff/links/import/page-source.php @@ -25,9 +25,9 @@ while ($row = $result->fetch_assoc()) { $url = $row["url"]; echo $linkId; - echo "
"; + echo "\n"; echo $url; - echo "
"; + echo "\n"; $checkQuery = "SELECT lt.link_id FROM link_text lt inner join links l on l.id = lt.link_id and l.url = '$url'"; $checkResult = $conn->query($checkQuery); @@ -35,61 +35,82 @@ while ($row = $result->fetch_assoc()) { if ($checkResult) { if ($checkResult->num_rows == 0) { $htmlSource = downloadHtmlSource($url); - if ($htmlSource != "err"){ + if ($htmlSource != "n/a"){ $stmt = $conn->prepare("INSERT INTO link_text (link_id, content) VALUES (?, ?)"); $stmt->bind_param("is", $linkId, $htmlSource); $stmt->execute(); - echo "Saved
\n"; + echo "Saved\n"; $processed += 1; } else{ - echo "Ignored, err
\n"; + echo "Ignored\n"; } } else { - echo "Skipping
\n"; + echo "Skipped\n"; } } else { - echo "Error: " . $checkQuery . "

\n" . $conn->error; + echo "Error: " . $checkQuery . "\n" . $conn->error; } $updateQuery = "UPDATE links SET check_content = 1 WHERE id = '$linkId'"; $conn->query($updateQuery); - echo "Checked
\n"; + echo "Updated\n\n"; } $conn->close(); -// Function to download HTML source code +function filterUrls($url) { + $urlsToIgnore = array("bing", "google", "duckduck", "localhost", "127.0.0.1", "hostnet", "internettoday", "workday", "intranet", "gmail", "outlook", "onedrive", "amysql"); + foreach ($urlsToIgnore as $item) { + $pattern = "/" . $item . "/i"; + if (preg_match($pattern, $url)) { + return true; + } + } + return (!filter_var($url, FILTER_VALIDATE_URL)); + } + function downloadHtmlSource($url) { $text = ""; - if (filter_var($url, FILTER_VALIDATE_URL) === FALSE) { - return "err"; + if (filterUrls($url)) { + return "n/a"; } - else{ - try{ - // Get the HTML content from the URL - $html = file_get_contents($url); - - // Create a DOM object from the HTML - $dom = new DOMDocument(); - @$dom->loadHTML($html); - // Get the body element - $body = $dom->getElementsByTagName('body')->item(0); + try{ + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + $text = curl_exec($ch); + $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); - // Get the text content of the body - $text = $body->textContent; - } - catch(Throwable $e){ - $text = "err"; + if ($httpCode == 401 || $httpCode == 403) { + $text = "n/a"; } + + // if ($html = @file_get_contents($url)){ + // $dom = new DOMDocument(); + // @$dom->loadHTML($html); + + // $body = $dom->getElementsByTagName('body')->item(0); + + // $text = $body->textContent; + // } + // else{ + // $text = "n/a"; + // } + } + catch(Throwable $e){ + $text = "n/a"; } return $text; } ?> + + -- 2.39.2 (Apple Git-143) From 8e15d3f473d1435dd3e0bfd34de0bc3b4e04fc72 Mon Sep 17 00:00:00 2001 From: Sergio Giraldo Date: Fri, 4 Aug 2023 14:43:49 +0200 Subject: [PATCH 5/5] searching full-text:: by sergio giraldo @ 20230804T1443CEST, gpg signed --- .../Http/Controllers/Traits/SearchesLinks.php | 18 +++++++----------- stuff/links/import/page-source.php | 5 ++++- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php b/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php index a7a9e8be..89a631ca 100644 --- a/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php +++ b/stuff/links/app/Http/Controllers/Traits/SearchesLinks.php @@ -40,7 +40,8 @@ trait SearchesLinks protected function buildDatabaseQuery(SearchRequest $request): Builder { // Start building the search - $search = Link::byUser($request->user()->id)->with(['tags']); + $search = Link::leftJoin('link_text', 'links.id', '=', 'link_text.link_id')->with(['tags']); + $search->where('user_id','=',$request->user()->id); // Search for the URL if ($this->searchQuery = $request->input('query', false)) { @@ -57,17 +58,12 @@ trait SearchesLinks if ($this->searchDescription = $request->input('search_description', false)) { $search->orWhere('description', 'like', $query); } - }); - } - - // Search in the source of the page - if ($this->searchFullText = $request->input('search_fulltext', false)) { - $search->where('check_content', '>', -1); - //TODO - // SELECT * FROM `links` - // LEFT JOIN `link_text` on links.id = link_text.link_id - // WHERE (title like '%luigi%' or content like '%luigi%') + // Search in the source of the page + if ($this->searchFullText = $request->input('search_fulltext', false)) { + $search->orWhere('content', 'like', $query); + } + }); } // Show private only if applicable diff --git a/stuff/links/import/page-source.php b/stuff/links/import/page-source.php index 553338eb..913dfc05 100644 --- a/stuff/links/import/page-source.php +++ b/stuff/links/import/page-source.php @@ -1,11 +1,14 @@ connect_error) { -- 2.39.2 (Apple Git-143)