feat!:Version 0.21.0 (#234)

* fix: solr accents (#232) * fix: remove double encoding * fix: exclude from search * fix(search): retain skipped items * fix(solr): re-index failed document * style: formatting * feat: add function to abstract class. * chore: remove SolrDate * feat: empty core * fix: syntax problem * feat: DateTime for SearchItem * style: formatting
NotFoundNL · May 24, 2024 · 3beb762 · 3beb762
1 parent 3fc8ed5
commit 3beb762
Show file tree

Hide file tree

Showing 7 changed files with 93 additions and 75 deletions.
diff --git a/src/Http/Controllers/Pages/PageController.php b/src/Http/Controllers/Pages/PageController.php
@@ -171,15 +171,4 @@ public function searchSubitems(): array
     {
         return [];
     }
-
-    public function solrDate($pageId)
-    {
-        $this->pageId = $pageId;
-        $date = $this->menuItem()->updated_at;
-        if ($date) {
-            return $date->toIso8601String();
-        } else {
-            return '2023-01-01T10:00:00Z';
-        }
-    }
 }
diff --git a/src/Models/CmsSearch.php b/src/Models/CmsSearch.php
@@ -40,7 +40,12 @@ class CmsSearch extends BaseModel
 {
     protected $table = 'cms_search';
 
-    private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND'];
+    protected $casts = [
+        'updated_at' => 'datetime',
+        'created_at' => 'datetime',
+        'deleted_at' => 'datetime',
+    ];
+    //    private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND'];
 
     private static array $skipStatus = ['NOT_INDEXABLE', 'NOT_FOUND'];
 

diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php
@@ -6,7 +6,6 @@
 use NotFound\Framework\Mail\Indexer\FileIndexError;
 use NotFound\Framework\Mail\Indexer\QueryError;
 use NotFound\Framework\Models\BaseModel;
-use NotFound\Framework\Models\CmsSearch;
 use NotFound\Framework\Services\Indexer\SearchItem;
 
 /**
@@ -71,25 +70,20 @@ public function __construct($debug = false)
 
     public function emptyCore()
     {
-        $searchItems = CmsSearch::all();
-        if (count($searchItems) == 0) {
-            $curl = $this->solrHandler();
-            $url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt);
-            curl_setopt($curl, CURLOPT_URL, $url);
-            $payload = ['delete' => ['query' => '*:*']];
-
-            curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload));
-            $result = curl_exec($curl);
+        $curl = $this->solrHandler();
+        $url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt);
+        curl_setopt($curl, CURLOPT_URL, $url);
+        $payload = ['delete' => ['query' => '*:*']];
 
-            $json = json_decode($result);
+        curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload));
+        $result = curl_exec($curl);
 
-            if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) {
-                $this->mailQueryError($url, $result);
+        $json = json_decode($result);
 
-                return false;
-            }
+        if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) {
+            $this->mailQueryError($url, $result);
 
-            return true;
+            return false;
         }
 
         return true;

diff --git a/src/Services/Indexer/AbstractIndexService.php b/src/Services/Indexer/AbstractIndexService.php
@@ -2,6 +2,7 @@
 
 namespace NotFound\Framework\Services\Indexer;
 
+use DateTime;
 use NotFound\Framework\Models\CmsSearch;
 
 abstract class AbstractIndexService
@@ -12,14 +13,20 @@ abstract class AbstractIndexService
 
     public ?string $domain;
 
-    abstract public function __construct($debug = false);
+    protected bool $debug;
+
+    protected bool $fresh;
+
+    abstract public function __construct(bool $debug = false, bool $fresh = false);
 
     abstract public function startUpdate(): bool;
 
     abstract public function finishUpdate(): object;
 
     abstract public function upsertItem(SearchItem $searchItem): object;
 
+    abstract public function retainItem(string $url): void;
+
     abstract public function checkConnection(): bool;
 
     public function clean(): bool
@@ -29,10 +36,10 @@ public function clean(): bool
         return true;
     }
 
-    public function urlNeedsUpdate(string $url, $updated): bool
+    public function urlNeedsUpdate(string $url, ?DateTime $updated): bool
     {
         $searchItem = CmsSearch::whereUrl($this->siteUrl($url))->first();
-        if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at->timestamp > $updated)) {
+        if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at >= $updated)) {
             CmsSearch::whereUrl($url)->update(['search_status' => 'SKIPPED']);
 
             return false;

diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php
@@ -10,19 +10,17 @@
 
 class IndexBuilderService
 {
-    private bool $debug;
-
-    private bool $fresh;
-
     private $locales;
 
     private $domain;
 
     private $sitemapFile;
 
+    private $padding = 140;
+
     private AbstractIndexService $searchServer;
 
-    public function __construct($debug = false, $fresh = false)
+    public function __construct(private bool $debug = false, private bool $fresh = false)
     {
         $serverType = config('indexer.engine');
         $this->debug = $debug;
@@ -32,14 +30,14 @@ public function __construct($debug = false, $fresh = false)
         $this->domain = rtrim(env('APP_URL', ''), '/');
         switch ($serverType) {
             case 'solr':
-                $this->searchServer = new SolrIndexService($this->debug);
+                $this->searchServer = new SolrIndexService($this->debug, $this->fresh);
                 break;
             default:
                 exit('Unknown search index type');
         }
     }
 
-    public function run()
+    public function run(): void
     {
         if (! $this->searchServer->checkConnection()) {
             $this->writeDebug("\n\n Error connecting to search server! \n\n");
@@ -64,7 +62,7 @@ public function run()
                     $this->sitemapFile = fopen($sitemapFileName, 'w') or exit('Could not open sitemap file for writing');
                 } else {
                     $this->sitemapFile = false;
-                    $this->writeDebug("   skipping sitemap\n");
+                    $this->writeDebug(" ⏭️ Skipping sitemap\n");
                 }
 
                 $siteId = $site->id;
@@ -74,41 +72,42 @@ public function run()
                 $this->searchServer->languageId = 1;
 
                 // insert all pages, starting from the root
-                $this->writeDebug("   INDEXING PAGES\n   ==============\n");
+                $this->writeDebug("┃ INDEXING PAGES\n   ==============\n");
                 $this->indexChildPages($site->root);
 
                 if ($this->sitemapFile) {
                     fclose($this->sitemapFile);
                 }
-                $finish = $this->searchServer->finishUpdate();
-
-                $this->writeDebug($finish->message);
             }
         } else {
             $this->writeDebug("No sites to index\n");
+
+            return;
         }
+        $finish = $this->searchServer->finishUpdate();
+
+        $this->writeDebug($finish->message);
     }
 
     private function indexChildPages($parentId)
     {
         $childPages = Menu::whereParent_id($parentId)->whereEnabled(1)->get();
         foreach ($childPages as $page) {
-            $this->writeDebug(sprintf("    * Page \e[1m%s\e[0m (id: %d)", $page->url, $page->id));
+            $this->writeDebug("┃\n");
+            $this->writeDebug(sprintf('%s (id: %d)', $page->url, $page->id), true, '┣━┓  📂 Page ');
 
             if (! isset($page->template->id)) {
-                $this->writeDebug("   skipping, no template found\n");
+                $this->writeDebug(": ❌ Fail, skipping, no template found\n");
 
                 continue;
             }
 
             $menu = Menu::whereId($page->id)->firstOrFail();
 
             if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) {
-                $this->writeDebug("   skipping, template not searchable\n");
-
+                $this->writeDebug(": ⏭️ Skipping, template excluded from search\n");
             } elseif (isset($page->properties->excludeFromSearch) && $page->properties->excludeFromSearch == true) {
-                $this->writeDebug("  skipping, page not searchable\n");
-
+                $this->writeDebug(": ⏭️  Skipping, page excluded from search\n");
             } else {
 
                 foreach ($this->locales as $lang) {
@@ -133,7 +132,7 @@ private function updatePage($menu, $lang)
             $url = $menu->getLocalizedPath();
         }
 
-        if ($this->searchServer->urlNeedsUpdate($url, strtotime($menu->updated_at))) {
+        if ($this->searchServer->urlNeedsUpdate($url, $menu->updated_at)) {
             $this->writeDebug(': update needed: ');
 
             $searchText = '';
@@ -149,7 +148,7 @@ private function updatePage($menu, $lang)
 
             $c = null;
             $priority = 1;
-            $solrDate = '';
+
             if (class_exists($className)) {
                 $c = new $className();
                 if (method_exists($className, 'customSearchValues')) {
@@ -158,30 +157,31 @@ private function updatePage($menu, $lang)
                 if (method_exists($className, 'searchPriority')) {
                     $priority = $c->searchPriority();
                 }
-                if (method_exists($className, 'solrDate')) {
-                    $solrDate = $c->solrDate($menu->id);
-                }
             }
             $searchText = rtrim($searchText, ', ');
             if (! empty($title) && ! empty($searchText)) {
 
                 $searchItem = new SearchItem($url, $title);
-                $searchItem->setContent($searchText)->setLanguage($lang->url)->setPriority($priority)->setPublicationDate(new DateTime($menu->updated_at));
+                $searchItem->setContent($searchText)
+                    ->setLanguage($lang->url)
+                    ->setPriority($priority)
+                    ->setPublicationDate(new DateTime($menu->updated_at));
                 foreach ($customValues as $key => $value) {
                     $searchItem->setCustomValue($key, $value);
                 }
                 $result = $this->searchServer->upsertItem($searchItem);
 
                 if ($result->errorCode == 0) {
-                    $this->writeDebug(" success\n");
+                    $this->writeDebug(": ✅ Success\n");
                 } else {
-                    $this->writeDebug(" FAILED\n");
+                    $this->writeDebug(": ❌ FAILED\n");
                 }
             } else {
-                $this->writeDebug(" empty page or title\n");
+                $this->writeDebug(": ❌ Empty page or title\n");
             }
         } else {
-            $this->writeDebug(": Does not need updating\n");
+            $this->writeDebug(": ✅ Page does not need updating\n");
+            $this->searchServer->retainItem($url);
         }
 
         if ($this->sitemapFile) {
@@ -235,9 +235,9 @@ private function updateSubitems($class, $lang)
                 $success = false;
                 if ((new \ReflectionClass($searchItem))->getShortName() == 'SearchItem') {
                     $url = $searchItem->url();
-                    $this->writeDebug($url);
+                    $this->writeDebug($url, true, '┃ ┣━ 📄 ');
 
-                    if ($this->searchServer->urlNeedsUpdate($url, strtotime($searchItem->lastUpdated()))) {
+                    if ($this->searchServer->urlNeedsUpdate($url, $searchItem->lastUpdated())) {
 
                         $searchItem->setLanguage($lang->url);
                         $success = $this->searchServer->upsertItem($searchItem);
@@ -251,12 +251,13 @@ private function updateSubitems($class, $lang)
                         }
 
                         if ($success->errorCode == 0) {
-                            $this->writeDebug(" success\n");
+                            $this->writeDebug(": ✅ Success\n");
                         } else {
                             $this->writeDebug($success->message);
                         }
                     } else {
-                        $this->writeDebug(": Does not need updating\n");
+                        $this->writeDebug(": ✅ Item does not need updating\n");
+                        $success = $this->searchServer->retainItem($url);
                     }
                 } else {
                     dd('Please use the SearchItem class');
@@ -275,10 +276,15 @@ private function createFolderIfNotExists($fullFilePath)
         }
     }
 
-    private function writeDebug($text)
+    private function writeDebug($text, $padding = false, $prefix = '')
     {
         if ($this->debug) {
-            printf($text);
+
+            if ($padding) {
+                $text = substr($text, 0, $this->padding - strlen($prefix));
+                $text = str_pad($text, $this->padding - strlen($prefix), ' ');
+            }
+            printf("\e[1m".$prefix."\e[0m".$text);
         }
     }
 

diff --git a/src/Services/Indexer/SearchItem.php b/src/Services/Indexer/SearchItem.php
@@ -4,6 +4,7 @@
 
 use DateTime;
 use NotFound\Framework\Models\Lang;
+use NotFound\Framework\View\Components\Forms\Fields\Date;
 
 final class SearchItem
 {
@@ -155,9 +156,9 @@ public function publicationDate(): ?string
      * This is when the content was last updated
      * This is used to determine if the content needs to be re-indexed
      */
-    public function lastUpdated(): ?string
+    public function lastUpdated(): ?DateTime
     {
-        return $this->toDateString($this->lastUpdated ?? $this->publicationDate);
+        return $this->lastUpdated ?? $this->publicationDate;
     }
 
     public function customValues(): ?array