diff --git a/src/Http/Controllers/Pages/PageController.php b/src/Http/Controllers/Pages/PageController.php index 01273381..8fcffa88 100644 --- a/src/Http/Controllers/Pages/PageController.php +++ b/src/Http/Controllers/Pages/PageController.php @@ -171,15 +171,4 @@ public function searchSubitems(): array { return []; } - - public function solrDate($pageId) - { - $this->pageId = $pageId; - $date = $this->menuItem()->updated_at; - if ($date) { - return $date->toIso8601String(); - } else { - return '2023-01-01T10:00:00Z'; - } - } } diff --git a/src/Models/CmsSearch.php b/src/Models/CmsSearch.php index 7959891e..4d313c0d 100644 --- a/src/Models/CmsSearch.php +++ b/src/Models/CmsSearch.php @@ -40,7 +40,12 @@ class CmsSearch extends BaseModel { protected $table = 'cms_search'; - private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND']; + protected $casts = [ + 'updated_at' => 'datetime', + 'created_at' => 'datetime', + 'deleted_at' => 'datetime', + ]; + // private array $status = ['PENDING', 'ADDED', 'SKIPPED', 'UPDATED', 'NOT_INDEXABLE', 'NOT_FOUND']; private static array $skipStatus = ['NOT_INDEXABLE', 'NOT_FOUND']; diff --git a/src/Models/Indexes/SolrIndex.php b/src/Models/Indexes/SolrIndex.php index 79bac3c7..c962e500 100644 --- a/src/Models/Indexes/SolrIndex.php +++ b/src/Models/Indexes/SolrIndex.php @@ -6,7 +6,6 @@ use NotFound\Framework\Mail\Indexer\FileIndexError; use NotFound\Framework\Mail\Indexer\QueryError; use NotFound\Framework\Models\BaseModel; -use NotFound\Framework\Models\CmsSearch; use NotFound\Framework\Services\Indexer\SearchItem; /** @@ -71,25 +70,20 @@ public function __construct($debug = false) public function emptyCore() { - $searchItems = CmsSearch::all(); - if (count($searchItems) == 0) { - $curl = $this->solrHandler(); - $url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt); - curl_setopt($curl, CURLOPT_URL, $url); - $payload = ['delete' => ['query' => '*:*']]; - - curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload)); - $result = curl_exec($curl); + $curl = $this->solrHandler(); + $url = sprintf('%s/update/?wt=%s&commit=true*', $this->getSolrBaseUrl(), $this->wt); + curl_setopt($curl, CURLOPT_URL, $url); + $payload = ['delete' => ['query' => '*:*']]; - $json = json_decode($result); + curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload)); + $result = curl_exec($curl); - if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) { - $this->mailQueryError($url, $result); + $json = json_decode($result); - return false; - } + if (! $json || ! isset($json->responseHeader) || $json->responseHeader->status !== 0) { + $this->mailQueryError($url, $result); - return true; + return false; } return true; diff --git a/src/Services/Indexer/AbstractIndexService.php b/src/Services/Indexer/AbstractIndexService.php index e76f78c2..258c02b5 100644 --- a/src/Services/Indexer/AbstractIndexService.php +++ b/src/Services/Indexer/AbstractIndexService.php @@ -2,6 +2,7 @@ namespace NotFound\Framework\Services\Indexer; +use DateTime; use NotFound\Framework\Models\CmsSearch; abstract class AbstractIndexService @@ -12,7 +13,11 @@ abstract class AbstractIndexService public ?string $domain; - abstract public function __construct($debug = false); + protected bool $debug; + + protected bool $fresh; + + abstract public function __construct(bool $debug = false, bool $fresh = false); abstract public function startUpdate(): bool; @@ -20,6 +25,8 @@ abstract public function finishUpdate(): object; abstract public function upsertItem(SearchItem $searchItem): object; + abstract public function retainItem(string $url): void; + abstract public function checkConnection(): bool; public function clean(): bool @@ -29,10 +36,10 @@ public function clean(): bool return true; } - public function urlNeedsUpdate(string $url, $updated): bool + public function urlNeedsUpdate(string $url, ?DateTime $updated): bool { $searchItem = CmsSearch::whereUrl($this->siteUrl($url))->first(); - if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at->timestamp > $updated)) { + if ($searchItem && ($searchItem->updated_at !== null && $searchItem->updated_at >= $updated)) { CmsSearch::whereUrl($url)->update(['search_status' => 'SKIPPED']); return false; diff --git a/src/Services/Indexer/IndexBuilderService.php b/src/Services/Indexer/IndexBuilderService.php index 711bdea7..859e1739 100644 --- a/src/Services/Indexer/IndexBuilderService.php +++ b/src/Services/Indexer/IndexBuilderService.php @@ -10,19 +10,17 @@ class IndexBuilderService { - private bool $debug; - - private bool $fresh; - private $locales; private $domain; private $sitemapFile; + private $padding = 140; + private AbstractIndexService $searchServer; - public function __construct($debug = false, $fresh = false) + public function __construct(private bool $debug = false, private bool $fresh = false) { $serverType = config('indexer.engine'); $this->debug = $debug; @@ -32,14 +30,14 @@ public function __construct($debug = false, $fresh = false) $this->domain = rtrim(env('APP_URL', ''), '/'); switch ($serverType) { case 'solr': - $this->searchServer = new SolrIndexService($this->debug); + $this->searchServer = new SolrIndexService($this->debug, $this->fresh); break; default: exit('Unknown search index type'); } } - public function run() + public function run(): void { if (! $this->searchServer->checkConnection()) { $this->writeDebug("\n\n Error connecting to search server! \n\n"); @@ -64,7 +62,7 @@ public function run() $this->sitemapFile = fopen($sitemapFileName, 'w') or exit('Could not open sitemap file for writing'); } else { $this->sitemapFile = false; - $this->writeDebug(" skipping sitemap\n"); + $this->writeDebug(" ⏭️ Skipping sitemap\n"); } $siteId = $site->id; @@ -74,29 +72,32 @@ public function run() $this->searchServer->languageId = 1; // insert all pages, starting from the root - $this->writeDebug(" INDEXING PAGES\n ==============\n"); + $this->writeDebug("┃ INDEXING PAGES\n ==============\n"); $this->indexChildPages($site->root); if ($this->sitemapFile) { fclose($this->sitemapFile); } - $finish = $this->searchServer->finishUpdate(); - - $this->writeDebug($finish->message); } } else { $this->writeDebug("No sites to index\n"); + + return; } + $finish = $this->searchServer->finishUpdate(); + + $this->writeDebug($finish->message); } private function indexChildPages($parentId) { $childPages = Menu::whereParent_id($parentId)->whereEnabled(1)->get(); foreach ($childPages as $page) { - $this->writeDebug(sprintf(" * Page \e[1m%s\e[0m (id: %d)", $page->url, $page->id)); + $this->writeDebug("┃\n"); + $this->writeDebug(sprintf('%s (id: %d)', $page->url, $page->id), true, '┣━┓ 📂 Page '); if (! isset($page->template->id)) { - $this->writeDebug(" skipping, no template found\n"); + $this->writeDebug(": ❌ Fail, skipping, no template found\n"); continue; } @@ -104,11 +105,9 @@ private function indexChildPages($parentId) $menu = Menu::whereId($page->id)->firstOrFail(); if (! isset($page->template->properties->searchable) || $page->template->properties->searchable == 0) { - $this->writeDebug(" skipping, template not searchable\n"); - + $this->writeDebug(": ⏭️ Skipping, template excluded from search\n"); } elseif (isset($page->properties->excludeFromSearch) && $page->properties->excludeFromSearch == true) { - $this->writeDebug(" skipping, page not searchable\n"); - + $this->writeDebug(": ⏭️ Skipping, page excluded from search\n"); } else { foreach ($this->locales as $lang) { @@ -133,7 +132,7 @@ private function updatePage($menu, $lang) $url = $menu->getLocalizedPath(); } - if ($this->searchServer->urlNeedsUpdate($url, strtotime($menu->updated_at))) { + if ($this->searchServer->urlNeedsUpdate($url, $menu->updated_at)) { $this->writeDebug(': update needed: '); $searchText = ''; @@ -149,7 +148,7 @@ private function updatePage($menu, $lang) $c = null; $priority = 1; - $solrDate = ''; + if (class_exists($className)) { $c = new $className(); if (method_exists($className, 'customSearchValues')) { @@ -158,30 +157,31 @@ private function updatePage($menu, $lang) if (method_exists($className, 'searchPriority')) { $priority = $c->searchPriority(); } - if (method_exists($className, 'solrDate')) { - $solrDate = $c->solrDate($menu->id); - } } $searchText = rtrim($searchText, ', '); if (! empty($title) && ! empty($searchText)) { $searchItem = new SearchItem($url, $title); - $searchItem->setContent($searchText)->setLanguage($lang->url)->setPriority($priority)->setPublicationDate(new DateTime($menu->updated_at)); + $searchItem->setContent($searchText) + ->setLanguage($lang->url) + ->setPriority($priority) + ->setPublicationDate(new DateTime($menu->updated_at)); foreach ($customValues as $key => $value) { $searchItem->setCustomValue($key, $value); } $result = $this->searchServer->upsertItem($searchItem); if ($result->errorCode == 0) { - $this->writeDebug(" success\n"); + $this->writeDebug(": ✅ Success\n"); } else { - $this->writeDebug(" FAILED\n"); + $this->writeDebug(": ❌ FAILED\n"); } } else { - $this->writeDebug(" empty page or title\n"); + $this->writeDebug(": ❌ Empty page or title\n"); } } else { - $this->writeDebug(": Does not need updating\n"); + $this->writeDebug(": ✅ Page does not need updating\n"); + $this->searchServer->retainItem($url); } if ($this->sitemapFile) { @@ -235,9 +235,9 @@ private function updateSubitems($class, $lang) $success = false; if ((new \ReflectionClass($searchItem))->getShortName() == 'SearchItem') { $url = $searchItem->url(); - $this->writeDebug($url); + $this->writeDebug($url, true, '┃ ┣━ 📄 '); - if ($this->searchServer->urlNeedsUpdate($url, strtotime($searchItem->lastUpdated()))) { + if ($this->searchServer->urlNeedsUpdate($url, $searchItem->lastUpdated())) { $searchItem->setLanguage($lang->url); $success = $this->searchServer->upsertItem($searchItem); @@ -251,12 +251,13 @@ private function updateSubitems($class, $lang) } if ($success->errorCode == 0) { - $this->writeDebug(" success\n"); + $this->writeDebug(": ✅ Success\n"); } else { $this->writeDebug($success->message); } } else { - $this->writeDebug(": Does not need updating\n"); + $this->writeDebug(": ✅ Item does not need updating\n"); + $success = $this->searchServer->retainItem($url); } } else { dd('Please use the SearchItem class'); @@ -275,10 +276,15 @@ private function createFolderIfNotExists($fullFilePath) } } - private function writeDebug($text) + private function writeDebug($text, $padding = false, $prefix = '') { if ($this->debug) { - printf($text); + + if ($padding) { + $text = substr($text, 0, $this->padding - strlen($prefix)); + $text = str_pad($text, $this->padding - strlen($prefix), ' '); + } + printf("\e[1m".$prefix."\e[0m".$text); } } diff --git a/src/Services/Indexer/SearchItem.php b/src/Services/Indexer/SearchItem.php index 0189a4b0..f83919d3 100644 --- a/src/Services/Indexer/SearchItem.php +++ b/src/Services/Indexer/SearchItem.php @@ -4,6 +4,7 @@ use DateTime; use NotFound\Framework\Models\Lang; +use NotFound\Framework\View\Components\Forms\Fields\Date; final class SearchItem { @@ -155,9 +156,9 @@ public function publicationDate(): ?string * This is when the content was last updated * This is used to determine if the content needs to be re-indexed */ - public function lastUpdated(): ?string + public function lastUpdated(): ?DateTime { - return $this->toDateString($this->lastUpdated ?? $this->publicationDate); + return $this->lastUpdated ?? $this->publicationDate; } public function customValues(): ?array diff --git a/src/Services/Indexer/SolrIndexService.php b/src/Services/Indexer/SolrIndexService.php index 17153bb9..96492def 100644 --- a/src/Services/Indexer/SolrIndexService.php +++ b/src/Services/Indexer/SolrIndexService.php @@ -8,20 +8,28 @@ class SolrIndexService extends AbstractIndexService { - private bool $debug = false; - public int $siteId; public ?string $domain; public $solrIndex; - public function __construct($debug = false) + public function __construct(bool $debug = false, bool $fresh = false) { $this->debug = $debug; + $this->fresh = $fresh; $this->solrIndex = new SolrIndex(); } + public function retainItem(string $url): void + { + $cmsSearchItem = CmsSearch::whereUrl($this->domain.$url)->first(); + if ($cmsSearchItem) { + $cmsSearchItem->search_status = 'UPDATED'; + $cmsSearchItem->save(); + } + } + public function upsertItem(SearchItem $searchItem): object { $return = $this->returnvalue(); @@ -62,11 +70,11 @@ public function upsertItem(SearchItem $searchItem): object $cmsSearchItem = CmsSearch::firstOrNew(['url' => $this->solrIndex->siteUrl($searchItem->url(), $this->domain)]); $cmsSearchItem->setValues($searchItem, $cmsSearchItemStatus); $cmsSearchItem->url = $this->solrIndex->siteUrl($searchItem->url(), $this->domain); - $cmsSearchItem->save(); - if ($cmsSearchItemStatus == 'FAILED') { + + if (in_array($cmsSearchItemStatus, ['NOT_FOUND', 'FAILED'])) { $cmsSearchItem->updated_at = null; - $cmsSearchItem->save(); } + $cmsSearchItem->save(); return $return; } @@ -76,7 +84,10 @@ public function startUpdate(): bool if ($this->debug) { printf("\n ** Starting SOLR update"); } - $emptyResult = $this->solrIndex->emptyCore(); + $emptyResult = true; + if ($this->fresh) { + $emptyResult = $this->solrIndex->emptyCore(); + } CmsSearch::setAllPending(); return $emptyResult; @@ -84,8 +95,13 @@ public function startUpdate(): bool public function finishUpdate(): object { + if ($this->debug) { + printf("\n ** Removing all pending items\n"); + } $return = $this->removeAllPending(); - + if ($this->debug) { + printf("\n ** Rebuilding suggester\n"); + } $build = $this->solrIndex->buildSuggester(); if ($build->error) {