Merge branch 'develop'

art-institute-of-chicago · Jun 23, 2020 · 3ca7d89 · 3ca7d89
2 parents 3f8b14c + b0416f9
commit 3ca7d89
Show file tree

Hide file tree

Showing 13 changed files with 168 additions and 73 deletions.
diff --git a/.env.example b/.env.example
@@ -32,6 +32,8 @@ ELASTICSEARCH_HOST='localhost'
 ELASTICSEARCH_PORT=9200
 ELASTICSEARCH_SCHEME='http'
 
+ELASTICSEARCH_AWS_REGION=
+
 ELASTICSEARCH_CACHE_ENABLED=false
 ELASTICSEARCH_CACHE_TTL=1800 # 30 min
 ELASTICSEARCH_CACHE_VERSION='v1'

diff --git a/app/Console/Commands/Delete/DeleteAssets.php b/app/Console/Commands/Delete/DeleteAssets.php
@@ -9,7 +9,8 @@ class DeleteAssets extends AbstractImportCommand
 {
 
     protected $signature = 'delete:assets
-                            {--since= : How far back to scan for records}';
+                            {--since= : How far back to scan for records}
+                            {--deep : Loop through the whole database}';
 
     protected $description = 'Delete records that have been removed from the DAMS';
 
@@ -18,13 +19,24 @@ class DeleteAssets extends AbstractImportCommand
      *
      * @var array
      */
-    protected $models = [
+    protected $modelClasses = [
         \App\Models\Collections\Image::class,
         \App\Models\Collections\Sound::class,
         \App\Models\Collections\Text::class,
         \App\Models\Collections\Video::class,
     ];
 
+    public function handle()
+    {
+        $this->api = env('ASSETS_DATA_SERVICE_URL');
+
+        if ($this->option('deep')) {
+            $this->deep();
+        } else {
+            $this->shallow();
+        }
+    }
+
     /**
      * LAKE creates "tombstone" records with `deleted:true` for any record that has been
      * unpublished. These records are minimal, containing only the timestamp, deleted
@@ -33,10 +45,8 @@ class DeleteAssets extends AbstractImportCommand
      * successful run. It'll take the UUID of each record, loop through the `$models`
      * until it finds a match, and delete that record.
      */
-    public function handle()
+    private function shallow()
     {
-        $this->api = env('ASSETS_DATA_SERVICE_URL');
-
         $current = 1;
 
         $json = $this->query('deletes', $current);
@@ -61,14 +71,14 @@ public function handle()
 
                 // Now execute an actual delete
                 // Loop through all model types
-                foreach ($this->models as $model)
+                foreach ($this->modelClasses as $modelClass)
                 {
                     // Check if a resource with a matching lake_guid exists
-                    if ($m = $model::where('lake_guid', '=', $datum->lake_guid)->first())
+                    if ($model = $modelClass::where('lake_guid', '=', $datum->lake_guid)->first())
                     {
                         // If it does, destroy the model and break
-                        $this->warn('Deleting ' . $model . ' ' . $datum->lake_guid);
-                        $m->delete();
+                        $this->warn('Deleting ' . $modelClass . ' ' . $datum->lake_guid);
+                        $model->delete();
                         break;
                     }
                 }
@@ -81,4 +91,46 @@ public function handle()
         }
     }
 
+    /**
+     * Sometimes, assets get deleted in LAKE, but no "tombstone" record is generated in LPM Solr.
+     * This is our work-around for those issues. We will loop through all of the models in our
+     * database that come from LAKE, and delete any that do not exist in LPM Solr.
+     */
+    private function deep()
+    {
+        foreach ($this->modelClasses as $modelClass) {
+            $endpoints = array_filter(config('resources.inbound.assets'), function($value) use ($modelClass) {
+                return $value['model'] === $modelClass;
+            });
+
+            $endpoint = array_keys($endpoints)[0];
+
+            $modelClass::select('lake_guid')->chunk(175, function ($models) use ($endpoint) {
+
+                $url = $this->api . '/' . $endpoint . '?' . http_build_query([
+                    'ids' => implode(',', $models->pluck('lake_guid')->all()),
+                    'limit' => 175,
+                    'flo' => 'id',
+                    'quiet' => 1,
+                ]);
+
+                $result = $this->fetch($url, true);
+
+                $validIds = collect($result->data)->pluck('id');
+
+                if ($validIds->count() === 175) {
+                    return;
+                }
+
+                $invalidModels = $models->filter(function($model) use ($validIds) {
+                    return !$validIds->contains($model->lake_guid);
+                });
+
+                $invalidModels->each(function($model) {
+                    $this->info($model->lake_guid);
+                    $model->delete();
+                });
+            });
+        }
+    }
 }
diff --git a/app/Console/Commands/Import/ImportUlan.php b/app/Console/Commands/Import/ImportUlan.php
@@ -7,58 +7,49 @@
 class ImportUlan extends AbstractImportCommand
 {
 
-    protected $signature = 'import:ulan';
+    protected $signature = 'import:ulan
+                            {--full : Run for all agents}';
 
     protected $description = 'Set the `ulan_uri` for Agents.';
 
     public function handle()
     {
-        // TODO: Set up monthly-ish download of new data set + refresh all agents then
+        $agents = Agent::query();
 
-        // Grab all agents that (1) don't have a URI
-        $agents = Agent::whereNull('ulan_uri');
-
-        // ...and (2) were updated since successful last run
-        $agents = $agents->where('source_modified_at', '>=', $this->command->last_success_at);
+        if (!$this->option('full')) {
+            $agents->whereNull('ulan_uri');
+            $agents->where('source_modified_at', '>=', $this->command->last_success_at);
+        }
 
         // Then, loop through them in a memory-friendly way
-        foreach ($agents->cursor() as $agent)
-        {
-            $this->info('Trying agent #' . $agent->citi_id . ', ' . $agent->title);
-
-            // Query ulan service with birth date
-            $result = $this->fetchUlan($agent, $agent->birth_date, null);
-
-            $gotit = $this->updateUlan($agent, $result, 'with birth year');
+        foreach ($agents->cursor() as $agent) {
+            $this->info('Trying agent #' . $agent->citi_id . ', "' . $agent->title . '"');
 
-            // If the birth date didn't work, try with the death date
-            if (!$gotit && $agent->death_date)
-            {
-                $result = $this->fetchUlan($agent, null, $agent->death_date);
-
-                $gotit = $this->updateUlan($agent, $result, 'with death year');
+            $gotit = false;
 
-                // Now let's try both, just for the record
-                if (!$gotit)
-                {
-                    $result = $this->fetchUlan($agent, $agent->birth_date, $agent->death_date);
+            if (isset($agent->birth_date) && isset($agent->death_date)) {
+                $result = $this->fetchUlan($agent, $agent->birth_date, $agent->death_date);
+                $gotit = $this->updateUlan($agent, $result, 'with birth and death year', 0);
+            }
 
-                    $gotit = $this->updateUlan($agent, $result, 'with birth and death year');
+            if (!$gotit && isset($agent->birth_date)) {
+                $result = $this->fetchUlan($agent, $agent->birth_date, null);
+                $gotit = $this->updateUlan($agent, $result, 'with birth year', 1);
+            }
 
-                    // Now let's try only name
-                    if (!$gotit)
-                    {
-                        $result = $this->fetchUlan($agent, null, null);
+            if (!$gotit && $agent->death_date) {
+                $result = $this->fetchUlan($agent, null, $agent->death_date);
+                $gotit = $this->updateUlan($agent, $result, 'with death year', 1);
+            }
 
-                        $gotit = $this->updateUlan($agent, $result, 'with no years');
-                    }
-                }
+            if (!$gotit) {
+                $result = $this->fetchUlan($agent, null, null);
+                $gotit = $this->updateUlan($agent, $result, 'with no years', 2);
             }
 
-            // If there are no results, try with just the last name or first word
-            if (count($result->results) === 0)
-            {
-                continue;
+            if (!$gotit) {
+                $result = $this->fetchUlan($agent, null, null);
+                $gotit = $this->updateUlan($agent, $result, 'with no years', 3, true);
             }
         }
 
@@ -67,37 +58,43 @@ public function handle()
     private function fetchUlan($agent, $birth_date = 0, $death_date = 0)
     {
         $url = env('ULAN_DATA_SERVICE_URL') .
-            '?q=' . urlencode($agent->title) .
+            '?q=' . urlencode($agent->sort_title ?? $agent->title) .
             ($birth_date ? '&by=' . $birth_date : '') .
             ($death_date ? '&dy=' . $death_date : '');
 
         return $this->fetch($url, true);
     }
 
-    private function updateUlan($agent, $result, $message = '')
+    private function updateUlan($agent, $result, $message = '', $certainty = null, $permissive = false)
     {
         // If there's only one result, set the ULAN URI
         if (count($result->results) === 1) {
             $this->info('... exact name matched ' . $message . ' ' . $result->results[0]->uri);
             $agent->ulan_uri = $result->results[0]->uri;
+            $agent->ulan_certainty = $certainty;
             $agent->save();
             return true;
         }
 
         // If there's more than one result, try to find an exact match
         if (count($result->results) > 1) {
             // Make a distinct list of IDs, because the service sometimes returns dups
-            $uris = [];
+            $uris = array_unique(array_map(function($item) {
+                return $item->uri;
+            }, $result->results));
 
-            foreach ($result->results as $res) {
-                $uris[] = $res->uri;
+            if (count($uris) === 1) {
+                $this->info('... exact name matched distinct results ' . $message . ' ' . $uris[0]);
+                $agent->ulan_uri = $uris[0];
+                $agent->ulan_certainty = $certainty;
+                $agent->save();
+                return true;
             }
 
-            $uris = array_unique($uris);
-
-            if (count($uris) === 1) {
-                $this->info('... exact name matched distinct results' . $message);
+            if (count($uris) > 1 && $permissive) {
+                $this->info('... exact name matched non-distinct results ' . $message . ' ' . $uris[0]);
                 $agent->ulan_uri = $uris[0];
+                $agent->ulan_certainty = $certainty;
                 $agent->save();
                 return true;
             }

diff --git a/app/Console/Commands/Report/ReportUlan.php b/app/Console/Commands/Report/ReportUlan.php
@@ -21,18 +21,20 @@ public function handle()
         $csv->insertOne([
             'id',
             'ulan_uri',
+            'ulan_certainty',
             'title',
             'birth_date',
             'death_date',
         ]);
 
-        $artists = \App\Models\Collections\Agent::whereNotNull('ulan_uri');
+        $artists = \App\Models\Collections\Agent::whereNotNull('ulan_uri')->orderBy('ulan_certainty','asc')->orderBy('sort_title', 'asc');
 
         foreach ($artists->cursor() as $artist) {
             $row = [
                 'id' => $artist->citi_id,
                 'ulan_uri' => $artist->ulan_uri,
-                'title' => $artist->title,
+                'ulan_certainty' => $artist->ulan_certainty,
+                'title' => $artist->sort_title ?? $artist->title,
                 'birth_date' => $artist->birth_date,
                 'death_date' => $artist->death_date,
             ];

diff --git a/app/Models/Collections/Agent.php b/app/Models/Collections/Agent.php
@@ -39,6 +39,11 @@ public function createdArtworks()
         return $this->belongsToMany('App\Models\Collections\Artwork', 'artwork_artist')->artworks();
     }
 
+    public function createdArtworkIds()
+    {
+        return $this->belongsToMany('App\Models\Collections\Artwork', 'artwork_artist')->pluck('artwork_citi_id');
+    }
+
     public function sites()
     {
         return $this->belongsToMany('App\Models\StaticArchive\Site', 'agent_site', 'agent_citi_id');

diff --git a/app/Models/Collections/Exhibition.php b/app/Models/Collections/Exhibition.php
@@ -22,6 +22,10 @@ class Exhibition extends CollectionsModel
         'date_aic_end' => 'datetime',
     ];
 
+    protected $with = [
+        'webExhibition',
+    ];
+
     public function artworks()
     {
         return $this->belongsToMany('App\Models\Collections\Artwork')->artworks();

diff --git a/app/Scopes/PublishedScope.php b/app/Scopes/PublishedScope.php
@@ -20,31 +20,32 @@ class PublishedScope implements Scope
      */
     public function apply(Builder $builder, Model $model)
     {
+        $columns = collect(Schema::getColumnListing($model->getTable()));
         $builder
-            ->when(Schema::hasColumn($model->getTable(), 'published'), function ($query) {
+            ->when($columns->contains('published'), function ($query) {
                 return $query->where('published', '=', true);
             })
-            ->when(Schema::hasColumn($model->getTable(), 'is_published'), function ($query) {
+            ->when($columns->contains('is_published'), function ($query) {
                 return $query->where('is_published', '=', true);
             })
             // Logic borrows from area17/twill->/src/Models/Model->scopeVisible
-            ->when(Schema::hasColumn($model->getTable(), 'publish_start_date'), function ($query) {
+            ->when($columns->contains('publish_start_date'), function ($query) {
                 return $query->where(function ($query2) {
                     return $query2->where('publish_start_date', '<=', Carbon::now())
                         ->orWhereNull('publish_start_date');
                 });
             })
-            ->when(Schema::hasColumn($model->getTable(), 'publish_end_date'), function ($query) {
+            ->when($columns->contains('publish_end_date'), function ($query) {
                 return $query->where(function ($query2) {
                     $query2->where('publish_end_date', '>=', Carbon::now())
                         ->orWhereNull('publish_end_date');
                 });
             })
             // Account of other field names
-            ->when(Schema::hasColumn($model->getTable(), 'is_private'), function ($query) {
+            ->when($columns->contains('is_private'), function ($query) {
                 return $query->where('is_private', '=', false);
             })
-            ->when(Schema::hasColumn($model->getTable(), 'active'), function ($query) {
+            ->when($columns->contains('active'), function ($query) {
                 return $query->where('active', '=', true);
             });
     }

diff --git a/app/Transformers/Outbound/Collections/Agent.php b/app/Transformers/Outbound/Collections/Agent.php
@@ -155,7 +155,7 @@ protected function getFields()
                 'type' => 'array',
                 'elasticsearch' => 'integer',
                 'value' => function ($item) {
-                    return $item->createdArtworks->pluck('citi_id');
+                    return $item->createdArtworkIds();
                 },
             ],
             'site_ids' => [

diff --git a/config/aic.php b/config/aic.php
@@ -12,7 +12,7 @@
     | or any other location as required by the application or its packages.
     */
 
-    'version' => '1.0-rc11',
+    'version' => '1.0-rc12',
 
     /*
     |--------------------------------------------------------------------------

diff --git a/config/elasticsearch.php b/config/elasticsearch.php
@@ -129,7 +129,7 @@
              * @link https://github.com/jeskew/amazon-es-php
              */
 
-            'httpHandler' => new Aws\ElasticsearchService\ElasticsearchPhpHandler('us-east-2'),
+            'httpHandler' => new Aws\ElasticsearchService\ElasticsearchPhpHandler(env('ELASTICSEARCH_AWS_REGION', 'us-east-1')),
 
             /**
              * Connection Pool