Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
nikhiltri committed Sep 15, 2020
2 parents 0e93b91 + 8acb85e commit f25c76c
Show file tree
Hide file tree
Showing 304 changed files with 800 additions and 1,284 deletions.
28 changes: 3 additions & 25 deletions .env.testing.example
Original file line number Diff line number Diff line change
@@ -1,35 +1,13 @@
APP_NAME='Data Aggregator'
APP_NAME='Art Institute of Chicago API'
APP_KEY=
APP_DEBUG=true
APP_LOG_LEVEL='debug'
APP_URL='http://data-aggregator.test'
APP_URL='http://api.test'

# Also defined in phpunit.xml?
APP_ENV='testing'
BCRYPT_ROUNDS=4
CACHE_DRIVER='array'
MAIL_DRIVER='array'
QUEUE_CONNECTION='sync'
SESSION_DRIVER='array'

# WEB-347: Use MySQL instead of SQLite for testing
DB_CONNECTION='sqlite'
DB_DATABASE=':memory:'

# DB_CONNECTION='mysql'
# DB_HOST='127.0.0.1'
# DB_PORT=3306
# DB_DATABASE='data_aggregator_test'
# DB_USERNAME='homestead'
# DB_PASSWORD='secret'

DEBUGBAR_ENABLED=false
DB_CONNECTION=testing

SCOUT_DRIVER=null

PRIMO_API_SOURCE='FOOBAR'

# WEB-1189: Write tests for authentication behavior
APP_RESTRICTED=false

PRODUCTION_URL=
2 changes: 0 additions & 2 deletions .env.travis
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,5 @@ APP_LOG_LEVEL=debug
APP_URL=http://localhost

DB_CONNECTION=testing
DB_TEST_USERNAME=root
DB_TEST_PASSWORD=

SCOUT_DRIVER=null
5 changes: 1 addition & 4 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,16 @@ matrix:
php:
- 7.2

services:
- mysql

before_script:
- cp .env.travis .env
- sudo mysql -e 'create database testing;'
- composer self-update
- composer install --no-interaction
- php artisan key:generate
- php artisan migrate

script:
- vendor/bin/phpunit
- vendor/bin/phpcs --warning-severity=6

- language: node_js
node_js:
Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,17 @@ You are in the right place! The aggregator contains all of our public APIs, whic

For example, here's an endpoint that lists all of our published artworks:

https://aggregator-data.artic.edu/api/v1/artworks
https://api.artic.edu/api/v1/artworks

...and here's a query that shows identifiers, titles, and last modified dates for all artworks that have been updated in our collections system in the past seven days from this moment, sorted in reverse chronological order:

https://aggregator-data.artic.edu/api/v1/artworks/search?fields=id,title,last_updated_source&query[range][last_updated_source][gte]=now-7d&sort[last_updated_source][order]=desc
https://api.artic.edu/api/v1/artworks/search?fields=id,title,last_updated_source&query[range][last_updated_source][gte]=now-7d&sort[last_updated_source][order]=desc

Our API is a wrapper around [Elasticsearch's Query DSL](https://www.elastic.co/guide/en/elasticsearch/reference/6.0/query-dsl.html). Depending on your needs, these queries can get quite complex.

Here are some resources to get you started:

* [Art Institute of Chicago — API Documentation](https://aggregator-data.artic.edu/home) (fields and endpoints)
* [Art Institute of Chicago — API Documentation](https://api.artic.edu/docs) (fields and endpoints)
* [Elasticsearch 6.0 — Query DSL](https://www.elastic.co/guide/en/elasticsearch/reference/6.0/query-dsl.html) (query syntax)
* [Art Institute of Chicago — Open Access — Public API](https://www.artic.edu/open-access/public-api) (example projects)

Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.0-rc16
1.0
17 changes: 4 additions & 13 deletions app/Behaviors/ImportsData.php
Original file line number Diff line number Diff line change
Expand Up @@ -177,13 +177,10 @@ protected function import($source, $model, $endpoint, $current = 1)
$json = $this->query($endpoint, $current);

if ($this->isTest) {

$pages = 1;

$this->warn('Testing import of a single page for model ' . $model);

} else {

// Assumes the dataservice has standardized pagination
try {
$pages = $json->pagination->total_pages;
Expand All @@ -196,25 +193,20 @@ protected function import($source, $model, $endpoint, $current = 1)
// Ensure dataservice can be reached before doing this!

$this->warn('Found ' . $pages . ' page(s) for model ' . $model);

}

while ($current <= $pages)
{
while ($current <= $pages) {
$this->warn('Importing ' . $current . ' of ' . $pages . ' for model ' . $model);

// Assumes the dataservice wraps its results in a `data` field
foreach ($json->data as $datum)
{
foreach ($json->data as $datum) {
// TODO: Careful, this conflicts w/ partial imports – running on one endpoint counts for all!
// Break if this is a partial import + this datum is older than last run
if ($this->isPartial && isset($datum->{$model::$sourceLastUpdateDateField}))
{
if ($this->isPartial && isset($datum->{$model::$sourceLastUpdateDateField})) {
$sourceTime = new Carbon($datum->{$model::$sourceLastUpdateDateField});
$sourceTime->timezone = config('app.timezone');

if ($this->since->gt($sourceTime))
{
if ($this->since->gt($sourceTime)) {
break 2;
}
}
Expand Down Expand Up @@ -315,5 +307,4 @@ protected function updateSentryTags($datum = null, $endpoint = null, $source = n
});
}
}

}
1 change: 0 additions & 1 deletion app/BelongsToManyOrOne.php
Original file line number Diff line number Diff line change
Expand Up @@ -68,5 +68,4 @@ public function isAlternative()
{
return $this->wherePivot('preferred', '=', false)->expectMany();
}

}
1 change: 0 additions & 1 deletion app/Command.php
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,4 @@ public static function never()
{
return Carbon::createFromTimestampUTC(1);
}

}
1 change: 0 additions & 1 deletion app/Console/Commands/BaseCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -68,5 +68,4 @@ protected function getModels()
})
->collapse();
}

}
1 change: 0 additions & 1 deletion app/Console/Commands/Bulk/BulkAll.php
Original file line number Diff line number Diff line change
Expand Up @@ -97,5 +97,4 @@ public function handle()
$command->save();
}
}

}
4 changes: 1 addition & 3 deletions app/Console/Commands/Bulk/BulkImport.php
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ public function handle()

$bar = $this->output->createProgressBar($total);

for ($currentPage = 1; $currentPage <= $totalPages; $currentPage++)
{
for ($currentPage = 1; $currentPage <= $totalPages; $currentPage++) {
$json = $this->query($source, $endpoint, $currentPage, $this->chunkSize, $ids);

$data = collect($json->data)->map(function ($datum) use ($transformer, $table) {
Expand Down Expand Up @@ -177,5 +176,4 @@ protected function fetch($file, &$headers = null)

return $contents;
}

}
1 change: 0 additions & 1 deletion app/Console/Commands/CleanSeed.php
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,4 @@ public function handle()
{
\DatabaseSeeder::clean();
}

}
18 changes: 7 additions & 11 deletions app/Console/Commands/Delete/DeleteAssets.php
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,11 @@ private function shallow()

$this->warn('Found ' . $pages . ' pages');

while ($current <= $pages)
{
while ($current <= $pages) {
$this->warn('Deleting ' . $current . ' of ' . $pages);

// Assumes the dataservice wraps its results in a `data` field
foreach ($json->data as $datum)
{
foreach ($json->data as $datum) {
// Break if we're past the last time we checked
$sourceTime = new Carbon($datum->indexed_at);
$sourceTime->timezone = config('app.timezone');
Expand All @@ -71,11 +69,9 @@ private function shallow()

// Now execute an actual delete
// Loop through all model types
foreach ($this->modelClasses as $modelClass)
{
foreach ($this->modelClasses as $modelClass) {
// Check if a resource with a matching lake_guid exists
if ($model = $modelClass::where('lake_guid', '=', $datum->lake_guid)->first())
{
if ($model = $modelClass::where('lake_guid', '=', $datum->lake_guid)->first()) {
// If it does, destroy the model and break
$this->warn('Deleting ' . $modelClass . ' ' . $datum->lake_guid);
$model->delete();
Expand All @@ -99,7 +95,7 @@ private function shallow()
private function deep()
{
foreach ($this->modelClasses as $modelClass) {
$endpoints = array_filter(config('resources.inbound.assets'), function($value) use ($modelClass) {
$endpoints = array_filter(config('resources.inbound.assets'), function ($value) use ($modelClass) {
return $value['model'] === $modelClass;
});

Expand All @@ -122,11 +118,11 @@ private function deep()
return;
}

$invalidModels = $models->filter(function($model) use ($validIds) {
$invalidModels = $models->filter(function ($model) use ($validIds) {
return !$validIds->contains($model->lake_guid);
});

$invalidModels->each(function($model) {
$invalidModels->each(function ($model) {
$this->info($model->lake_guid);
$model->delete();
});
Expand Down
7 changes: 2 additions & 5 deletions app/Console/Commands/Delete/DeleteCollections.php
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,10 @@ private function deleteByEndpoint()
$total = $json->pagination->total;
$totalPages = ceil($total / $this->chunkSize);

for ($currentPage = 1; $currentPage <= $totalPages; $currentPage++)
{
for ($currentPage = 1; $currentPage <= $totalPages; $currentPage++) {
$json = $this->query('deletes', $currentPage, $this->chunkSize);

foreach ($json->data as $datum)
{
foreach ($json->data as $datum) {
try {
$resource = app('Resources')->getResourceForInboundEndpoint($datum->type, 'collections');
$modelClass = $resource['model'];
Expand Down Expand Up @@ -119,5 +117,4 @@ private function getId($id, $type)
return $id;
}
}

}
1 change: 0 additions & 1 deletion app/Console/Commands/Docs/AbstractDocCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,4 @@ abstract class AbstractDocCommand extends BaseCommand
{

private $models;

}
13 changes: 6 additions & 7 deletions app/Console/Commands/Docs/CreateEndpointDocs.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,19 @@ public function handle()
{
$doc = "## Endpoints\n\n";

foreach ($this->getCategories() as $namespace => $heading)
{
foreach ($this->getCategories() as $namespace => $heading) {
$doc .= "### ${heading}\n\n";

foreach ($this->getModelsForNamespace($namespace) as $model)
{
foreach ($this->getModelsForNamespace($namespace) as $model) {
$doc .= $model::instance()->docEndpoints();
}
}

Storage::disk('local')->put('ENDPOINTS.md', $doc);

copy(storage_path('app/ENDPOINTS.md'),
base_path('docs/.sections/endpoints.md'));
copy(
storage_path('app/ENDPOINTS.md'),
base_path('docs/.sections/endpoints.md')
);
}

}
14 changes: 6 additions & 8 deletions app/Console/Commands/Docs/CreateFieldsDocs.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,19 @@ public function handle()
{
$doc = "## Fields\n\n";

foreach ($this->getCategories() as $namespace => $heading)
{
foreach ($this->getCategories() as $namespace => $heading) {
$doc .= "### ${heading}\n\n";

foreach ($this->getModelsForNamespace($namespace) as $model)
{
foreach ($this->getModelsForNamespace($namespace) as $model) {
$doc .= $model::instance()->docFields();
}
}

Storage::disk('local')->put('FIELDS.md', $doc);

copy(storage_path('app/FIELDS.md'),
base_path('docs/.sections/fields.md'));

copy(
storage_path('app/FIELDS.md'),
base_path('docs/.sections/fields.md')
);
}

}
19 changes: 6 additions & 13 deletions app/Console/Commands/Docs/CreateSwaggerDoc.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@ class CreateSwaggerDoc extends AbstractDocCommand

public function handle()
{
if ($this->argument('appUrl'))
{
if ($this->argument('appUrl')) {
$this->appUrl = $this->argument('appUrl');
}

Expand Down Expand Up @@ -50,10 +49,8 @@ public function handle()
/**
* Endpoints
*/
foreach ($this->getCategories() as $namespace => $heading)
{
foreach ($this->getModelsForNamespace($namespace) as $model)
{
foreach ($this->getCategories() as $namespace => $heading) {
foreach ($this->getModelsForNamespace($namespace) as $model) {
$doc .= $model::instance()->swaggerEndpoints($this->appUrl);
}
}
Expand Down Expand Up @@ -99,10 +96,8 @@ public function handle()
$doc .= " }\n";
$doc .= " },\n";

foreach ($this->getCategories() as $namespace => $heading)
{
foreach ($this->getModelsForNamespace($namespace) as $model)
{
foreach ($this->getCategories() as $namespace => $heading) {
foreach ($this->getModelsForNamespace($namespace) as $model) {
$doc .= $model::instance()->swaggerFields($this->appUrl);
}
}
Expand Down Expand Up @@ -159,8 +154,7 @@ public function handle()
$instance->docSearchParametersRaw()
);

foreach ($params as $param => $description)
{
foreach ($params as $param => $description) {
$doc .= ' "' . $param . "\": {\n";
$doc .= ' "name": "' . $param . "\",\n";
$doc .= " \"in\": \"query\",\n";
Expand All @@ -180,5 +174,4 @@ public function handle()

Storage::disk('local')->put('swagger.json', $doc);
}

}
Loading

0 comments on commit f25c76c

Please sign in to comment.