diff --git a/src/Bus/Alliance.php b/src/Bus/Alliance.php new file mode 100644 index 0000000..6ec5545 --- /dev/null +++ b/src/Bus/Alliance.php @@ -0,0 +1,71 @@ +token = $token; + $this->alliance_id = $alliance_id; + } + + public function fire() + { + Info::withChain([ + new Members($this->alliance_id), + new Labels($this->alliance_id, $this->token), + new Contacts($this->alliance_id, $this->token), + ])->dispatch($this->alliance_id)->delay(now()->addSeconds(rand(120, 600))); + // in order to prevent ESI to receive massive income of all existing SeAT instances in the world + // add a bit of randomize when job can be processed - we use seconds here, so we have more flexibility + // https://github.com/eveseat/seat/issues/731 + } +} diff --git a/src/Bus/Character.php b/src/Bus/Character.php index 9fc212f..2b7ba95 100644 --- a/src/Bus/Character.php +++ b/src/Bus/Character.php @@ -41,11 +41,9 @@ use Seat\Eveapi\Jobs\Clones\Implants; use Seat\Eveapi\Jobs\Contacts\Character\Contacts; use Seat\Eveapi\Jobs\Contacts\Character\Labels as ContactLabels; -use Seat\Eveapi\Jobs\Contracts\Character\Contracts; use Seat\Eveapi\Jobs\Fittings\Character\Fittings; use Seat\Eveapi\Jobs\Industry\Character\Jobs; use Seat\Eveapi\Jobs\Industry\Character\Mining; -use Seat\Eveapi\Jobs\Killmails\Character\Recent; use Seat\Eveapi\Jobs\Location\Character\Location; use Seat\Eveapi\Jobs\Location\Character\Online; use Seat\Eveapi\Jobs\Location\Character\Ship; @@ -111,7 +109,6 @@ public function fire() // collect military informations new Fittings($this->token), - new Recent($this->token), new Fatigue($this->token), new Medals($this->token), @@ -124,7 +121,6 @@ public function fire() // collect financial informations new Orders($this->token), - new Contracts($this->token), new Planets($this->token), new Balance($this->token), new Journal($this->token), @@ -149,6 +145,9 @@ public function fire() new Names($this->token), new Locations($this->token), new CharacterStructures($this->token), - ])->dispatch($this->token->character_id); + ])->dispatch($this->token->character_id)->delay(now()->addSeconds(rand(10, 120))); + // in order to prevent ESI to receive massive income of all existing SeAT instances in the world + // add a bit of randomize when job can be processed - we use seconds here, so we have more flexibility + // https://github.com/eveseat/seat/issues/731 } } diff --git a/src/Bus/Corporation.php b/src/Bus/Corporation.php index be3ade4..b614ba7 100644 --- a/src/Bus/Corporation.php +++ b/src/Bus/Corporation.php @@ -27,7 +27,6 @@ use Seat\Eveapi\Jobs\Assets\Corporation\Names; use Seat\Eveapi\Jobs\Contacts\Corporation\Contacts; use Seat\Eveapi\Jobs\Contacts\Corporation\Labels; -use Seat\Eveapi\Jobs\Contracts\Corporation\Contracts; use Seat\Eveapi\Jobs\Corporation\AllianceHistory; use Seat\Eveapi\Jobs\Corporation\Blueprints; use Seat\Eveapi\Jobs\Corporation\ContainerLogs; @@ -52,7 +51,6 @@ use Seat\Eveapi\Jobs\Industry\Corporation\Mining\Extractions; use Seat\Eveapi\Jobs\Industry\Corporation\Mining\ObserverDetails; use Seat\Eveapi\Jobs\Industry\Corporation\Mining\Observers; -use Seat\Eveapi\Jobs\Killmails\Corporation\Recent; use Seat\Eveapi\Jobs\Market\Corporation\Orders; use Seat\Eveapi\Jobs\PlanetaryInteraction\Corporation\CustomsOfficeLocations; use Seat\Eveapi\Jobs\PlanetaryInteraction\Corporation\CustomsOffices; @@ -116,9 +114,6 @@ public function fire() new Medals($this->corporation_id, $this->token), new IssuedMedals($this->corporation_id, $this->token), - // collect military informations - new Recent($this->corporation_id, $this->token), - // collect industrial informations new Blueprints($this->corporation_id, $this->token), new Facilities($this->corporation_id, $this->token), @@ -128,7 +123,6 @@ public function fire() // collect financial informations new Orders($this->corporation_id, $this->token), - new Contracts($this->corporation_id, $this->token), new Shareholders($this->corporation_id, $this->token), new Balances($this->corporation_id, $this->token), new Journals($this->corporation_id, $this->token), @@ -153,6 +147,9 @@ public function fire() new Locations($this->corporation_id, $this->token), new Names($this->corporation_id, $this->token), new CorporationStructures($this->corporation_id, $this->token), - ])->dispatch($this->corporation_id); + ])->dispatch($this->corporation_id)->delay(now()->addSeconds(rand(120, 300))); + // in order to prevent ESI to receive massive income of all existing SeAT instances in the world + // add a bit of randomize when job can be processed - we use seconds here, so we have more flexibility + // https://github.com/eveseat/seat/issues/731 } } diff --git a/src/Commands/Esi/Update/Alliances.php b/src/Commands/Esi/Update/Alliances.php index 21000c9..f9f4bde 100644 --- a/src/Commands/Esi/Update/Alliances.php +++ b/src/Commands/Esi/Update/Alliances.php @@ -23,11 +23,10 @@ namespace Seat\Console\Commands\Esi\Update; use Illuminate\Console\Command; +use Seat\Console\Bus\Alliance as AllianceBus; use Seat\Eveapi\Jobs\Alliances\Alliances as AlliancesJob; use Seat\Eveapi\Jobs\Alliances\Info; use Seat\Eveapi\Jobs\Alliances\Members; -use Seat\Eveapi\Jobs\Contacts\Alliance\Contacts; -use Seat\Eveapi\Jobs\Contacts\Alliance\Labels; use Seat\Eveapi\Models\Alliances\Alliance; use Seat\Eveapi\Models\RefreshToken; @@ -48,10 +47,40 @@ class Alliances extends Command */ protected $description = 'Schedule update jobs for alliances'; + /** + * @var array + */ + private $alliance_blacklist = []; + /** * Execute the console command. */ public function handle() + { + $this->processTokens(); + $this->processPublics(); + } + + private function processTokens() + { + $tokens = RefreshToken::whereHas('character.affiliation', function ($query) { + $query->whereNotNull('alliance_id'); + })->when($this->argument('alliance_ids'), function ($tokens) { + return $tokens->whereIn('character.affiliation.alliance_id', $this->argument('alliance_ids')); + })->get()->unique('character.affiliation.alliance_id')->each(function ($token) { + + // init a blacklist which will be seed by token loop in order to prevent multiple jobs targetting same alliance + // to be queued + $this->alliance_blacklist[] = $token->character->affiliation->alliance_id; + + // Fire the class to update alliance information + (new AllianceBus($token->character->affiliation->alliance_id, $token))->fire(); + }); + + $this->info('Processed ' . $tokens->count() . ' refresh tokens.'); + } + + private function processPublics() { // collect optional alliance ID from arguments $alliance_ids = $this->argument('alliance_ids') ?: []; @@ -65,25 +94,14 @@ public function handle() // loop over alliances and queue detailed jobs // if we don't have any alliance registered -> queue a global job to collect them if ($alliances->get()->each(function ($alliance) { - Info::dispatch($alliance->alliance_id); - Members::dispatch($alliance->alliance_id); - })->isEmpty() && empty($alliance_ids)) AlliancesJob::dispatch(); - $tokens = RefreshToken::all() - ->when(count($this->argument('alliance_ids')) > 0, function ($tokens) { + // ignore already processed alliances + if (in_array($alliance->alliance_id, $this->alliance_blacklist)) + return true; - return $tokens->whereIn('character.affiliation.alliance_id', $this->argument('alliance_ids')); - }) - ->each(function ($token) { - - // Fire the class to update alliance information - if ($token->character->affiliation->alliance_id != null) { - Contacts::withChain([ - new Labels($token->character->affiliation->alliance_id, $token), - ])->dispatch($token->character->affiliation->alliance_id, $token); - } - }); - - $this->info('Processed ' . $tokens->count() . ' refresh tokens.'); + Info::withChain([ + new Members($alliance->alliance_id), + ])->dispatch($alliance->alliance_id)->delay(now()->addSeconds(rand(20, 300))); + })->isEmpty() && empty($alliance_ids)) AlliancesJob::dispatch(); } } diff --git a/src/Commands/Esi/Update/Contracts.php b/src/Commands/Esi/Update/Contracts.php index a32d4a1..bc1c2bc 100644 --- a/src/Commands/Esi/Update/Contracts.php +++ b/src/Commands/Esi/Update/Contracts.php @@ -74,19 +74,22 @@ public function handle() private function enqueueContractsListJobs() { + // process all tokens character contracts by batch of 100 RefreshToken::chunk(100, function ($tokens) { foreach ($tokens as $token) { - - // enqueue job to list contracts for that character CharacterContracts::dispatch($token); - - // in case this character is also a Director, enqueue job to list contracts for his corporation - if ($token->whereHas('character.corporation_roles', function ($query) { - $query->where('scope', 'roles'); - $query->where('role', 'Director'); - })) CorporationContracts::dispatch($token->character->affiliation->corporation_id, $token); } }); + + // process all tokens corporation contracts with a Director role + RefreshToken::whereHas('character.affiliation', function ($query) { + $query->whereNotNull('corporation_id'); + })->whereHas('character.corporation_roles', function ($query) { + $query->where('scope', 'roles'); + $query->where('role', 'Director'); + })->get()->unique('character.affiliation.corporation_id')->each(function ($token) { + CorporationContracts::dispatch($token->character->affiliation->corporation_id, $token); + }); } /** diff --git a/src/Commands/Esi/Update/Corporations.php b/src/Commands/Esi/Update/Corporations.php index f6c3af0..7bd7b02 100644 --- a/src/Commands/Esi/Update/Corporations.php +++ b/src/Commands/Esi/Update/Corporations.php @@ -39,7 +39,7 @@ class Corporations extends Command * @var string */ protected $signature = 'esi:update:corporations {character_id? : Optional character_id to update ' . - 'corporation information for}'; + 'corporation information for}'; /** * The console command description. @@ -53,18 +53,21 @@ class Corporations extends Command */ public function handle() { + // to prevent excessive calls, we queue only jobs for tokens with Director role. + // more than 80% of corporation endpoints are requiring this role anyway. + // https://github.com/eveseat/seat/issues/731 + $tokens = RefreshToken::whereHas('character.affiliation', function ($query) { + $query->whereNotNull('corporation_id'); + })->whereHas('character.corporation_roles', function ($query) { + $query->where('scope', 'roles'); + $query->where('role', 'Director'); + })->when($this->argument('character_id'), function ($tokens) { + return $tokens->where('character_id', $this->argument('character_id')); + })->get()->unique('character.affiliation.corporation_id')->each(function ($token) { - $tokens = RefreshToken::all() - ->when($this->argument('character_id'), function ($tokens) { - - return $tokens->where('character_id', $this->argument('character_id')); - }) - ->each(function ($token) { - - // Fire the class to update corporation information - if ($token->character->affiliation->corporation_id != null) - (new Corporation($token->character->affiliation->corporation_id, $token))->fire(); - }); + // Fire the class to update corporation information + (new Corporation($token->character->affiliation->corporation_id, $token))->fire(); + }); $this->info('Processed ' . $tokens->count() . ' refresh tokens.'); diff --git a/src/Commands/Esi/Update/Killmails.php b/src/Commands/Esi/Update/Killmails.php index 1873d8c..718526a 100644 --- a/src/Commands/Esi/Update/Killmails.php +++ b/src/Commands/Esi/Update/Killmails.php @@ -65,11 +65,19 @@ public function handle() if ($killmails->get()->each(function ($killmail) { Detail::dispatch($killmail->killmail_id, $killmail->killmail_hash); })->isEmpty() && empty($killmail_ids)) { - RefreshToken::with('character', 'affiliation', 'character.corporation_roles')->get()->each(function ($token) { - RecentCharacterKills::dispatch($token); + RefreshToken::chunk(100, function ($tokens) { + $tokens->each(function ($token) { + RecentCharacterKills::dispatch($token); + }); + }); - if ($token->character->corporation_roles->where('role', 'Director')->isNotEmpty()) - RecentCorporationKills::dispatch($token->character->affiliation->corporation_id, $token); + RefreshToken::whereHas('character.affiliation', function ($query) { + $query->whereNotNull('corporation_id'); + })->whereHas('character.corporation_roles', function ($query) { + $query->where('scope', 'roles'); + $query->where('role', 'Director'); + })->get()->unique('character.affiliation.corporation_id')->each(function ($token) { + RecentCorporationKills::dispatch($token->character->affiliation->corporation_id, $token); }); } } diff --git a/src/Commands/Esi/Update/PublicInfo.php b/src/Commands/Esi/Update/PublicInfo.php index 0d0b50e..d34783f 100644 --- a/src/Commands/Esi/Update/PublicInfo.php +++ b/src/Commands/Esi/Update/PublicInfo.php @@ -64,7 +64,7 @@ class PublicInfo extends Command */ public function handle() { - // NPC stations + // NPC stations using HQ CorporationInfo::whereNotIn('home_station_id', UniverseStation::FAKE_STATION_ID) ->select('home_station_id') ->orderBy('home_station_id') @@ -73,7 +73,7 @@ public function handle() Stations::dispatch($corporations->pluck('home_station_id')->toArray()); }); - // corporation assets + // NPC stations using corporation assets CorporationAsset::where('location_type', 'station') ->select('location_id') ->orderBy('location_id') @@ -90,13 +90,21 @@ public function handle() Insurances::dispatch(); CharacterInfo::doesntHave('refresh_token')->each(function ($character) { - CharacterInfoJob::dispatch($character->character_id); - CorporationHistory::dispatch($character->character_id); + CharacterInfoJob::withChain([ + new CorporationHistory($character->character_id), + ])->dispatch($character->character_id)->delay(rand(10, 120)); + // in order to prevent ESI to receive massive income of all existing SeAT instances in the world + // add a bit of randomize when job can be processed - we use seconds here, so we have more flexibility + // https://github.com/eveseat/seat/issues/731 }); CorporationInfo::all()->each(function ($corporation) { - CorporationInfoJob::dispatch($corporation->corporation_id); - AllianceHistory::dispatch($corporation->corporation_id); + CorporationInfoJob::withChain([ + new AllianceHistory($corporation->corporation_id), + ])->dispatch($corporation->corporation_id)->delay(rand(120, 300)); + // in order to prevent ESI to receive massive income of all existing SeAT instances in the world + // add a bit of randomize when job can be processed - we use seconds here, so we have more flexibility + // https://github.com/eveseat/seat/issues/731 }); } } diff --git a/src/database/seeds/ScheduleSeeder.php b/src/database/seeds/ScheduleSeeder.php index 0bd76c7..674aae4 100644 --- a/src/database/seeds/ScheduleSeeder.php +++ b/src/database/seeds/ScheduleSeeder.php @@ -23,6 +23,7 @@ namespace Seat\Console\database\seeds; use Illuminate\Database\Seeder; +use Illuminate\Support\Arr; use Illuminate\Support\Facades\DB; /** @@ -34,7 +35,7 @@ class ScheduleSeeder extends Seeder /** * @var array */ - protected $schedule = [ + protected $schedules = [ [ // ESI Status | Every Minute 'command' => 'esi:update:status', @@ -68,7 +69,7 @@ class ScheduleSeeder extends Seeder 'ping_before' => null, 'ping_after' => null, ], - [ // EVE Map | Daily at 12am + [ // Public Data | Daily at 12am 'command' => 'esi:update:public', 'expression' => '0 0 * * *', 'allow_overlap' => false, @@ -149,6 +150,9 @@ class ScheduleSeeder extends Seeder */ public function run() { + // add randomness to default schedules + $this->seedRandomize(); + // // drop SeAT 3.x deprecated commands // @@ -156,22 +160,48 @@ public function run() DB::table('schedules')->where('command', 'esi:update:serverstatus')->delete(); DB::table('schedules')->where('command', 'esi:update:esistatus')->delete(); - // - // fix SeAT 4 released schedules - // - DB::table('schedules') - ->where('command', 'esi:update:affiliations') - ->where('expression', '* */2 * * *') - ->update([ - 'expression' => '0 */2 * * *', - ]); - // Check if we have the schedules, else, // insert them - foreach ($this->schedule as $job) { - - if (! DB::table('schedules')->where('command', $job['command'])->first()) + foreach ($this->schedules as $job) { + if (DB::table('schedules')->where('command', $job['command'])->exists()) { + DB::table('schedules')->where('command', $job['command'])->update([ + 'expression' => $job['expression'], + ]); + } else { DB::table('schedules')->insert($job); + } + } + } + + /** + * To prevent massive request wave from all installed instances in the world, + * we add some randomness to seeded schedules. + * + * @see https://github.com/eveseat/seat/issues/731 + */ + private function seedRandomize() + { + // except utc 11 and utc 12 + $hours = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]; + + foreach ($this->schedules as $key => $schedule) { + switch ($schedule['command']) { + // use random minute - every hour + case 'esi:update:characters': + $this->schedules[$key]['expression'] = sprintf('%d * * * *', rand(0, 59)); + break; + // use random minute - every 2 hours + case 'esi:update:affiliations': + case 'esi:update:corporations': + $this->schedules[$key]['expression'] = sprintf('%d */2 * * *', rand(0, 59)); + break; + // use random minute and hour, once a day + case 'esi:update:public': + case 'esi:update:prices': + case 'esi:update:alliances': + $this->schedules[$key]['expression'] = sprintf('%d %d * * *', rand(0, 59), Arr::random($hours)); + break; + } } } }