From acdc47ca041140e7af0bc297f9696e473e702a2a Mon Sep 17 00:00:00 2001 From: Bjorn Twachtmann Date: Tue, 2 May 2017 17:21:01 +0100 Subject: [PATCH] "Fix" APCU cache isses by checking for them in status controller --- src/Controller/StatusController.php | 68 +++++++++++++++++++++ src/Resources/views/Status/status.html.twig | 1 + tests/Controller/StatusControllerTest.php | 49 ++++++++++++++- 3 files changed, 117 insertions(+), 1 deletion(-) diff --git a/src/Controller/StatusController.php b/src/Controller/StatusController.php index 0e604fe..30a651a 100644 --- a/src/Controller/StatusController.php +++ b/src/Controller/StatusController.php @@ -2,10 +2,18 @@ namespace BBC\CliftonBundle\Controller; +use BBC\ProgrammesPagesService\Domain\ValueObject\Pid; use Symfony\Bundle\FrameworkBundle\Controller\Controller; use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use DateTime; +use PDOException; +use Exception; +use Doctrine\DBAL\ConnectionException as ConnectionExceptionDBAL; +use Doctrine\DBAL\Exception\ConnectionException; +use Doctrine\DBAL\Exception\DriverException; +use Symfony\Component\Debug\Exception\ContextErrorException; +use Doctrine\DBAL\DBALException; /** * Class StatusController @@ -21,7 +29,11 @@ class StatusController extends Controller public function statusAction(Request $request) { // If the load balancer is pinging us then give them a plain OK + $dbCacheIsOk = $this->verifyNoDatabaseCacheIssues(); if ($request->headers->get('User-Agent') == 'ELB-HealthChecker/1.0') { + if (!$dbCacheIsOk) { + return new Response('ERROR', Response::HTTP_INTERNAL_SERVER_ERROR, ['content-type' => 'text/plain']); + } return new Response('OK', Response::HTTP_OK, ['content-type' => 'text/plain']); } @@ -31,6 +43,62 @@ public function statusAction(Request $request) return $this->render('@Clifton/Status/status.html.twig', [ 'now' => new DateTime(), 'dbConnectivity' => $dbalConnection->isConnected() || $dbalConnection->connect(), + 'dbCacheIsOk' => $dbCacheIsOk, ]); } + + /** + * This profanity is copy/pasted from Faucet. It detects whether an Exception indicates that the + * database is down. No, there is no single exception for that. + * When the database is down we return a 200 status. Other DB exceptions return a 500 status. + * See programmes ticket https://jira.dev.bbc.co.uk/browse/PROGRAMMES-5534 + * + * Please remove this kludge once the underlying issues in APCU are fixed + * + * returns TRUE if there are no issues or if the database is down + * returns FALSE if there are non-connection related database issues (e.g. APCU problems) + */ + private function verifyNoDatabaseCacheIssues(): bool + { + try { + $pid = new Pid('b006m86d'); //Eastenders + $this->get('pps.programmes_service')->findByPidFull($pid); + } catch (ConnectionExceptionDBAL | ConnectionException $e) { + return true; + } catch (PDOException $e) { + if ($e->getCode() === 0 && stristr($e->getMessage(), 'There is no active transaction')) { + // I am aware of how horrible this is. PDOExcetion is very generic. The only + // way I can see to be specific to the case of "DB server went down" + // is to do a string compare on the error message. + return true; + } elseif ($e->getCode() == 2002) { + // Connection timeout + return true; + } + return false; + } catch (DriverException $e) { + if ($e->getErrorCode() == 1213 || $e->getErrorCode() == 1205) { + // This is thrown on a MySQL deadlock error 1213 or 1205 lock wait timeout. We catch it + // and exit with a zero exit status allowing the processor + // to restart + return true; + } elseif ($e->getErrorCode() == 2006) { + // General error: 2006 MySQL server has gone away + return true; + } + return false; + } catch (DBALException | ContextErrorException $e) { + $msg = $e->getMessage(); + if ($e->getCode() === 0 && + (stristr($msg, 'server has gone away') || stristr($msg, 'There is no active transaction.')) + ) { + // This is what happens when the SQL server goes away while the process is active + return true; + } + return false; + } catch (Exception $e) { + return false; + } + return true; + } } diff --git a/src/Resources/views/Status/status.html.twig b/src/Resources/views/Status/status.html.twig index fead554..d9fcf20 100644 --- a/src/Resources/views/Status/status.html.twig +++ b/src/Resources/views/Status/status.html.twig @@ -12,5 +12,6 @@

Status at {{now|date('c')}}

Database Connectivity: {{ dbConnectivity ? "YES" : "NO" }}

+

Database Cache Error Check: {{ dbCacheIsOk ? "PASS" : "FAIL" }}

diff --git a/tests/Controller/StatusControllerTest.php b/tests/Controller/StatusControllerTest.php index eefc7a6..6a2caa0 100644 --- a/tests/Controller/StatusControllerTest.php +++ b/tests/Controller/StatusControllerTest.php @@ -2,6 +2,10 @@ namespace Tests\BBC\CliftonBundle\Controller; +use BBC\ProgrammesPagesService\Domain\ValueObject\Pid; +use BBC\ProgrammesPagesService\Service\ProgrammesService; +use Doctrine\DBAL\ConnectionException; +use Doctrine\DBAL\DBALException; use Tests\BBC\CliftonBundle\BaseWebTestCase; class StatusControllerTest extends BaseWebTestCase @@ -20,9 +24,52 @@ public function testStatusFromElb() $client = static::createClient([], [ 'HTTP_USER_AGENT' => 'ELB-HealthChecker/1.0', ]); - $crawler = $client->request('GET', '/status'); + $client->request('GET', '/status'); + + $this->assertResponseStatusCode($client, 200); + $this->assertEquals('OK', $client->getResponse()->getContent()); + } + + public function testNonConnectionDBErrorFromElb() + { + $client = static::createClient([], [ + 'HTTP_USER_AGENT' => 'ELB-HealthChecker/1.0', + ]); + // Create mock service to throw exception and inject into container + $mockService = $this->createMockProgrammesService(); + $mockService->expects($this->once()) + ->method('findByPidFull') + ->with(new Pid('b006m86d')) + ->willThrowException(new DBALException("Something bad happened.")); + $client->getContainer()->set('pps.programmes_service', $mockService); + + $client->request('GET', '/status'); + + $this->assertResponseStatusCode($client, 500); + $this->assertEquals('ERROR', $client->getResponse()->getContent()); + } + + public function testConnectionDBErrorFromElb() + { + $client = static::createClient([], [ + 'HTTP_USER_AGENT' => 'ELB-HealthChecker/1.0', + ]); + // Create mock service to throw exception and inject into container + $mockService = $this->createMockProgrammesService(); + $mockService->expects($this->once()) + ->method('findByPidFull') + ->with(new Pid('b006m86d')) + ->willThrowException(new ConnectionException("Cannot Connect.")); + $client->getContainer()->set('pps.programmes_service', $mockService); + + $client->request('GET', '/status'); $this->assertResponseStatusCode($client, 200); $this->assertEquals('OK', $client->getResponse()->getContent()); } + + private function createMockProgrammesService() + { + return $this->createMock(ProgrammesService::class); + } }