"Fix" APCU cache isses by checking for them in status controller

src/Controller/StatusController.php

@@ -2,10 +2,18 @@
 
 namespace BBC\CliftonBundle\Controller;
 
+use BBC\ProgrammesPagesService\Domain\ValueObject\Pid;
 use Symfony\Bundle\FrameworkBundle\Controller\Controller;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
 use DateTime;
+use PDOException;
+use Exception;
+use Doctrine\DBAL\ConnectionException as ConnectionExceptionDBAL;
+use Doctrine\DBAL\Exception\ConnectionException;
+use Doctrine\DBAL\Exception\DriverException;
+use Symfony\Component\Debug\Exception\ContextErrorException;
+use Doctrine\DBAL\DBALException;
 
 /**
  * Class StatusController
@@ -21,7 +29,11 @@ class StatusController extends Controller
     public function statusAction(Request $request)
     {
         // If the load balancer is pinging us then give them a plain OK
+        $dbCacheIsOk = $this->verifyNoDatabaseCacheIssues();
         if ($request->headers->get('User-Agent') == 'ELB-HealthChecker/1.0') {
+            if (!$dbCacheIsOk) {
+                return new Response('ERROR', Response::HTTP_INTERNAL_SERVER_ERROR, ['content-type' => 'text/plain']);
+            }
             return new Response('OK', Response::HTTP_OK, ['content-type' => 'text/plain']);
         }
 
@@ -31,6 +43,62 @@ public function statusAction(Request $request)
         return $this->render('@Clifton/Status/status.html.twig', [
             'now' => new DateTime(),
             'dbConnectivity' => $dbalConnection->isConnected() || $dbalConnection->connect(),
+            'dbCacheIsOk' => $dbCacheIsOk,
         ]);
     }
+
+    /**
+     * This profanity is copy/pasted from Faucet. It detects whether an Exception indicates that the
+     * database is down. No, there is no single exception for that.
+     * When the database is down we return a 200 status. Other DB exceptions return a 500 status.
+     * See programmes ticket https://jira.dev.bbc.co.uk/browse/PROGRAMMES-5534
+     *
+     * Please remove this kludge once the underlying issues in APCU are fixed
+     *
+     * returns TRUE if there are no issues or if the database is down
+     * returns FALSE if there are non-connection related database issues (e.g. APCU problems)
+     */
+    private function verifyNoDatabaseCacheIssues(): bool
+    {
+        try {
+            $pid = new Pid('b006m86d'); //Eastenders
+            $this->get('pps.programmes_service')->findByPidFull($pid);
+        } catch (ConnectionExceptionDBAL | ConnectionException $e) {
+            return true;
+        } catch (PDOException $e) {
+            if ($e->getCode() === 0 && stristr($e->getMessage(), 'There is no active transaction')) {
+                // I am aware of how horrible this is. PDOExcetion is very generic. The only
+                // way I can see to be specific to the case of "DB server went down"
+                // is to do a string compare on the error message.
+                return true;
+            } elseif ($e->getCode() == 2002) {
+                // Connection timeout
+                return true;
+            }
+            return false;
+        } catch (DriverException $e) {
+            if ($e->getErrorCode() == 1213 || $e->getErrorCode() == 1205) {
+                // This is thrown on a MySQL deadlock error 1213 or 1205 lock wait timeout. We catch it
+                // and exit with a zero exit status allowing the processor
+                // to restart
+                return true;
+            } elseif ($e->getErrorCode() == 2006) {
+                // General error: 2006 MySQL server has gone away
+                return true;
+            }
+            return false;
+        } catch (DBALException | ContextErrorException $e) {
+            $msg = $e->getMessage();
+            if ($e->getCode() === 0 &&
+                (stristr($msg, 'server has gone away') || stristr($msg, 'There is no active transaction.'))
+            ) {
+                // This is what happens when the SQL server goes away while the process is active
+                return true;
+            }
+            return false;
+        } catch (Exception $e) {
+            return false;
+        }
+        return true;
+    }
 }

src/Resources/views/Status/status.html.twig

@@ -12,5 +12,6 @@
         <h1>Status at {{now|date('c')}}</h1>
 
         <p data-test-name="db-connectivity">Database Connectivity: <span class="{{dbConnectivity ? 'pass' : 'fail'}}">{{ dbConnectivity ? "YES" : "NO" }}</span></p>
+        <p data-test-name="db-error">Database Cache Error Check: <span class="{{dbCacheIsOk ? 'pass' : 'fail'}}">{{ dbCacheIsOk ? "PASS" : "FAIL" }}</span></p>
     </body>
 </html>

tests/Controller/StatusControllerTest.php

@@ -2,6 +2,10 @@
 
 namespace Tests\BBC\CliftonBundle\Controller;
 
+use BBC\ProgrammesPagesService\Domain\ValueObject\Pid;
+use BBC\ProgrammesPagesService\Service\ProgrammesService;
+use Doctrine\DBAL\ConnectionException;
+use Doctrine\DBAL\DBALException;
 use Tests\BBC\CliftonBundle\BaseWebTestCase;
 
 class StatusControllerTest extends BaseWebTestCase
@@ -20,9 +24,52 @@ public function testStatusFromElb()
         $client = static::createClient([], [
             'HTTP_USER_AGENT' => 'ELB-HealthChecker/1.0',
         ]);
-        $crawler = $client->request('GET', '/status');
+        $client->request('GET', '/status');
+
+        $this->assertResponseStatusCode($client, 200);
+        $this->assertEquals('OK', $client->getResponse()->getContent());
+    }
+
+    public function testNonConnectionDBErrorFromElb()
+    {
+        $client = static::createClient([], [
+            'HTTP_USER_AGENT' => 'ELB-HealthChecker/1.0',
+        ]);
+        // Create mock service to throw exception and inject into container
+        $mockService = $this->createMockProgrammesService();
+        $mockService->expects($this->once())
+            ->method('findByPidFull')
+            ->with(new Pid('b006m86d'))
+            ->willThrowException(new DBALException("Something bad happened."));
+        $client->getContainer()->set('pps.programmes_service', $mockService);
+
+        $client->request('GET', '/status');
+
+        $this->assertResponseStatusCode($client, 500);
+        $this->assertEquals('ERROR', $client->getResponse()->getContent());
+    }
+
+    public function testConnectionDBErrorFromElb()
+    {
+        $client = static::createClient([], [
+            'HTTP_USER_AGENT' => 'ELB-HealthChecker/1.0',
+        ]);
+        // Create mock service to throw exception and inject into container
+        $mockService = $this->createMockProgrammesService();
+        $mockService->expects($this->once())
+            ->method('findByPidFull')
+            ->with(new Pid('b006m86d'))
+            ->willThrowException(new ConnectionException("Cannot Connect."));
+        $client->getContainer()->set('pps.programmes_service', $mockService);
+
+        $client->request('GET', '/status');
 
         $this->assertResponseStatusCode($client, 200);
         $this->assertEquals('OK', $client->getResponse()->getContent());
     }
+
+    private function createMockProgrammesService()
+    {
+        return $this->createMock(ProgrammesService::class);
+    }
 }