vncviewer: support for back/forward mouse buttons

This commit implements the pseudo-encoding ExtendedMouseButtons which
makes it possible to use the back/forward mouse buttons.

This commit contains work originally done by
PixelSmith <manny33@frontbuffer.com>.

common/rfb/CConnection.cxx

@@ -835,6 +835,7 @@ void CConnection::updateEncodings()
   encodings.push_back(pseudoEncodingContinuousUpdates);
   encodings.push_back(pseudoEncodingFence);
   encodings.push_back(pseudoEncodingQEMUKeyEvent);
+  encodings.push_back(pseudoEncodingExtendedMouseButtons);
 
   if (Decoder::supported(preferredEncoding)) {
     encodings.push_back(preferredEncoding);

common/rfb/CMsgHandler.cxx

@@ -75,6 +75,11 @@ void CMsgHandler::endOfContinuousUpdates()
   server.supportsContinuousUpdates = true;
 }
 
+void CMsgHandler::supportsExtendedMouseButtons()
+{
+  server.supportsExtendedMouseButtons = true;
+}
+
 void CMsgHandler::supportsQEMUKeyEvent()
 {
   server.supportsQEMUKeyEvent = true;

common/rfb/CMsgHandler.h

@@ -57,6 +57,7 @@ namespace rfb {
     virtual void fence(uint32_t flags, unsigned len, const uint8_t data[]);
     virtual void endOfContinuousUpdates();
     virtual void supportsQEMUKeyEvent();
+    virtual void supportsExtendedMouseButtons();
     virtual void serverInit(int width, int height,
                             const PixelFormat& pf,
                             const char* name) = 0;

common/rfb/CMsgReader.cxx

@@ -202,6 +202,10 @@ bool CMsgReader::readMsg()
       handler->supportsQEMUKeyEvent();
       ret = true;
       break;
+    case pseudoEncodingExtendedMouseButtons:
+      handler->supportsExtendedMouseButtons();
+      ret = true;
+      break;
     default:
       ret = readRect(dataRect, rectEncoding);
       break;

common/rfb/CMsgWriter.cxx

@@ -22,6 +22,7 @@
 #endif
 
 #include <stdio.h>
+#include <assert.h>
 
 #include <rdr/OutStream.h>
 #include <rdr/MemOutStream.h>
@@ -173,18 +174,47 @@ void CMsgWriter::writeKeyEvent(uint32_t keysym, uint32_t keycode, bool down)
 }
 
 
-void CMsgWriter::writePointerEvent(const Point& pos, uint8_t buttonMask)
+void CMsgWriter::writePointerEvent(const Point& pos, uint16_t buttonMask)
 {
   Point p(pos);
+  bool extendedMouseButtons;
+
   if (p.x < 0) p.x = 0;
   if (p.y < 0) p.y = 0;
   if (p.x >= server->width()) p.x = server->width() - 1;
   if (p.y >= server->height()) p.y = server->height() - 1;
 
+  /* The highest bit in buttonMask is never sent to the server */
+  assert(!(buttonMask & 0x8000));
+
+  /* Only send extended pointerEvent message when needed */
+  extendedMouseButtons = buttonMask & 0x7f80;
+
   startMsg(msgTypePointerEvent);
-  os->writeU8(buttonMask);
-  os->writeU16(p.x);
-  os->writeU16(p.y);
+  if (server->supportsExtendedMouseButtons && extendedMouseButtons) {
+    int higherBits;
+    int lowerBits;
+
+    higherBits = (buttonMask  >> 7) & 0xff;
+    assert(!(higherBits & 0xfc)); /* Bits 2-7 are reserved */
+
+    lowerBits = buttonMask & 0x7f;
+    lowerBits |= 0x80; /* Set marker bit to 1 */
+
+    os->writeU8(lowerBits);
+    os->writeU16(p.x);
+    os->writeU16(p.y);
+    os->writeU8(higherBits);
+  } else {
+    /* Marker bit must be set to 0, otherwise the server might confuse
+     * the marker bit with the highest bit in a normal PointerEvent
+     * message.
+    */
+    buttonMask &= 0x7f;
+    os->writeU8(buttonMask);
+    os->writeU16(p.x);
+    os->writeU16(p.y);
+  }
   endMsg();
 }
 

common/rfb/CMsgWriter.h

@@ -54,7 +54,7 @@ namespace rfb {
     void writeFence(uint32_t flags, unsigned len, const uint8_t data[]);
 
     void writeKeyEvent(uint32_t keysym, uint32_t keycode, bool down);
-    void writePointerEvent(const Point& pos, uint8_t buttonMask);
+    void writePointerEvent(const Point& pos, uint16_t buttonMask);
 
     void writeClientCutText(const char* str);
 

common/rfb/ServerParams.cxx

@@ -32,7 +32,7 @@ ServerParams::ServerParams()
   : majorVersion(0), minorVersion(0),
     supportsQEMUKeyEvent(false),
     supportsSetDesktopSize(false), supportsFence(false),
-    supportsContinuousUpdates(false),
+    supportsContinuousUpdates(false), supportsExtendedMouseButtons(false),
     width_(0), height_(0),
     ledState_(ledUnknown)
 {

common/rfb/ServerParams.h

@@ -79,6 +79,7 @@ namespace rfb {
     bool supportsSetDesktopSize;
     bool supportsFence;
     bool supportsContinuousUpdates;
+    bool supportsExtendedMouseButtons;
 
   private:
 

tests/unit/emulatemb.cxx

@@ -42,14 +42,14 @@ rfb::BoolParameter emulateMiddleButton("dummy_name", "dummy_desc", true);
 class TestClass : public EmulateMB
 {
 public:
-  void sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask) override;
+  void sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask) override;
 
-  struct PointerEventParams {rfb::Point pos; uint8_t mask; };
+  struct PointerEventParams {rfb::Point pos; uint16_t mask; };
 
   std::vector<PointerEventParams> results;
 };
 
-void TestClass::sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void TestClass::sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   PointerEventParams params;
   params.pos = pos;

vncviewer/EmulateMB.cxx

@@ -199,7 +199,7 @@ EmulateMB::EmulateMB()
 {
 }
 
-void EmulateMB::filterPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void EmulateMB::filterPointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   int btstate;
   int action1, action2;
@@ -280,7 +280,7 @@ void EmulateMB::filterPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
 void EmulateMB::handleTimeout(rfb::Timer *t)
 {
   int action1, action2;
-  uint8_t buttonMask;
+  uint16_t buttonMask;
 
   if (&timer != t)
     return;
@@ -312,7 +312,7 @@ void EmulateMB::handleTimeout(rfb::Timer *t)
   state = stateTab[state][4][2];
 }
 
-void EmulateMB::sendAction(const rfb::Point& pos, uint8_t buttonMask, int action)
+void EmulateMB::sendAction(const rfb::Point& pos, uint16_t buttonMask, int action)
 {
   assert(action != 0);
 
@@ -325,7 +325,7 @@ void EmulateMB::sendAction(const rfb::Point& pos, uint8_t buttonMask, int action
   sendPointerEvent(pos, buttonMask);
 }
 
-int EmulateMB::createButtonMask(uint8_t buttonMask)
+int EmulateMB::createButtonMask(uint16_t buttonMask)
 {
   // Unset left and right buttons in the mask
   buttonMask &= ~0x5;

vncviewer/EmulateMB.h

@@ -26,22 +26,22 @@ class EmulateMB : public rfb::Timer::Callback {
 public:
   EmulateMB();
 
-  void filterPointerEvent(const rfb::Point& pos, uint8_t buttonMask);
+  void filterPointerEvent(const rfb::Point& pos, uint16_t buttonMask);
 
 protected:
-  virtual void sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask)=0;
+  virtual void sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask)=0;
 
   void handleTimeout(rfb::Timer *t) override;
 
 private:
-  void sendAction(const rfb::Point& pos, uint8_t buttonMask, int action);
+  void sendAction(const rfb::Point& pos, uint16_t buttonMask, int action);
 
-  int createButtonMask(uint8_t buttonMask);
+  int createButtonMask(uint16_t buttonMask);
 
 private:
   int state;
-  uint8_t emulatedButtonMask;
-  uint8_t lastButtonMask;
+  uint16_t emulatedButtonMask;
+  uint16_t lastButtonMask;
   rfb::Point lastPos, origPos;
   rfb::Timer timer;
 };

vncviewer/Viewport.cxx

@@ -606,6 +606,20 @@ int Viewport::handle(int event)
     if (Fl::event_button3())
       buttonMask |= 1 << 2;
 
+  // The back/forward buttons are not supported by FTLK 1.3 and require
+  // a patch which adds these buttons to the FLTK API. These buttons
+  // will be part of the upcoming 1.4 API:
+  //   * https://github.com/fltk/fltk/pull/1081
+  //
+  // A backport for branch-1.3 is available here:
+  //   * https://github.com/fltk/fltk/pull/1083
+#if defined(FL_BUTTON4) && defined(FL_BUTTON5)
+    if (Fl::event_button4())
+      buttonMask |= 1 << 7;
+    if (Fl::event_button5())
+      buttonMask |= 1 << 8;
+#endif
+
     if (event == FL_MOUSEWHEEL) {
       wheelMask = 0;
       if (Fl::event_dy() < 0)
@@ -660,7 +674,7 @@ int Viewport::handle(int event)
   return Fl_Widget::handle(event);
 }
 
-void Viewport::sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void Viewport::sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   if (viewOnly)
       return;
@@ -790,7 +804,7 @@ void Viewport::flushPendingClipboard()
 }
 
 
-void Viewport::handlePointerEvent(const rfb::Point& pos, uint8_t buttonMask)
+void Viewport::handlePointerEvent(const rfb::Point& pos, uint16_t buttonMask)
 {
   filterPointerEvent(pos, buttonMask);
 }
@@ -937,6 +951,8 @@ int Viewport::handleSystemEvent(void *event, void *data)
       (msg->message == WM_RBUTTONUP) ||
       (msg->message == WM_MBUTTONDOWN) ||
       (msg->message == WM_MBUTTONUP) ||
+      (msg->message == WM_XBUTTONDOWN) ||
+      (msg->message == WM_XBUTTONUP) ||
       (msg->message == WM_MOUSEWHEEL) ||
       (msg->message == WM_MOUSEHWHEEL)) {
     // We can't get a mouse event in the middle of an AltGr sequence, so

vncviewer/Viewport.h

@@ -70,7 +70,7 @@ class Viewport : public Fl_Widget, public EmulateMB {
   int handle(int event) override;
 
 protected:
-  void sendPointerEvent(const rfb::Point& pos, uint8_t buttonMask) override;
+  void sendPointerEvent(const rfb::Point& pos, uint16_t buttonMask) override;
 
 private:
   bool hasFocus();
@@ -81,7 +81,7 @@ class Viewport : public Fl_Widget, public EmulateMB {
 
   void flushPendingClipboard();
 
-  void handlePointerEvent(const rfb::Point& pos, uint8_t buttonMask);
+  void handlePointerEvent(const rfb::Point& pos, uint16_t buttonMask);
   static void handlePointerTimeout(void *data);
 
   void resetKeyboard();
@@ -111,7 +111,7 @@ class Viewport : public Fl_Widget, public EmulateMB {
   PlatformPixelBuffer* frameBuffer;
 
   rfb::Point lastPointerPos;
-  uint8_t lastButtonMask;
+  uint16_t lastButtonMask;
 
   typedef std::map<int, uint32_t> DownMap;
   DownMap downKeySym;