Peer service

docs/configuration.rst

@@ -65,6 +65,12 @@ REQUEST_VALIDATION
     they conform to the protocol. This may result in clients with poor standards
     compliance receiving errors rather than the expected results.
 
+INITIAL_PEERS
+    When starting, you can set a list of initial peers to contact using a
+    simple text file. Add a URL per line for peers you would like to add to
+    the registry as initial peers. Each time the server starts, this set of
+    peers will be announced and added to the registry.
+
 LANDING_MESSAGE_HTML
     The server provides a simple landing page at its root. By setting this
     value to point at a file containing an HTML block element it is possible to

docs/datarepo.rst

@@ -72,6 +72,46 @@ understand the structure of the repository that they are managing.
 
     $ ga4gh_repo list registry.db
 
+------------------
+list-announcements
+------------------
+
+To view the announcements the server has received, use this command. It will
+output the announcements as a TSV that can be imported into a spreadsheet
+application.
+
+.. argparse::
+   :module: ga4gh.server.cli.repomanager
+   :func: getRepoManagerParser
+   :prog: ga4gh_repo
+   :path: list-announcements
+   :nodefault:
+
+**Examples:**
+
+.. code-block:: bash
+
+    $ ga4gh_repo list-announcements registry.db > announcements.tsv
+
+-------------------
+clear-announcements
+-------------------
+
+To clear the received announcements run this command. This cannot be undone.
+
+.. argparse::
+   :module: ga4gh.server.cli.repomanager
+   :func: getRepoManagerParser
+   :prog: ga4gh_repo
+   :path: clear-announcements
+   :nodefault:
+
+**Examples:**
+
+.. code-block:: bash
+
+    $ ga4gh_repo clear-announcements registry.db
+
 
 ------
 verify
@@ -102,6 +142,44 @@ well in their repository.
 
     $ ga4gh_repo verify registry.db
 
+--------
+add-peer
+--------
+
+The server maintains a list of known peers. To add a peer to this list use
+the ``add-peer`` command.
+
+.. argparse::
+   :module: ga4gh.server.cli.repomanager
+   :func: getRepoManagerParser
+   :prog: ga4gh_repo
+   :path: add-peer
+   :nodefault:
+
+**Examples:**
+
+.. code-block:: bash
+
+    $ ga4gh_repo add-peer http://1kgenomes.ga4gh.org
+
+-----------
+remove-peer
+-----------
+
+You can remove a peer from the list of peers by its URL.
+
+.. argparse::
+   :module: ga4gh.server.cli.repomanager
+   :func: getRepoManagerParser
+   :prog: ga4gh_repo
+   :path: remove-peer
+   :nodefault:
+
+**Examples:**
+
+.. code-block:: bash
+
+    $ ga4gh_repo remove-peer http://1kgenomes.ga4gh.org
 
 -----------
 add-dataset

ga4gh/server/backend.py

@@ -571,6 +571,15 @@ def expressionLevelsGenerator(self, request):
             request, rnaQuant)
         return iterator
 
+    def peersGenerator(self, request):
+        """
+        Returns a generator over the (peer, nextPageToken) pairs
+        defined by the specified request.
+        """
+        return paging.PeerIterator(
+            request,
+            self.getDataRepository())
+
     ###########################################################
     #
     # Public API methods. Each of these methods implements the
@@ -676,6 +685,65 @@ def runGetCallSet(self, id_):
         callSet = variantSet.getCallSet(id_)
         return self.runGetRequest(callSet)
 
+    def runGetInfo(self, request):
+        """
+        Returns information about the service including protocol version.
+        """
+        return protocol.toJson(protocol.GetInfoResponse(
+            protocol_version=protocol.version))
+
+    def runAddAnnouncement(self, flaskrequest):
+        """
+        Takes a flask request from the frontend and attempts to parse
+        into an AnnouncePeerRequest. If successful, it will log the
+        announcement to the `announcement` table with some other metadata
+        gathered from the request.
+        """
+        announcement = {}
+        # We want to parse the request ourselves to collect a little more
+        # data about it.
+        try:
+            requestData = protocol.fromJson(
+                flaskrequest.get_data(), protocol.AnnouncePeerRequest)
+            announcement['hostname'] = flaskrequest.host_url
+            announcement['remote_addr'] = flaskrequest.remote_addr
+            announcement['user_agent'] = flaskrequest.headers.get('User-Agent')
+        except AttributeError:
+            # Sometimes in testing we will send protocol requests instead
+            # of flask requests and so the hostname and user agent won't
+            # be present.
+            try:
+                requestData = protocol.fromJson(
+                    flaskrequest, protocol.AnnouncePeerRequest)
+            except Exception as e:
+                raise exceptions.InvalidJsonException(e)
+        except Exception as e:
+            raise exceptions.InvalidJsonException(e)
+
+        # Validate the url before accepting the announcement
+        peer = datamodel.peers.Peer(requestData.peer.url)
+        peer.setAttributesJson(protocol.toJson(
+                requestData.peer.attributes))
+        announcement['url'] = peer.getUrl()
+        announcement['attributes'] = peer.getAttributes()
+        try:
+            self.getDataRepository().insertAnnouncement(announcement)
+        except:
+            raise exceptions.BadRequestException(announcement['url'])
+        return protocol.toJson(
+            protocol.AnnouncePeerResponse(success=True))
+
+    def runListPeers(self, request):
+        """
+        Takes a ListPeersRequest and returns a ListPeersResponse using
+        a page_token and page_size if provided.
+        """
+        return self.runSearchRequest(
+            request,
+            protocol.ListPeersRequest,
+            protocol.ListPeersResponse,
+            self.peersGenerator)
+
     def runGetVariant(self, id_):
         """
         Returns a variant with the given id

ga4gh/server/cli/repomanager.py

@@ -24,6 +24,7 @@
 import ga4gh.server.datamodel.sequence_annotations as sequence_annotations
 import ga4gh.server.datamodel.continuous as continuous
 import ga4gh.server.datamodel.variants as variants
+import ga4gh.server.datamodel.peers as peers
 import ga4gh.server.datarepo as datarepo
 import ga4gh.server.exceptions as exceptions
 import ga4gh.server.repo.rnaseq2ga as rnaseq2ga
@@ -128,6 +129,20 @@ def list(self):
         # TODO this is _very_ crude. We need much more options and detail here.
         self._repo.printSummary()
 
+    def listAnnouncements(self):
+        """
+        Lists all the announcements the repo has received.
+        """
+        self._openRepo()
+        self._repo.printAnnouncements()
+
+    def clearAnnouncements(self):
+        """
+        Clears the list of announcements from the repo.
+        """
+        self._openRepo()
+        self._repo.clearAnnouncements()
+
     def verify(self):
         """
         Checks that the data pointed to in the repository works and
@@ -508,6 +523,33 @@ def func():
             self._updateRepo(self._repo.removeIndividual, individual)
         self._confirmDelete("Individual", individual.getLocalId(), func)
 
+    def addPeer(self):
+        """
+        Adds a new peer into this repo
+        """
+        self._openRepo()
+        try:
+            peer = peers.Peer(
+                self._args.url, json.loads(self._args.attributes))
+        except exceptions.BadUrlException:
+            raise exceptions.RepoManagerException("The URL for the peer was "
+                                                  "malformed.")
+        except ValueError as e:
+            raise exceptions.RepoManagerException(
+                "The attributes message "
+                "was malformed. {}".format(e))
+        self._updateRepo(self._repo.insertPeer, peer)
+
+    def removePeer(self):
+        """
+        Removes a peer by URL from this repo
+        """
+        self._openRepo()
+
+        def func():
+            self._updateRepo(self._repo.removePeer, self._args.url)
+        self._confirmDelete("Peer", self._args.url, func)
+
     def removeOntology(self):
         """
         Removes an ontology from the repo.
@@ -654,6 +696,12 @@ def addOntologyNameArgument(cls, subparser):
             "ontologyName",
             help="the name of the ontology")
 
+    @classmethod
+    def addUrlArgument(cls, subparser):
+        subparser.add_argument(
+            "url",
+            help="The URL of the given resource")
+
     @classmethod
     def addReadGroupSetNameArgument(cls, subparser):
         subparser.add_argument(
@@ -783,6 +831,33 @@ def getParser(cls):
         listParser.set_defaults(runner="list")
         cls.addRepoArgument(listParser)
 
+        listAnnouncementsParser = common_cli.addSubparser(
+            subparsers, "list-announcements", "List the announcements in"
+                                              "the repo.")
+        listAnnouncementsParser.set_defaults(runner="listAnnouncements")
+        cls.addRepoArgument(listAnnouncementsParser)
+
+        clearAnnouncementsParser = common_cli.addSubparser(
+            subparsers, "clear-announcements", "List the announcements in"
+                                               "the repo.")
+        clearAnnouncementsParser.set_defaults(runner="clearAnnouncements")
+        cls.addRepoArgument(clearAnnouncementsParser)
+
+        addPeerParser = common_cli.addSubparser(
+            subparsers, "add-peer", "Add a peer to the registry by URL.")
+        addPeerParser.set_defaults(runner="addPeer")
+        cls.addRepoArgument(addPeerParser)
+        cls.addUrlArgument(addPeerParser)
+        cls.addAttributesArgument(addPeerParser)
+
+        removePeerParser = common_cli.addSubparser(
+            subparsers, "remove-peer", "Remove a peer from "
+                                       "the registry by URL.")
+        removePeerParser.set_defaults(runner="removePeer")
+        cls.addRepoArgument(removePeerParser)
+        cls.addUrlArgument(removePeerParser)
+        cls.addForceOption(removePeerParser)
+
         addDatasetParser = common_cli.addSubparser(
             subparsers, "add-dataset", "Add a dataset to the data repo")
         addDatasetParser.set_defaults(runner="addDataset")