adjust aircoding to actual traph codebase

medialab · Aug 5, 2019 · 1632666 · 1632666
1 parent ec1c4aa
commit 1632666
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 24 deletions.
diff --git a/doc/api.md b/doc/api.md
@@ -120,6 +120,7 @@ The API will always answer as such:
     * __`get_webentity_subwebentities`__
     * __`get_webentity_parentwebentities`__
     * __`get_webentity_pagelinks_network`__
+    * __`paginate_webentity_pagelinks_network`__
     * __`get_webentity_referrers`__
     * __`get_webentity_referrals`__
     * __`get_webentity_ego_network`__
@@ -836,7 +837,7 @@ The API will always answer as such:
 
 - __`paginate_webentity_pages`:__
   + _`webentity_id`_ (mandatory)
-  + _`count`_ (optional, default: `1000`)
+  + _`count`_ (optional, default: `5000`)
   + _`pagination_token`_ (optional, default: `null`)
   + _`onlyCrawled`_ (optional, default: `false`)
   + _`corpus`_ (optional, default: `"--hyphe--"`)
@@ -874,7 +875,17 @@ The API will always answer as such:
   + _`include_external_links`_ (optional, default: `false`)
   + _`corpus`_ (optional, default: `"--hyphe--"`)
 
- Returns for a `corpus` the list of all internal NodeLinks of a WebEntity defined by `webentity_id`. Optionally add external NodeLinks (the frontier) by setting `include_external_links` to "true".
+ Warning: this method can be very slow on webentities with many pages or links, privilege paginate_webentity_pagelinks_network whenever possible. Returns for a `corpus` the list of all internal NodeLinks of a WebEntity defined by `webentity_id`. Optionally add external NodeLinks (the frontier) by setting `include_external_links` to "true".
+
+
+- __`paginate_webentity_pagelinks_network`:__
+  + _`webentity_id`_ (optional, default: `null`)
+  + _`count`_ (optional, default: `10`)
+  + _`pagination_token`_ (optional, default: `null`)
+  + _`include_external_outlinks`_ (optional, default: `false`)
+  + _`corpus`_ (optional, default: `"--hyphe--"`)
+
+ Returns for a `corpus` internal page links for `count` source pages of a WebEntity defined by `webentity_id` and returns a `pagination_token` to reuse to collect the following links. Optionally add external NodeLinks (the frontier) by setting `include_external_outlinks` to "true".
 
 
 - __`get_webentity_referrers`:__

diff --git a/hyphe_backend/core.tac b/hyphe_backend/core.tac
@@ -2786,11 +2786,11 @@ class Memory_Structure(customJSONRPC):
         returnD(format_result(res))
 
     @inlineCallbacks
-    def jsonrpc_paginate_webentity_pagelinks_network(self, webentity_id=None, count=5000, pagination_token=None, include_external_links=False, corpus=DEFAULT_CORPUS):
-        """Returns for a `corpus` `count` internal NodeLinks of a WebEntity defined by `webentity_id` and returns a `pagination_token` to reuse to collect the following links. Optionally add external NodeLinks (the frontier) by setting `include_external_links` to "true"."""
+    def jsonrpc_paginate_webentity_pagelinks_network(self, webentity_id=None, count=10, pagination_token=None, include_external_outlinks=False, corpus=DEFAULT_CORPUS):
+        """Returns for a `corpus` internal page links for `count` source pages of a WebEntity defined by `webentity_id` and returns a `pagination_token` to reuse to collect the following links. Optionally add external NodeLinks (the frontier) by setting `include_external_outlinks` to "true"."""
         if not self.parent.corpus_ready(corpus):
             returnD(self.parent.corpus_error(corpus))
-        include_external = test_bool_arg(include_external_links)
+        include_external = test_bool_arg(include_external_outlinks)
         WE = yield self.db.get_WE(corpus, webentity_id)
         if not WE:
             returnD(format_error("No webentity found for id %s" % webentity_id))
@@ -2801,24 +2801,26 @@ class Memory_Structure(customJSONRPC):
             returnD(format_error("count should be a positive integer"))
         if pagination_token:
             try:
-                total, token = pagination_token.split('|')
+                total, total_pages, token = pagination_token.split('|')
                 total = int(total)
+                total_pages = int(total_pages)
             except:
                 returnD(format_error("Pagination token '%s' seems wrong, it should look like <int>|<b64_string>" % pagination_token))
         else:
-            total, token = (0, None)
-        links = yield self.traphs.call(corpus, "paginate_webentity_pagelinks", webentity_id, WE["prefixes"], links_count=count, pagination_token=token, include_inbound=include_external, include_outbound=include_external)
+            total, total_pages, token = (0, 0, None)
+        links = yield self.traphs.call(corpus, "paginate_webentity_pagelinks", webentity_id, WE["prefixes"], source_page_count=count, pagination_token=token, include_outbound=include_external_outlinks)
         if is_error(links):
             returnD(links)
         links = links['result']
-        total += links['count']
+        total += links['count_pagelinks']
+        total_pages += links['count_sourcepages']
         if links.get('token'):
-            token = '|'.join([str(total), links['token']])
+            token = '|'.join([str(total), str(total_pages), links['token']])
         else:
             token = None
         returnD(format_result({
             'token': token,
-            'links': [list(l) for l in links['links']]
+            'links': [list(l) for l in links['pagelinks']]
         }))
 
     @inlineCallbacks

diff --git a/hyphe_frontend/app/js/service_hyphe_api.js b/hyphe_frontend/app/js/service_hyphe_api.js
@@ -49,7 +49,7 @@ angular.module('hyphe.service_hyphe_api', [])
     API.WEBENTITY_MERGE_INTO                        = 'store.merge_webentity_into_another'
 
     API.WEBENTITY_LIST_TAG_VALUE_ADD                = 'store.add_webentities_tag_value'
-    API.WEBENTITY_LIST_TAG_VALUE_REMOVE               = 'store.rm_webentities_tag_value'
+    API.WEBENTITY_LIST_TAG_VALUE_REMOVE             = 'store.rm_webentities_tag_value'
 
     API.WEBENTITY_TAG_VALUE_ADD                     = 'store.add_webentity_tag_value'
     API.WEBENTITY_TAG_VALUE_REMOVE                  = 'store.rm_webentity_tag_value'
@@ -285,7 +285,7 @@ angular.module('hyphe.service_hyphe_api', [])
         ,function(settings){
           return [
               settings.webentityId
-              ,settings.count || 5000
+              ,settings.count || 10
               ,settings.token || null
               ,settings.includeExternalLinks
               ,corpus.getId()

diff --git a/hyphe_frontend/app/views/webentity_pagesNetwork.js b/hyphe_frontend/app/views/webentity_pagesNetwork.js
@@ -79,8 +79,7 @@ angular.module('hyphe.webentityPagesNetworkController', [])
           $scope.token = result.token
 
           var percent = 100 * $scope.pages.length / $scope.webentity.pages_total
-          //$scope.status = {message: 'Loading pages ' + Math.round(percent) + ' %', progress: percent / 4}
-          $scope.status = {message: 'Loading pages ' + Math.round(percent) + ' %', progress: percent}
+          $scope.status = {message: 'Loading pages ' + Math.round(percent) + ' %', progress: percent / 4}
           if ($scope.loadAll && $scope.token) {
             $timeout(loadPages, 0)
           } else if ($scope.token === null) {
@@ -95,28 +94,25 @@ angular.module('hyphe.webentityPagesNetworkController', [])
 
     function loadNetwork(){
       if (!$scope.token) {
-        //$scope.status = {message: 'Loading links 0 %', progress: 25}
-        $scope.status = {message: 'Loading links'}
+        $scope.status = {message: 'Loading links 0 %', progress: 25}
       }
-      //api.getPaginatedPagesNetwork({
-      api.getPagesNetwork({
+      api.getPaginatedPagesNetwork({
           webentityId: $scope.webentity.id
           ,includeExternalLinks: $scope.includeExternalLinks
-        //,token: $scope.token
+          ,token: $scope.token
         }
         ,function(result){
-          //$scope.links = $scope.links.concat(result.links)
-          $scope.links = result
+          $scope.links = $scope.links.concat(result.links)
           $scope.token = result.token
 
           if ($scope.loadAll && $scope.token) {
-            var percent = $scope.links.length / ($scope.webentity.pages_total + 50 * $scope.webentity.pages_crawled)
+            var percent = $scope.links.length / ($scope.webentity.pages_total + 100 * $scope.webentity.pages_crawled)
             $scope.status = {message: 'Loading links ' + Math.round(100 * percent)+ ' %', progress: 25 + 75 * percent}
             $timeout(loadNetwork, 0)
           } else if (!$scope.token) {
             $scope.status = {message: 'Building network'}
             $scope.webentity.loading = false
-            buildNetwork()
+            $timeout(buildNetwork, 0)
           }
         }
         ,function(){