allow to set obey_robots from a corpus' settings page (cf #421)

medialab · Oct 22, 2021 · 259e786 · 259e786
1 parent e4b0bb3
commit 259e786
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 1 deletion.
diff --git a/hyphe_backend/core.tac b/hyphe_backend/core.tac
@@ -171,7 +171,7 @@ class Core(customJSONRPC):
             self.corpora[corpus]['options']['proxy'].update(options.pop("proxy"))
         if "obey_robots" in options and options["obey_robots"] != self.corpora[corpus]["options"]["obey_robots"]:
             redeploy = True
-            self.corpora[corpus]['options']['obey_robots'].update(options.pop("obey_robots"))
+            self.corpora[corpus]['options']['obey_robots'] = options.pop("obey_robots")
         if 'phantom' in options and options['phantom'] != self.corpora[corpus]['options']['phantom']:
             redeploy = True
             self.corpora[corpus]["options"]["phantom"].update(options.pop("phantom"))

diff --git a/hyphe_frontend/app/views/settings.html b/hyphe_frontend/app/views/settings.html
@@ -89,6 +89,12 @@ <h3 style="margin:0px">CRAWLING</h3>
                     <input type="number" min="0" max="{{maxmax_depth}}" ng-model="ed_max_depth" aria-label="editMaxDepth">
                   </md-input-container>
 
+                  <dt>Should the crawler respect robots.txt rules if any?</dt>
+                  <md-radio-group ng-model="ed_obey_robots" class="md-primary">
+                    <md-radio-button value="1" class="md-primary">yes</md-radio-button>
+                    <md-radio-button value="0" class="md-primary">no</md-radio-button>
+                  </md-radio-group>
+
                   <div layout = "column">
                     <dt style="margin-bottom: 18px">What StartPages to use by default when crawling a new Web Entity?</dt>
                     <div flex="50">
@@ -111,6 +117,7 @@ <h3 style="margin:0px">CRAWLING</h3>
                       </md-input-container> most cited known pages
                     </div>
                   </div>
+
                 </dl>
               </md-content>
             </div>
@@ -277,6 +284,9 @@ <h3 style="margin:0px">CRAWLING</h3>
                 <dt>Maximum Depth</dt>
                 <dd>{{options.max_depth}}</dd>
 
+                <dt>Should the crawler respect robots.txt rules if any?</dt>
+                <dd>{{options.obey_robots ? 'yes' : 'no'}}</dd>
+
                 <dt>What Startpages to use by default when crawling a new Web Entity?</dt>
                 <dd>{{options.defaultStartpagesMode | arrayToString}}</dd>
 

diff --git a/hyphe_frontend/app/views/settings.js b/hyphe_frontend/app/views/settings.js
@@ -157,6 +157,7 @@ angular.module('hyphe.settingsController', [])
 
         var modifiedOptions = {
           "max_depth": $scope.ed_max_depth,
+          "obey_robots": !!(parseInt($scope.ed_obey_robots)),
           "defaultStartpagesMode": $scope.ed_defaultStartpagesMode,
           "proxy": {
             "port": $scope.ed_proxy_port,
@@ -197,6 +198,7 @@ angular.module('hyphe.settingsController', [])
 
     $scope.setEditableOptions = function() {
       $scope.ed_max_depth             = $scope.options.max_depth
+      $scope.ed_obey_robots           = !!$scope.options.obey_robots
       $scope.ed_proxy_host            = $scope.options.proxy.host + ""
       $scope.ed_proxy_port            = $scope.options.proxy.port + 0
       $scope.ed_timeout               = $scope.options.phantom.timeout + 0