Skip to content

Commit

Permalink
save a min_date available for each archive and ensure dates submitted…
Browse files Browse the repository at this point in the history 
… from front are valid and respect it (#372)
  • Loading branch information
@boogheta
boogheta committed Jul 6, 2021
1 parent 103c170 commit 9cbcca1
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 33 deletions.
8 changes: 5 additions & 3 deletions hyphe_backend/lib/webarchives.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@
"web.archive.org": {
"label": "Web.Archive.org",
"description": "crawl worldwide web archives maintained by Archive.org",
"url_prefix": "https://web.archive.org/web/"
"url_prefix": "https://web.archive.org/web/",
"min_date": "1996-01-01"
},
"archivesinternet.bnf.fr": {
"label": "ArchivesInternet.BNF.fr",
"description": "crawl France's official web archives maintained by BNF",
"url_prefix": "http://pfcarchivesinternet.bnf.fr",
"proxy": "pfcarchivesinternet.bnf.fr:9115"
"url_prefix": "http://archivesinternet.bnf.fr",
"proxy": "archivesinternet.bnf.fr:9115",
"min_date": "2007-01-01"
}
}

Expand Down
21 changes: 11 additions & 10 deletions hyphe_frontend/app/partials/webentitystartpagesmodal.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,19 +206,22 @@ <h3 class="md-title" md-colors="{'color':'default-warn'}">
<div layout="column" ng-style="{'opacity': webentity.webarchives.option ? 1 : 0, 'width': '257px'}">
<!-- TODO: DATE PICKER -->
<label><small>Date to try to approach:</small></label>
<md-input-container style="width: 95%; margin: 0px; height: 28px">
<input ng-model="webentity.webarchives.date" ng-change="setArchivesMinMaxDate()">
</md-input-container>
<div style="display: flex; justify-content: space-between; flex-wrap: nowrap">
<md-input-container style="margin: 0px; height: 28px; width: 40%; padding-bottom: 22px">
<input ng-model="webentity.webarchives.date" ng-change="setArchivesMinMaxDate()">
</md-input-container>
<small style="color: red; width: 55%" ng-show="date_error">{{ date_error }}</small>
</div>

<label><small>Delay to consider before and after the date</small></label>
<div layout="row" style="display: flex; justify-content: flex-start">
<md-input-container style="margin: 0px; height: 28px; width: 33%; text-align: center">
<md-input-container style="margin: 0px; height: 28px; width: 40%; text-align: center">
<md-select ng-model="ed_webarchive_daysrange_choice" ng-change="setArchivesMinMaxDate()">
<md-option ng-repeat="(key, value) in webarchives_periods" value="{{ key }}">{{ value }}</md-option>
</md-select>
</md-input-container>
<md-input-container ng-show="ed_webarchive_daysrange_choice === 'custom'" style="margin: 0px; height: 28px; width: 33%">
<input type="number" min="1" ng-model="ed_webarchive_daysrange_custom" ng-change="setArchivesMinMaxDate()" style="text-align: right">
<md-input-container ng-show="ed_webarchive_daysrange_choice === 'custom'" style="margin: 0px; height: 28px; width: 25%">
<input type="number" min="1" max="10000" ng-model="ed_webarchive_daysrange_custom" ng-change="setArchivesMinMaxDate()" style="text-align: right">
</md-input-container>
<span ng-show="ed_webarchive_daysrange_choice === 'custom'" style="padding-top: 5px">days</span>
</div>
Expand All @@ -227,10 +230,8 @@ <h3 class="md-title" md-colors="{'color':'default-warn'}">
<small ng-show="ed_webarchive_daysrange_choice !== 'infinity'"><em>{{ webarchives_mindate }} &#x2794; {{ webarchives_maxdate }}</em></small>
<b ng-show="ed_webarchive_daysrange_choice === 'infinity'"><em>&infin;</em></b>
</div>

</div>
</div>

</div>

<span style="margin: 8px"><span style="color:grey; filter: grayscale(100%);">🍪</span> Cookies</span>
Expand All @@ -245,7 +246,7 @@ <h3 class="md-title" md-colors="{'color':'default-warn'}">
max-rows="8"
></textarea>
<div
ng-show="cookiesError"
ng-show="cookies_error"
class="hint"
md-colors="{'color': 'default-warn'}"
>
Expand All @@ -260,7 +261,7 @@ <h3 class="md-title" md-colors="{'color':'default-warn'}">

<md-dialog-actions>
<span flex></span>
<md-button ng-click="hide()" ng-if="checkStartpagesConflicts==0 && !cookiesError">
<md-button ng-click="hide()" ng-disabled="checkStartpagesConflicts>0 || cookies_error || date_error">
Ok
</md-button>
<md-button ng-click="hide()" ng-if="checkStartpagesConflicts>0">
Expand Down
24 changes: 19 additions & 5 deletions hyphe_frontend/app/views/prepareCrawls.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -662,7 +662,8 @@ angular.module('hyphe.preparecrawlsController', [])
$scope.lookups = lookups
$scope.webentity = webentity
$scope.depthRange = depthRange
$scope.cookiesError = ""
$scope.cookies_error = ""
$scope.date_error = ""
$scope.webarchives_options = webarchives_options

$scope.webarchives_periods = {
Expand All @@ -680,6 +681,7 @@ angular.module('hyphe.preparecrawlsController', [])
$scope.infinityRange = 50 * 365

$scope.setArchivesMinMaxDate = function() {
$scope.date_error = ""
if (!$scope.webentity.webarchives.option) {
return
}
Expand All @@ -697,7 +699,11 @@ angular.module('hyphe.preparecrawlsController', [])
}
}
}


var chosen_option = $scope.webarchives_options.filter(function(o) { return o.id === $scope.webentity.webarchives.option})[0]
$scope.min_allowed_webarchives_date = new Date(chosen_option.min_date || "1995-01-01")
$scope.max_allowed_webarchives_date = new Date()

if ($scope.ed_webarchive_daysrange_choice === 'custom') {
$scope.webentity.webarchives.days_range = 2 * $scope.ed_webarchive_daysrange_custom
$scope.webarchives_days_range_display = $scope.webentity.webarchives.days_range + " days"
Expand All @@ -711,13 +717,21 @@ angular.module('hyphe.preparecrawlsController', [])
}

try {
if (!/^\d{4}-\d{2}-\d{2}$/.test($scope.webentity.webarchives.date)) {
$scope.date_error = "This is not a valid date, the format should be YYYY-MM-DD."
return
}
var dat = new Date($scope.webentity.webarchives.date)
if (dat < $scope.min_allowed_webarchives_date || dat > $scope.max_allowed_webarchives_date) {
$scope.date_error = "This web archive only ranges from " + $scope.min_allowed_webarchives_date.toISOString().slice(0, 10) + " to " + $scope.max_allowed_webarchives_date.toISOString().slice(0, 10)
return
}
dat.setDate(dat.getDate() - $scope.webentity.webarchives.days_range / 2)
$scope.webarchives_mindate = dat.toISOString().slice(0, 10)
dat.setDate(dat.getDate() + $scope.webentity.webarchives.days_range)
$scope.webarchives_maxdate = dat.toISOString().slice(0, 10)
} catch(e) {
console.log(e)
$scope.date_error = "This is not a valid date, the format should be YYYY-MM-DD."
}
}
$scope.setArchivesMinMaxDate()
Expand Down Expand Up @@ -1005,9 +1019,9 @@ angular.module('hyphe.preparecrawlsController', [])

$scope.validateCookiesString = function(){
if (! /^(\s*\w+\s*=\s*[^;]+)(;\s*\w+\s*=\s*[^;]+)*;?$/.test($scope.webentity.cookiesString) ) {
$scope.cookiesError = "This is not a valid cookies string (key1=value1; key2=value2; ...)."
$scope.cookies_error = "This is not a valid cookies string (key1=value1; key2=value2; ...)."
} else {
$scope.cookiesError = ""
$scope.cookies_error = ""
}
}

Expand Down
30 changes: 17 additions & 13 deletions hyphe_frontend/app/views/settings.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,27 +29,28 @@ <h1>
<!-- Loaded: action -->
<div ng-show="!loading">
<md-button
class="md-raised md-primary"
ng-click="editSettings()"
ng-show="!corpusNotEmpty && !corpusSettingsEditMode"
ng-disabled="saving"
class="md-raised md-primary"
ng-click="editSettings()"
ng-show="!corpusNotEmpty && !corpusSettingsEditMode"
ng-disabled="saving"
>
<md-icon>settings_applications</md-icon>
Edit settings
</md-button>

<md-button
class="md-raised md-primary"
ng-click="editSettings(true)"
ng-show="!corpusNotEmpty && corpusSettingsEditMode"
class="md-raised md-primary"
ng-click="editSettings(true)"
ng-show="!corpusNotEmpty && corpusSettingsEditMode"
ng-disabled="date_error"
>
<md-icon>done</md-icon>
Save Settings
</md-button>

<md-button
ng-click="editSettings(false)"
ng-show="!corpusNotEmpty && corpusSettingsEditMode"
ng-click="editSettings(false)"
ng-show="!corpusNotEmpty && corpusSettingsEditMode"
>
<md-icon>clear</md-icon>
Cancel
Expand Down Expand Up @@ -199,9 +200,12 @@ <h3 style="margin:0px">CRAWL WEB ARCHIVES (experimental)</h3>
<div ng-hide="ed_webarchive_option === ''">
<!-- TODO: DATE PICKER -->
<dt>Date to try to approach</dt>
<md-input-container style="width: 95%; margin: 0px; height: 28px; width: 33%">
<input ng-model="ed_webarchive_date" ng-change="setArchivesMinMaxDate()" style="text-align: center">
</md-input-container>
<div style="display: flex; justify-content: space-between; flex-wrap: nowrap">
<md-input-container style="margin: 0px; height: 28px; width: 33%">
<input ng-model="ed_webarchive_date" ng-change="setArchivesMinMaxDate()" style="text-align: center">
</md-input-container>
<small style="color: red; width: 62%" ng-show="date_error">{{ date_error }}</small>
</div>

<dt>Delay to consider before and after the date</dt>
<div layout="row" style="display: flex; justify-content: flex-start">
Expand All @@ -211,7 +215,7 @@ <h3 style="margin:0px">CRAWL WEB ARCHIVES (experimental)</h3>
</md-select>
</md-input-container>
<md-input-container ng-show="ed_webarchive_daysrange_choice === 'custom'" style="margin: 0px; height: 28px; width: 33%">
<input type="number" min="1" ng-model="ed_webarchive_daysrange_custom" ng-change="setArchivesMinMaxDate()" style="text-align: right">
<input type="number" min="1" max="10000" ng-model="ed_webarchive_daysrange_custom" ng-change="setArchivesMinMaxDate()" style="text-align: right">
</md-input-container>
<span ng-show="ed_webarchive_daysrange_choice === 'custom'" style="padding-top: 5px">days</span>
</div>
Expand Down
18 changes: 16 additions & 2 deletions hyphe_frontend/app/views/settings.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ angular.module('hyphe.settingsController', [])
$scope.destroying = false
$scope.resetting = false
$scope.saving = false
$scope.date_error = ""


$scope.destroy = function(){
Expand Down Expand Up @@ -79,6 +80,7 @@ angular.module('hyphe.settingsController', [])
$scope.infinityRange = 50 * 365

$scope.setArchivesMinMaxDate = function() {
$scope.date_error = ""
if ($scope.ed_webarchive_daysrange_custom === undefined || $scope.ed_webarchive_daysrange_choice === undefined) {
$scope.ed_webarchive_option = $scope.options.webarchives_option || ""
if ($scope.options.webarchives_days_range === $scope.infinityRange) {
Expand All @@ -94,6 +96,10 @@ angular.module('hyphe.settingsController', [])
}
}

$scope.webarchives_chosen_option = $scope.webarchives_options.filter(function(o) { return o.id === $scope.ed_webarchive_option})[0]
$scope.min_allowed_webarchives_date = new Date($scope.webarchives_chosen_option.min_date || "1995-01-01")
$scope.max_allowed_webarchives_date = new Date()

if ($scope.ed_webarchive_daysrange_choice === 'custom') {
$scope.ed_webarchive_daysrange = $scope.ed_webarchive_daysrange_custom + 0
$scope.webarchives_days_range_display = $scope.ed_webarchive_daysrange + " days"
Expand All @@ -107,20 +113,28 @@ angular.module('hyphe.settingsController', [])
}

try {
if (!/^\d{4}-\d{2}-\d{2}$/.test($scope.ed_webarchive_date)) {
$scope.date_error = "This is not a valid date, the format should be YYYY-MM-DD."
return
}
var dat = new Date($scope.ed_webarchive_date)
if (dat < $scope.min_allowed_webarchives_date || dat > $scope.max_allowed_webarchives_date) {
$scope.date_error = "This web archive only ranges from " + $scope.min_allowed_webarchives_date.toISOString().slice(0, 10) + " to " + $scope.max_allowed_webarchives_date.toISOString().slice(0, 10)
return
}
dat.setDate(dat.getDate() - $scope.ed_webarchive_daysrange)
$scope.webarchives_mindate = dat.toISOString().slice(0, 10)
dat.setDate(dat.getDate() + 2 * $scope.ed_webarchive_daysrange)
$scope.webarchives_maxdate = dat.toISOString().slice(0, 10)
} catch(e) {
console.log(e)
$scope.date_error = "This is not a valid date, the format should be YYYY-MM-DD."
}
}

$scope.editSettings = function(save){

$scope.saving = false
$scope.corpusSettingsEditMode = !$scope.corpusSettingsEditMode
$scope.corpusSettingsEditMode = !$scope.corpusSettingsEditMode
if (save) {
//construction of the array of startpages mode
$scope.ed_defaultStartpagesMode = []
Expand Down

0 comments on commit 9cbcca1

Please sign in to comment.