Skip to content

Commit

Permalink
fix crawls ordering issues + add nb_crawled_pages_error as sortable f…
Browse files Browse the repository at this point in the history 
…ield (closes #429)
  • Loading branch information
@boogheta
boogheta committed Dec 20, 2021
1 parent 2a01999 commit 3f028c6
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 21 deletions.
1 change: 1 addition & 0 deletions hyphe_frontend/app/app.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,7 @@ textarea.over-error {
.crawljobs-col-inde,
.crawljobs-col-craw,
.crawljobs-col-crok,
.crawljobs-col-crer,
.crawljobs-col-disp,
.crawljobs-col-disl {
padding: 0px 8px 0px 8px;
Expand Down
2 changes: 1 addition & 1 deletion hyphe_frontend/app/js/service_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -654,7 +654,6 @@ angular.module('hyphe.service_utils', [])
}

ns.consolidateJob = function(job){
job.globalStatus = ''
if(job.crawling_status == 'RUNNING'){
job.globalStatus = 'CRAWLING'
} else if(job.crawling_status != 'FINISHED'){
Expand All @@ -675,6 +674,7 @@ angular.module('hyphe.service_utils', [])
job.globalStatus = 'INDEXING ' + job.indexing_status
}
job.nb_pages_indexed = job.nb_crawled_pages - job.nb_unindexed_pages
job.nb_crawled_pages_error = job.nb_crawled_pages - job.nb_crawled_pages_200
job.duration = (job.finished_at || new Date().getTime()) - job.scheduled_at
return job
}
Expand Down
26 changes: 18 additions & 8 deletions hyphe_frontend/app/views/monitorCrawls.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@ <h3>No crawl job</h3>
<div layout="column" layout-align="center center" class="stats">
<div>
{{job.nb_crawled_pages_200}} page{{job.nb_crawled_pages_200 | plural}} crawled
<small ng-if="job.nb_crawled_pages - job.nb_crawled_pages_200">
(+{{job.nb_crawled_pages - job.nb_crawled_pages_200}} error{{job.nb_crawled_pages - job.nb_crawled_pages_200 | plural}})
<small ng-if="job.nb_crawled_pages_error">
(+{{job.nb_crawled_pages_error}} error{{job.nb_crawled_pages_error | plural}})
</small>
</div>
<div ng-show="job.nb_unindexed_pages">
Expand Down Expand Up @@ -185,19 +185,25 @@ <h3>No crawl job</h3>
md-colors="{'color': (sort == 'nb_pages_indexed') ? ('default-accent') : ('default-primary')}"
ng-click="toggleSort('nb_pages_indexed')"
>
<span><md-tooltip md-direction="top">Pages Indexed</md-tooltip>PI</span>
<span><md-tooltip md-direction="top">Pages indexed</md-tooltip>PI</span>
</div>
<div class="md-secondary crawljobs-col-craw"
<!--<div class="md-secondary crawljobs-col-craw"
md-colors="{'color': (sort == 'nb_crawled_pages') ? ('default-accent') : ('default-primary')}"
ng-click="toggleSort('nb_crawled_pages')"
>
<span><md-tooltip md-direction="top">Pages Crawled</md-tooltip>PC</span>
</div>
<span><md-tooltip md-direction="top">Pages crawled</md-tooltip>PC</span>
</div>-->
<div class="md-secondary crawljobs-col-crok"
md-colors="{'color': (sort == 'nb_crawled_pages_200') ? ('default-accent') : ('default-primary')}"
ng-click="toggleSort('nb_crawled_pages_200')"
>
<span><md-tooltip md-direction="top">Pages Crawled Successfully</md-tooltip>PC ✓</span>
<span><md-tooltip md-direction="top">Pages crawled successfully</md-tooltip>PC ✓</span>
</div>
<div class="md-secondary crawljobs-col-crer"
md-colors="{'color': (sort == 'nb_crawled_pages_error') ? ('default-accent') : ('default-primary')}"
ng-click="toggleSort('nb_crawled_pages_error')"
>
<span><md-tooltip md-direction="top">Pages badly crawled </md-tooltip>PC ✗</span>
</div>
<div class="md-secondary crawljobs-col-disp"
md-colors="{'color': (sort == 'nb_pages') ? ('default-accent') : ('default-primary')}"
Expand Down Expand Up @@ -268,8 +274,9 @@ <h3>No crawl job</h3>
</div>
<div class="md-secondary crawljobs-col-stat">{{job.globalStatus}}</div>
<div class="md-secondary crawljobs-col-inde">{{job.nb_pages_indexed}}</div>
<div class="md-secondary crawljobs-col-craw">{{job.nb_crawled_pages}}</div>
<!--<div class="md-secondary crawljobs-col-craw">{{job.nb_crawled_pages}}</div>-->
<div class="md-secondary crawljobs-col-crok">{{job.nb_crawled_pages_200}}</div>
<div class="md-secondary crawljobs-col-crer">{{job.nb_crawled_pages_error}}</div>
<div class="md-secondary crawljobs-col-disp">{{job.nb_pages}}</div>
<div class="md-secondary crawljobs-col-disl">{{job.nb_links}}</div>
<div class="md-secondary crawljobs-col-sche"><span><md-tooltip md-direction="left">{{job.created_at|date}}</md-tooltip>{{job.created_at|prettyDate}}</span></div>
Expand Down Expand Up @@ -440,6 +447,9 @@ <h3 style="margin-bottom:0px">CONTENT</h3>
<dt>Crawled pages successfully</dt>
<dd>{{crawljobsIndex[focusedJobId].nb_crawled_pages_200}}</dd>

<dt>Crawled pages with an error</dt>
<dd>{{crawljobsIndex[focusedJobId].nb_crawled_pages_error}}</dd>

<div ng-if="crawljobsIndex[focusedJobId].nb_unindexed_pages">
<dt>not indexed yet</dt>
<dd>{{crawljobsIndex[focusedJobId].nb_unindexed_pages}}</dd>
Expand Down
29 changes: 17 additions & 12 deletions hyphe_frontend/app/views/monitorCrawls.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ angular.module('hyphe.monitorcrawlsController', [])
,nb_crawled_pages: {
type: 'number'
}
,nb_crawled_pages_error: {
type: 'number'
}
,nb_crawled_pages_200: {
type: 'number'
}
Expand Down Expand Up @@ -310,15 +313,13 @@ angular.module('hyphe.monitorcrawlsController', [])
.map(utils.consolidateJob)
// Sort by currently working then reverse chronological order
.sort(function(a,b){
if (!$scope.sort) {
if (a.globalStatus === "CRAWLING" || a.globalStatus === "INDEXING")
return -1
if (b.globalStatus === "CRAWLING" || b.globalStatus === "INDEXING")
return 1
if (a.globalStatus === b.globalStatus)
return b.created_at - a.created_at
} else {
return b[$scope.sort] - b["$scope.sort"]
}
if (a.globalStatus === "CRAWLING" || a.globalStatus === "INDEXING")
return -1
if (b.globalStatus === "CRAWLING" || b.globalStatus === "INDEXING")
return 1
return b.created_at - a.created_at
})
)

Expand Down Expand Up @@ -570,6 +571,8 @@ angular.module('hyphe.monitorcrawlsController', [])
// Reset
$scope.sort = null
$scope.crawlJobs = $scope.crawlJobs.sort(function(a,b){
if (a.globalStatus === b.globalStatus)
return b.created_at - a.created_at
if (a.globalStatus === "CRAWLING" || a.globalStatus === "INDEXING")
return -1
if (b.globalStatus === "CRAWLING" || b.globalStatus === "INDEXING")
Expand All @@ -579,14 +582,16 @@ angular.module('hyphe.monitorcrawlsController', [])
} else {
$scope.sort = field
$scope.crawlJobs = $scope.crawlJobs.sort(function(a,b){
if (field == "webentity_name") {
if (a[field] < b[field]) {
if (field == "webentity_name" || field == "globalStatus") {
if (a[field].toLowerCase() < b[field].toLowerCase()) {
return -1;
} else if (b[field] < a[field]) {
} else if (b[field].toLowerCase() < a[field].toLowerCase()) {
return 1;
}
return 0;
return b.created_at - a.created_at;
}
if (a[$scope.sort] === b[$scope.sort])
return b.created_at - a.created_at
return b[$scope.sort] - a[$scope.sort]
})
}
Expand Down

0 comments on commit 3f028c6

Please sign in to comment.