From 2d0fc4b48de5eaef6879f76aee87a288c558b7e0 Mon Sep 17 00:00:00 2001 From: Mark Pittaway Date: Thu, 17 Jan 2019 16:49:14 +1100 Subject: [PATCH] [SDESK-3535] Update Time Report --- client/charts/directives/Table.js | 61 ++++ client/charts/directives/index.js | 1 + client/charts/index.js | 1 + client/charts/styles/charts.scss | 29 +- client/charts/tests/sda-table.spec.js | 213 +++++++++++++ client/charts/views/chart-form-options.html | 13 +- client/charts/views/table.html | 60 ++++ .../directives/FeaturemediaUpdatesTable.js | 45 +-- client/index.js | 2 + client/search/directives/SourceFilters.js | 19 +- client/search/services/SearchReport.js | 56 +++- client/search/views/source-fitlers.html | 15 + .../controllers/UpdateTimeReportController.js | 286 ++++++++++++++++++ .../update_time_report/controllers/index.js | 1 + .../directives/UpdateTimeReportPreview.js | 33 ++ .../directives/UpdateTimeTable.js | 183 +++++++++++ client/update_time_report/directives/index.js | 2 + client/update_time_report/index.js | 55 ++++ .../views/update-time-report-panel.html | 88 ++++++ .../views/update-time-report-parameters.html | 10 + .../views/update-time-report-preview.html | 12 + .../views/update-time-report-view.html | 1 + .../views/update-time-table.html | 10 + server/analytics/__init__.py | 2 + server/analytics/base_report/__init__.py | 19 +- .../analytics/base_report/base_report_test.py | 21 +- .../featuremedia_updates_report.py | 70 +---- .../production_time_report.py | 88 +----- server/analytics/stats/archive_statistics.py | 4 + .../analytics/stats/gen_archive_statistics.py | 33 ++ .../analytics/stats/stats_report_service.py | 147 +++++++++ .../analytics/update_time_report/__init__.py | 33 ++ .../update_time_report/update_time_report.py | 107 +++++++ .../user_acitivity_report.py | 78 +---- server/features/base_report.feature | 62 ++++ 35 files changed, 1598 insertions(+), 262 deletions(-) create mode 100644 client/charts/directives/Table.js create mode 100644 client/charts/tests/sda-table.spec.js create mode 100644 client/charts/views/table.html create mode 100644 client/update_time_report/controllers/UpdateTimeReportController.js create mode 100644 client/update_time_report/controllers/index.js create mode 100644 client/update_time_report/directives/UpdateTimeReportPreview.js create mode 100644 client/update_time_report/directives/UpdateTimeTable.js create mode 100644 client/update_time_report/directives/index.js create mode 100644 client/update_time_report/index.js create mode 100644 client/update_time_report/views/update-time-report-panel.html create mode 100644 client/update_time_report/views/update-time-report-parameters.html create mode 100644 client/update_time_report/views/update-time-report-preview.html create mode 100644 client/update_time_report/views/update-time-report-view.html create mode 100644 client/update_time_report/views/update-time-table.html create mode 100644 server/analytics/stats/stats_report_service.py create mode 100644 server/analytics/update_time_report/__init__.py create mode 100644 server/analytics/update_time_report/update_time_report.py diff --git a/client/charts/directives/Table.js b/client/charts/directives/Table.js new file mode 100644 index 000000000..03416a518 --- /dev/null +++ b/client/charts/directives/Table.js @@ -0,0 +1,61 @@ +Table.$inject = ['lodash']; + +/** + * @ngdoc directive + * @module superdesk.apps.analytics.charts + * @name sdaTable + * @requires lodash + * @description A directive that renders a sortable and clickable html table + */ +export function Table(_) { + return { + scope: { + title: '=', + subtitle: '=', + headers: '=', + rows: '=', + page: '=', + onCellClicked: '=', + }, + replace: true, + template: require('../views/table.html'), + link: function(scope) { + /** + * @ngdoc method + * @name sdaTable#onHeaderClicked + * @param {Object} header - The header column that was clicked + * @description Determines the sort filter based on the header data + */ + scope.onHeaderClicked = (header) => { + // If the header entry does not have a field attribute + // then sorting by this column is disabled + if (!header.field) { + return; + } + + let newOrder; + + if (_.get(scope, 'page.sort.field') === header.field) { + // If the header.field is already sorted, then toggle the sort order + if (_.get(scope, 'page.sort.order') === 'asc') { + newOrder = 'desc'; + } else { + newOrder = 'asc'; + } + } else { + // Otherwise set order to descending by default on first click + newOrder = 'desc'; + } + + scope.page = { + ...scope.page, + no: 1, + sort: { + field: header.field, + order: newOrder, + }, + }; + }; + }, + }; +} diff --git a/client/charts/directives/index.js b/client/charts/directives/index.js index 5215b4a62..e6edef497 100644 --- a/client/charts/directives/index.js +++ b/client/charts/directives/index.js @@ -1,2 +1,3 @@ export {Chart} from './Chart'; export {ChartContainer} from './ChartContainer'; +export {Table} from './Table'; diff --git a/client/charts/index.js b/client/charts/index.js index 6df6da7c3..1c43fcb88 100644 --- a/client/charts/index.js +++ b/client/charts/index.js @@ -45,5 +45,6 @@ angular.module('superdesk.analytics.charts', []) .directive('sdChart', directives.Chart) .directive('sdChartContainer', directives.ChartContainer) + .directive('sdaTable', directives.Table) .run(cacheIncludedTemplates); diff --git a/client/charts/styles/charts.scss b/client/charts/styles/charts.scss index 4aa0edbae..93e3a19a6 100644 --- a/client/charts/styles/charts.scss +++ b/client/charts/styles/charts.scss @@ -21,6 +21,9 @@ $colors: #7cb5ec #434348 #90ed7d #f7a35c #8085e9 #f15c80 #e4d354 #2b908f #f45b5b &--margin-bottom { margin-bottom: 3rem; + .sd-chart__table { + margin-bottom: 3rem; + } } &--multi-chart { @@ -39,10 +42,10 @@ $colors: #7cb5ec #434348 #90ed7d #f7a35c #8085e9 #f15c80 #e4d354 #2b908f #f45b5b .clickable { cursor: pointer; + } - &:hover { - border: 1px solid #9bb3ff; - } + td.clickable:hover { + border: 1px solid #9bb3ff; } .sd-chart__table-header { @@ -53,6 +56,21 @@ $colors: #7cb5ec #434348 #90ed7d #f7a35c #8085e9 #f15c80 #e4d354 #2b908f #f45b5b position: sticky; top: 0; background-color: $white; + + &--no-padding-bottom { + padding-bottom: 0; + } + + .pagination-box { + display: block; + position: absolute; + right: 0; + bottom: 0; + } + + .sd-pagination { + padding: 0; + } } .sd-chart__table-header-options { @@ -106,6 +124,11 @@ $colors: #7cb5ec #434348 #90ed7d #f7a35c #8085e9 #f15c80 #e4d354 #2b908f #f45b5b padding-top: 0; padding-bottom: 0; } + + .panel-info { + border-top: 1px solid rgba(0, 0, 0, 0.15); + padding-bottom: 10rem; + } } } diff --git a/client/charts/tests/sda-table.spec.js b/client/charts/tests/sda-table.spec.js new file mode 100644 index 000000000..251b4eb34 --- /dev/null +++ b/client/charts/tests/sda-table.spec.js @@ -0,0 +1,213 @@ +describe('sda-table', () => { + let $compile; + let scope; + let $rootScope; + let data; + let element; + + beforeEach(window.module('superdesk.core.notify')); + beforeEach(window.module('superdesk.analytics.charts')); + + beforeEach(inject((_$compile_, _$rootScope_) => { + $compile = _$compile_; + $rootScope = _$rootScope_; + + element = null; + + data = { + title: 'Test Table', + subtitle: 'Fake Data', + headers: [ + {title: 'one', field: 'one'}, + {title: 'two'}, + {title: 'three', field: 'three'}, + ], + rows: [ + [{label: 1}, {label: 2}, {label: 3, clickable: true, tooltip: 'View data', custom: {data: [3]}}], + [{label: 4}, {label: 5}, {label: 6, clickable: true, tooltip: 'View data'}], + [{label: 7}, {label: 8}, {label: 9, clickable: true, tooltip: 'View data'}], + ], + page: { + no: 1, + max: 5, + sort: { + field: 'one', + order: 'desc', + }, + }, + onCellClicked: jasmine.createSpy(), + }; + })); + + const setElement = () => { + scope = $rootScope.$new(); + Object.keys(data).forEach((key) => { + scope[key] = data[key]; + }); + + const template = angular.element(`
`); + + element = $compile(template)(scope); + + $rootScope.$digest(); + }; + + const getHeader = (index) => $( + element.find('thead') + .first() + .find('tr') + .first() + .find('th') + .get(index) + ); + + const getBodyRow = (index) => $( + element.find('tbody') + .first() + .find('tr') + .get(index) + ); + + const getCell = (row, column) => $( + getBodyRow(row) + .find('td') + .get(column) + ); + + const stripWhitespace = (elem) => ( + elem.text().replace(/\s+/g, '') + ); + + const trim = (elem) => ( + elem.text().trim() + ); + + it('renders the table', () => { + setElement(); + + // Test titles + expect(trim(element.find('.sd-chart__table-header-title'))).toBe('Test Table'); + expect(trim(element.find('.sd-chart__table-header-subtitle'))).toBe('Fake Data'); + + // Test header + expect(trim(getHeader(0))).toBe('one'); + expect(trim(getHeader(1))).toBe('two'); + expect(trim(getHeader(2))).toBe('three'); + + // Test size of the table + expect(element.find('table').length).toBe(1); + expect(element.find('tbody') + .first() + .find('tr') + .length + ).toBe(3); + + // Test data of each row + expect(getBodyRow(0).find('td').length).toBe(3); + expect(stripWhitespace(getBodyRow(0))).toBe('123'); + expect(stripWhitespace(getBodyRow(1))).toBe('456'); + expect(stripWhitespace(getBodyRow(2))).toBe('789'); + + // Renders the pagination directive if page.max > 1 + scope.page = {...data.page, max: 1}; + scope.$digest(); + expect(element.find('.pagination-box').length).toBe(0); + + scope.page = {...data.page, max: 5}; + scope.$digest(); + expect(element.find('.pagination-box').length).toBe(1); + + // Renders panel-info when no rows are to be rendered + expect(element.find('.panel-info').length).toBe(0); + scope.rows = []; + scope.$digest(); + expect(element.find('.panel-info').length).toBe(1); + }); + + it('executes callback on cell clicked', () => { + setElement(); + + // Test onCellClicked on cells with clickable=true + getCell(0, 0).click(); + getCell(0, 1).click(); + getCell(0, 2).click(); + expect(data.onCellClicked.calls.count()).toBe(1); + expect(data.onCellClicked.calls.mostRecent().args).toEqual( + [{label: 3, clickable: true, tooltip: 'View data', custom: {data: [3]}}] + ); + + getCell(1, 2).click(); + expect(data.onCellClicked.calls.count()).toBe(2); + expect(data.onCellClicked.calls.mostRecent().args).toEqual( + [{label: 6, clickable: true, tooltip: 'View data'}] + ); + + getCell(2, 2).click(); + expect(data.onCellClicked.calls.count()).toBe(3); + expect(data.onCellClicked.calls.mostRecent().args).toEqual( + [{label: 9, clickable: true, tooltip: 'View data'}] + ); + }); + + it('sorts on header click', () => { + setElement(); + + // Test initial values + expect(scope.page.sort).toEqual({field: 'one', order: 'desc'}); + expect(getHeader(0).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(0).find('.icon-chevron-down-thin').length).toBe(1); + expect(getHeader(1).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-down-thin').length).toBe(0); + + // Clicking on already selected header toggles the order + getHeader(0).click(); + expect(scope.page.sort).toEqual({field: 'one', order: 'asc'}); + expect(getHeader(0).find('.icon-chevron-up-thin').length).toBe(1); + expect(getHeader(0).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-down-thin').length).toBe(0); + + // Test clicking on non-sortable header field + // page value stays the same + getHeader(1).click(); + expect(scope.page.sort).toEqual({field: 'one', order: 'asc'}); + expect(getHeader(0).find('.icon-chevron-up-thin').length).toBe(1); + expect(getHeader(0).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-down-thin').length).toBe(0); + + // Clicking on a non-selected header defaults to descending order + getHeader(2).click(); + expect(scope.page.sort).toEqual({field: 'three', order: 'desc'}); + expect(getHeader(0).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(0).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-down-thin').length).toBe(1); + + // Again clicking on already selected header toggles the order + getHeader(2).click(); + expect(scope.page.sort).toEqual({field: 'three', order: 'asc'}); + expect(getHeader(0).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(0).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-up-thin').length).toBe(0); + expect(getHeader(1).find('.icon-chevron-down-thin').length).toBe(0); + expect(getHeader(2).find('.icon-chevron-up-thin').length).toBe(1); + expect(getHeader(2).find('.icon-chevron-down-thin').length).toBe(0); + }); +}); diff --git a/client/charts/views/chart-form-options.html b/client/charts/views/chart-form-options.html index 7028c08cf..9563f29d8 100644 --- a/client/charts/views/chart-form-options.html +++ b/client/charts/views/chart-form-options.html @@ -39,7 +39,7 @@
-
+
+
+
diff --git a/client/charts/views/table.html b/client/charts/views/table.html new file mode 100644 index 000000000..b9046086b --- /dev/null +++ b/client/charts/views/table.html @@ -0,0 +1,60 @@ +
+
+
+
+ {{title | translate}} +
+
+ {{subtitle | translate}} +
+
+
+
+
+
+ +
+

No data found

+

Change the search filters and try again!

+
+ + + + + + + + + + + +
+ {{:: header.title | translate}} + + +
+ + {{data.label}} + + + {{data.label}} + +
+
+
+
diff --git a/client/featuremedia_updates_report/directives/FeaturemediaUpdatesTable.js b/client/featuremedia_updates_report/directives/FeaturemediaUpdatesTable.js index df3afed2a..378e139fd 100644 --- a/client/featuremedia_updates_report/directives/FeaturemediaUpdatesTable.js +++ b/client/featuremedia_updates_report/directives/FeaturemediaUpdatesTable.js @@ -7,6 +7,8 @@ FeaturemediaUpdatesTable.$inject = [ 'config', 'api', 'lodash', + 'searchReport', + 'notify', ]; /** @@ -19,6 +21,8 @@ FeaturemediaUpdatesTable.$inject = [ * @requires config * @requires api * @requires lodash + * @requires searchReport + * @requires notify * @description Directive to render the interactive featuremedia updates table */ export function FeaturemediaUpdatesTable( @@ -27,7 +31,9 @@ export function FeaturemediaUpdatesTable( moment, config, api, - _ + _, + searchReport, + notify ) { return { replace: true, @@ -103,35 +109,6 @@ export function FeaturemediaUpdatesTable( }); }; - /** - * @ngdoc method - * @name sdaFeaturemediaUpdatesTable#loadItem - * @param {String} itemId - The ID of the item to load - * @return {Promise} The loaded item - * @description Loads the item by ID from either archive, published or archived collections - */ - const loadItem = (itemId) => ( - api.query('search', { - repo: 'archive,published,archived', - source: { - query: { - filtered: { - filter: { - or: [ - {term: {_id: itemId}}, - {term: {item_id: itemId}}, - ], - }, - }, - }, - sort: [{versioncreated: 'desc'}], - from: 0, - size: 1, - }, - }) - .then((result) => _.get(result, '_items[0]') || {}) - ); - /** * @ngdoc method * @name sdaFeaturemediaUpdatesTable#onSluglineClicked @@ -139,9 +116,11 @@ export function FeaturemediaUpdatesTable( * @description Loads the item then opens it in the preview */ scope.onSluglineClicked = (item) => { - loadItem(_.get(item, '_id')) + searchReport.loadArchiveItem(_.get(item, '_id')) .then((newsItem) => { scope.openPreview(newsItem); + }, (error) => { + notify.error(error); }); }; @@ -152,9 +131,11 @@ export function FeaturemediaUpdatesTable( * @description Loads the original image then opens the item in the preview */ scope.onOriginalClicked = (item) => { - loadItem(_.get(item, 'original._id')) + searchReport.loadArchiveItem(_.get(item, 'original._id')) .then((newsItem) => { scope.openPreview(newsItem); + }, (error) => { + notify.error(error); }); }; diff --git a/client/index.js b/client/index.js index 2103438d6..320307d09 100644 --- a/client/index.js +++ b/client/index.js @@ -33,6 +33,7 @@ import './desk_activity_report'; import './production_time_report'; import './user_activity_report'; import './featuremedia_updates_report'; +import './update_time_report'; angular.module('superdesk.analytics.reports', []) .provider('reports', svc.ReportsProvider); @@ -69,6 +70,7 @@ export default angular.module('superdesk.analytics', [ 'superdesk.analytics.production-time-report', 'superdesk.analytics.user-activity-report', 'superdesk.analytics.featuremedia-updates-report', + 'superdesk.analytics.update-time-report', 'superdesk-ui' ]) .service('analyticsWidgetSettings', svc.AnalyticsWidgetSettings) diff --git a/client/search/directives/SourceFilters.js b/client/search/directives/SourceFilters.js index 0a609e9e2..93da9b5c4 100644 --- a/client/search/directives/SourceFilters.js +++ b/client/search/directives/SourceFilters.js @@ -26,7 +26,8 @@ export const SOURCE_FILTERS = { ENTER: 'stats_desk_transition_enter', EXIT: 'stats_desk_transition_exit', } - } + }, + PUBLISH_PARS: 'publish_pars', }; /** @@ -114,6 +115,11 @@ export const SOURCE_FILTER_FIELDS = { source: null, field: 'desk_transitions.exit', }, + [SOURCE_FILTERS.PUBLISH_PARS]: { + paramName: SOURCE_FILTERS.PUBLISH_PARS, + source: null, + field: SOURCE_FILTERS.PUBLISH_PARS, + } }; /** @@ -529,7 +535,16 @@ export function SourceFilters( ); }, minLength: 1, - } + }, + [SOURCE_FILTERS.PUBLISH_PARS]: { + ...SOURCE_FILTER_FIELDS[SOURCE_FILTERS.PUBLISH_PARS], + label: gettextCatalog.getString('Publish Pars'), + selected: [], + exclude: false, + enabled: false, + fetch: () => $q.when(), + receive: () => [], + }, }; this.init(); diff --git a/client/search/services/SearchReport.js b/client/search/services/SearchReport.js index 046cbe99a..1caa70057 100644 --- a/client/search/services/SearchReport.js +++ b/client/search/services/SearchReport.js @@ -432,6 +432,19 @@ export function SearchReport(_, config, moment, api, $q, gettext, gettextCatalog payload.return_type = args.return_type; } + if (args.size) { + payload.size = args.size; + payload.max_results = args.size; + } + + if (args.page) { + payload.page = args.page; + } + + if (args.sort) { + payload.sort = args.sort; + } + return payload; }; @@ -519,6 +532,47 @@ export function SearchReport(_, config, moment, api, $q, gettext, gettextCatalog endpoint, asObject ? constructParams(params) : constructQuery(params) ) - .then((items) => $q.when(_.get(items, '_items[0]') || {})); + .then( + (response) => ( + _.get(response, '_items.length', 0) === 1 ? + response._items[0] : + response + ) + ); + }; + + /** + * @ngdoc method + * @name SearchReport#loadArchiveItem + * @param {String} itemId - The ID of the item to load + * @return {Promise} The item or an error + * @description Attempts to load an item from archive, published or archived using the provided ID + */ + this.loadArchiveItem = function(itemId) { + return api.query('search', { + repo: 'archive,published,archived', + source: { + query: { + filtered: { + filter: { + or: [ + {term: {_id: itemId}}, + {term: {item_id: itemId}}, + ], + }, + }, + }, + sort: [{versioncreated: 'desc'}], + from: 0, + size: 1, + }, + }) + .then((result) => { + if (_.get(result, '_items.length') < 1) { + return $q.reject(gettext('Item not found!')); + } + + return result._items[0]; + }); }; } diff --git a/client/search/views/source-fitlers.html b/client/search/views/source-fitlers.html index 952c12bd2..6959320d8 100644 --- a/client/search/views/source-fitlers.html +++ b/client/search/views/source-fitlers.html @@ -337,4 +337,19 @@ + + +
+
+ + +
+
+ diff --git a/client/update_time_report/controllers/UpdateTimeReportController.js b/client/update_time_report/controllers/UpdateTimeReportController.js new file mode 100644 index 000000000..1cd541ebd --- /dev/null +++ b/client/update_time_report/controllers/UpdateTimeReportController.js @@ -0,0 +1,286 @@ +import {DATE_FILTERS} from '../../search/directives/DateFilters'; +import {SOURCE_FILTERS} from '../../search/directives/SourceFilters'; +import {getErrorMessage} from '../../utils'; + +UpdateTimeReportController.$inject = [ + '$scope', + 'gettext', + 'gettextCatalog', + 'lodash', + 'savedReports', + 'searchReport', + 'notify', + 'moment', + 'config', + '$q', + 'chartConfig' +]; + +/** + * @ngdoc controller + * @module superdesk.apps.analytics.update-time-report + * @name UpdateTimeReportController + * @requires $scope + * @requires gettext + * @requires gettextCatalog + * @requires lodash + * @requires savedReports + * @requires searchReport + * @requires notify + * @requires moment + * @requires config + * @requires $q + * @requires chartConfig + * @description Controller for Update Time reports + */ +export function UpdateTimeReportController( + $scope, + gettext, + gettextCatalog, + _, + savedReports, + searchReport, + notify, + moment, + config, + $q, + chartConfig +) { + /** + * @ngdoc method + * @name UpdateTimeReportController#init + * @description Initialises the scope parameters + */ + this.init = () => { + $scope.currentTab = 'parameters'; + $scope.dateFilters = [ + DATE_FILTERS.YESTERDAY, + DATE_FILTERS.RELATIVE, + DATE_FILTERS.DAY, + DATE_FILTERS.RANGE, + DATE_FILTERS.LAST_WEEK, + DATE_FILTERS.LAST_MONTH, + DATE_FILTERS.RELATIVE_DAYS, + ]; + + $scope.sourceFilters = [ + SOURCE_FILTERS.DESKS, + SOURCE_FILTERS.USERS, + SOURCE_FILTERS.CATEGORIES, + SOURCE_FILTERS.GENRE, + SOURCE_FILTERS.SOURCES, + SOURCE_FILTERS.URGENCY, + SOURCE_FILTERS.INGEST_PROVIDERS, + SOURCE_FILTERS.PUBLISH_PARS, + ]; + + $scope.form = {submitted: false}; + + this.initDefaultParams(); + savedReports.selectReportFromURL(); + + this.chart = chartConfig.newConfig('chart', 'table'); + $scope.updateChartConfig(); + }; + + /** + * @ngdoc method + * @name UpdateTimeReportController#initDefaultParams + * @description Initialises the default report parameters + */ + this.initDefaultParams = () => { + $scope.item_states = searchReport.filterItemStates( + ['published', 'killed', 'corrected', 'recalled'] + ); + + $scope.currentParams = { + report: 'update_time_report', + params: { + dates: { + filter: DATE_FILTERS.YESTERDAY, + }, + must: { + categories: [], + genre: [], + sources: [], + urgency: [], + desks: [], + users: [], + publish_pars: 0, + }, + must_not: { + rewrites: false, + states: { + published: false, + killed: false, + corrected: false, + recalled: false, + }, + }, + repos: {archive_statistics: true}, + min: 1, + chart: { + type: 'table', + title: null, + subtitle: null, + }, + size: 15, + page: 1, + sort: [{time_to_next_update_publish: 'desc'}], + }, + }; + + $scope.defaultReportParams = _.cloneDeep($scope.currentParams); + }; + + /** + * @ngdoc method + * @name UpdateTimeReportController#updateChartConfig + * @description Updates the local HighchartConfig instance parameters + */ + $scope.updateChartConfig = () => { + this.chart.title = _.get($scope, 'currentParams.params.chart.title'); + this.chart.subtitle = _.get($scope, 'currentParams.params.chart.subtitle'); + }; + + /** + * @ngdoc method + * @name UpdateTimeReportController#generateTitle + * @return {String} + * @description Returns the title to use for the Highcharts config + */ + $scope.generateTitle = () => { + if (_.get($scope, 'currentParams.params.chart.title')) { + return $scope.currentParams.params.chart.title; + } + + return gettext('Update Time'); + }; + + /** + * @ngdoc method + * @name UpdateTimeReportController#generateSubtitle + * @return {String} + * @description Returns the subtitle to use for the Highcharts config based on the date parameters + */ + $scope.generateSubtitle = () => chartConfig.generateSubtitleForDates( + _.get($scope, 'currentParams.params') || {} + ); + + $scope.isDirty = () => true; + + $scope.$watch(() => savedReports.currentReport._id, (newReportId) => { + if (newReportId) { + $scope.currentParams = _.cloneDeep(savedReports.currentReport); + $scope.changePanel('advanced'); + } else { + $scope.currentParams = _.cloneDeep($scope.defaultReportParams); + } + }); + + $scope.$on('analytics:update-params', (e, newParams) => { + Object.keys(newParams).forEach((key) => { + $scope.currentParams.params[key] = newParams[key]; + }); + + $scope.generate(); + }); + + /** + * @ngdoc method + * @name UpdateTimeReportController#onDateFilterChange + * @description When the date filter changes, clear the date input fields if the filter is not 'range' + */ + $scope.onDateFilterChange = () => { + if ($scope.currentParams.params.dates.filter !== 'range') { + $scope.currentParams.params.dates.start = null; + $scope.currentParams.params.dates.end = null; + } + + $scope.updateChartConfig(); + }; + + /** + * @ngdoc method + * @name UpdateTimeReportController#runQuery + * @param {Object} params - The report parameters used to search the data + * @return {Object} + * @description Queries the ContentPublishing API and returns it's response + */ + $scope.runQuery = (params) => searchReport.query( + 'update_time_report', + params, + true + ); + + /** + * @ngdoc method + * @name UpdateTimeReportController#generate + * @description Updates the Highchart configs in the report's content view + */ + $scope.generate = () => { + $scope.changeContentView('report'); + + const params = _.cloneDeep($scope.currentParams.params); + + $scope.runQuery(params) + .then((data) => { + this.createChart( + Object.assign( + {}, + $scope.currentParams.params, + data + ) + ) + .then((chartConfig) => { + $scope.form.submitted = true; + $scope.changeReportParams(chartConfig); + }); + }, (error) => { + notify.error( + getErrorMessage( + error, + gettext('Error. The Publishing Report could not be generated!') + ) + ); + }); + }; + + /** + * @ngdoc method + * @name UpdateTimeReportController#changeTab + * @param {String} tabName - The name of the tab to change to + * @description Change the current tab in the filters panel + */ + $scope.changeTab = (tabName) => { + $scope.currentTab = tabName; + }; + + /** + * @ngdoc method + * @name UpdateTimeReportController#createChart + * @param {Object} report - The report parameters used to search the data + * @return {Object} + * @description Generates the Highcharts config from the report parameters + */ + this.createChart = (report) => ( + $q.when({ + charts: [report], + title: $scope.generateTitle(report), + subtitle: $scope.generateSubtitle(), + }) + ); + + /** + * @ngdoc method + * @name UpdateTimeReportController#getReportParams + * @return {Promise} + * @description Loads field translations for this report and returns them along with current report params + * This is used so that saving this report will also save the translations with it + */ + $scope.getReportParams = () => ( + $q.when(_.cloneDeep($scope.currentParams)) + ); + + this.init(); +} diff --git a/client/update_time_report/controllers/index.js b/client/update_time_report/controllers/index.js new file mode 100644 index 000000000..c5951b620 --- /dev/null +++ b/client/update_time_report/controllers/index.js @@ -0,0 +1 @@ +export {UpdateTimeReportController} from './UpdateTimeReportController'; diff --git a/client/update_time_report/directives/UpdateTimeReportPreview.js b/client/update_time_report/directives/UpdateTimeReportPreview.js new file mode 100644 index 000000000..c06b468c8 --- /dev/null +++ b/client/update_time_report/directives/UpdateTimeReportPreview.js @@ -0,0 +1,33 @@ +UpdateTimeReportPreview.$inject = [ + 'lodash', + 'gettext', + 'chartConfig', +]; + +/** + * @ngdoc directive + * @module superdesk.apps.analytics.update-time-report + * @name sdaUpdateTimeReportPreview + * @requires lodash + * @requires gettext + * @requires chartConfig + * @description Directive to render the preview for UpdateTime report in Schedules page + */ +export function UpdateTimeReportPreview( + _, + gettext, + chartConfig +) { + return { + template: require('../views/update-time-report-preview.html'), + link: function(scope) { + const params = _.get(scope.report, 'params') || {}; + + scope.title = _.get(params, 'chart.title') ? + params.chart.title : + gettext('Update Time'); + + scope.subtitle = chartConfig.generateSubtitleForDates(params); + } + }; +} diff --git a/client/update_time_report/directives/UpdateTimeTable.js b/client/update_time_report/directives/UpdateTimeTable.js new file mode 100644 index 000000000..4e33c6bd2 --- /dev/null +++ b/client/update_time_report/directives/UpdateTimeTable.js @@ -0,0 +1,183 @@ +UpdateTimeTable.$inject = [ + 'gettext', + 'userList', + 'moment', + 'config', + 'api', + 'lodash', + '$interpolate', + '$q', + 'notify', + '$rootScope', + 'searchReport', +]; + +/** + * @ngdoc directive + * @module superdesk.apps.analytics.update-time-report + * @name sdaUpdateTimeTable + * @requires gettext + * @requires userList + * @requires moment + * @requires config + * @requires api + * @requires lodash + * @requires $interpolate + * @requires $q + * @requires notify + * @requires $rootScope + * @requires searchReport + * @description Directive to render the interactive featuremedia updates table + */ +export function UpdateTimeTable( + gettext, + userList, + moment, + config, + api, + _, + $interpolate, + $q, + notify, + $rootScope, + searchReport +) { + return { + replace: true, + require: '^sdAnalyticsContainer', + template: require('../views/update-time-table.html'), + link: function(scope, element) { + /** + * @ngdoc method + * @name sdaUpdateTimeTable#init + * @description Initialises the scope parameters and loads list of users + */ + const init = () => { + scope.selected = {preview: null}; + scope.itemUpdates = []; + scope.page = { + no: 1, + size: 5, + max: 5, + sort: { + field: 'time_to_next_update_publish', + order: 'desc', + } + }; + + scope.headers = [ + {title: gettext('Published'), field: 'firstpublished'}, + {title: gettext('Slugline')}, + {title: gettext('Headline')}, + {title: gettext('Updated In'), field: 'time_to_next_update_publish'}, + ]; + }; + + /** + * @ngdoc method + * @name sdaUpdateTimeTable#updateTable + * @description Updates the data used to display the table + */ + const updateTable = () => { + const report = _.get(scope, 'reportConfigs.charts[0]') || {}; + const items = _.get(report, '_items') || []; + + const meta = _.get(report, '_meta') || {}; + + scope.page.no = meta.page || 1; + scope.page.size = report.size; + scope.page.max = Math.ceil(meta.total / report.size); + + const genDateStr = (date) => ( + moment(date).format(config.view.dateformat + ' ' + config.view.timeformat) + ); + + const getUpdateString = (seconds) => { + const times = moment() + .startOf('day') + .seconds(seconds) + .format('H:m:s') + .split(':'); + + if (times[0] > 0) { + return $interpolate( + gettext('{{hours}} hours, {{minutes}} minutes') + )({hours: times[0], minutes: times[1]}); + } + + return $interpolate( + gettext('{{minutes}} minutes') + )({minutes: times[1]}); + }; + + scope.rows = []; + + items.forEach((item) => { + const publishTime = _.get(item, 'firstpublished'); + const updateTime = moment(publishTime) + .add(_.get(item, 'time_to_next_update_publish'), 'seconds'); + const updated = getUpdateString(_.get(item, 'time_to_next_update_publish')) + + ` (${genDateStr(updateTime)})`; + + scope.rows.push([{ + label: genDateStr(publishTime), + clickable: true, + tooltip: gettext('View Original'), + id: _.get(item, '_id'), + }, { + label: _.get(item, 'slugline'), + clickable: false, + }, { + label: _.get(item, 'headline'), + clickable: false, + }, { + label: updated, + clickable: true, + tooltip: gettext('View Update'), + id: _.get(item, 'rewritten_by'), + }]); + }); + }; + + /** + * @ngdoc method + * @name sdaUpdateTimeTable#onCellClicked + * @param {Object} data - The data from the cell that was clicked + * @description Loads the item then opens it in the preview + */ + scope.onCellClicked = (data) => { + searchReport.loadArchiveItem(_.get(data, 'id')) + .then((newsItem) => { + scope.openPreview(newsItem); + }, (error) => { + notify.error(error); + }); + }; + + // Scroll to the top when the report configs change + scope.$watch('reportConfigs.charts', () => { + element.parent().scrollTop(0); + scope.closePreview(); + updateTable(); + }, true); + + scope.$watch('page', (newPage, oldPage) => { + const newParams = {}; + + if (newPage.no !== oldPage.no) { + newParams.page = newPage.no; + } + + if (newPage.sort.field !== oldPage.sort.field || newPage.sort.order !== oldPage.sort.order) { + newParams.sort = [{[newPage.sort.field]: newPage.sort.order}]; + } + + if (Object.keys(newParams).length > 0) { + $rootScope.$broadcast('analytics:update-params', newParams); + } + }, true); + + init(); + }, + }; +} diff --git a/client/update_time_report/directives/index.js b/client/update_time_report/directives/index.js new file mode 100644 index 000000000..7f586ccfc --- /dev/null +++ b/client/update_time_report/directives/index.js @@ -0,0 +1,2 @@ +export {UpdateTimeTable} from './UpdateTimeTable'; +export {UpdateTimeReportPreview} from './UpdateTimeReportPreview'; diff --git a/client/update_time_report/index.js b/client/update_time_report/index.js new file mode 100644 index 000000000..954e28fc9 --- /dev/null +++ b/client/update_time_report/index.js @@ -0,0 +1,55 @@ +/** + * This file is part of Superdesk. + * + * Copyright 2018 Sourcefabric z.u. and contributors. + * + * For the full copyright and license information, please see the + * AUTHORS and LICENSE files distributed with this source code, or + * at https://www.sourcefabric.org/superdesk/license + */ + +import * as ctrl from './controllers'; +import * as directives from './directives'; + +function cacheIncludedTemplates($templateCache) { + $templateCache.put( + 'update-time-report-panel.html', + require('./views/update-time-report-panel.html') + ); + $templateCache.put( + 'update-time-report-parameters.html', + require('./views/update-time-report-parameters.html') + ); + $templateCache.put( + 'update-time-report-view.html', + require('./views/update-time-report-view.html') + ); +} +cacheIncludedTemplates.$inject = ['$templateCache']; + +/** + * @ngdoc module + * @module superdesk.analytics.update-time-report + * @name superdesk.analytics.update-time-report + * @packageName analytics.update-time-report + * @description Superdesk analytics generate report of time to first publish of updates. + */ +angular.module('superdesk.analytics.update-time-report', []) + .controller('UpdateTimeReportController', ctrl.UpdateTimeReportController) + + .directive('sdaUpdateTimeReportPreview', directives.UpdateTimeReportPreview) + .directive('sdaUpdateTimeTable', directives.UpdateTimeTable) + + .run(cacheIncludedTemplates) + + .config(['reportsProvider', 'gettext', function(reportsProvider, gettext) { + reportsProvider.addReport({ + id: 'update_time_report', + label: gettext('Update Time'), + sidePanelTemplate: 'update-time-report-panel.html', + priority: 600, + privileges: {update_time_report: 1}, + allowScheduling: true, + reportTemplate: 'update-time-report-view.html', + }); + }]); diff --git a/client/update_time_report/views/update-time-report-panel.html b/client/update_time_report/views/update-time-report-panel.html new file mode 100644 index 000000000..aa29752d8 --- /dev/null +++ b/client/update_time_report/views/update-time-report-panel.html @@ -0,0 +1,88 @@ +
+
+
+ + + +
+
    +
  • + +
    • +
  • + +
    • +
+
+ +
+ +
+
+ + + +
+ +
+
+
+
+ +
+
+
+
+ +
+
diff --git a/client/update_time_report/views/update-time-report-parameters.html b/client/update_time_report/views/update-time-report-parameters.html new file mode 100644 index 000000000..3a17a711c --- /dev/null +++ b/client/update_time_report/views/update-time-report-parameters.html @@ -0,0 +1,10 @@ +
+ Generate a report showing the time between original publish and update publish. +
+ +
diff --git a/client/update_time_report/views/update-time-report-preview.html b/client/update_time_report/views/update-time-report-preview.html new file mode 100644 index 000000000..1a8ff03ab --- /dev/null +++ b/client/update_time_report/views/update-time-report-preview.html @@ -0,0 +1,12 @@ +
+ +

{{title}}

+
+
+ +

{{subtitle}}

+
+ + + + diff --git a/client/update_time_report/views/update-time-report-view.html b/client/update_time_report/views/update-time-report-view.html new file mode 100644 index 000000000..8159c22fa --- /dev/null +++ b/client/update_time_report/views/update-time-report-view.html @@ -0,0 +1 @@ +
diff --git a/client/update_time_report/views/update-time-table.html b/client/update_time_report/views/update-time-table.html new file mode 100644 index 000000000..3b4360a02 --- /dev/null +++ b/client/update_time_report/views/update-time-table.html @@ -0,0 +1,10 @@ +
+
+
diff --git a/server/analytics/__init__.py b/server/analytics/__init__.py index fe92147c4..1c49c3390 100644 --- a/server/analytics/__init__.py +++ b/server/analytics/__init__.py @@ -29,6 +29,7 @@ from analytics.production_time_report import init_app as init_production_time_report from analytics.user_activity_report import init_app as init_user_acitivity_report from analytics.featuremedia_updates_report import init_app as init_featuremedia_updates_report +from analytics.update_time_report import init_app as init_update_time_report from analytics.commands import SendScheduledReports # noqa from analytics.common import is_highcharts_installed, register_report @@ -144,6 +145,7 @@ def init_app(app): init_production_time_report(app) init_user_acitivity_report(app) init_featuremedia_updates_report(app) + init_update_time_report(app) # If this app is for testing, then create an endpoint for the base reporting service # so the core searching/aggregation functionality can be tested diff --git a/server/analytics/base_report/__init__.py b/server/analytics/base_report/__init__.py index e412b17b1..6580a7737 100644 --- a/server/analytics/base_report/__init__.py +++ b/server/analytics/base_report/__init__.py @@ -9,7 +9,7 @@ # at https://www.sourcefabric.org/superdesk/license from flask import json, current_app as app -from eve_elastic.elastic import set_filters +from eve_elastic.elastic import set_filters, ElasticCursor from superdesk import get_resource_service, es_utils from superdesk.utils import ListCursor @@ -46,6 +46,9 @@ def generate_report(self, docs, args): """ Overwrite this method to generate a report based on the aggregation data """ + if args.get('aggs', 1) == 0: + return docs + return self.get_aggregation_buckets(docs.hits) def generate_highcharts_config(self, docs, args): @@ -275,6 +278,12 @@ def get(self, req, **lookup): if 'include_items' in args and int(args['include_items']): report['_items'] = list(docs) + if isinstance(report, list): + return ListCursor(report) + elif isinstance(report, ListCursor): + return report + elif isinstance(report, ElasticCursor): + return report return ListCursor([report]) def get_utc_offset(self): @@ -370,6 +379,9 @@ def _es_set_repos(self, query, params): def _es_set_size(self, query, params): query['size'] = params.get('size') or 0 + if 'page' in params: + query['page'] = params['page'] + def _es_set_sort(self, query, params): query['sort'] = params.get('sort') or [{self.date_filter_field: 'desc'}] @@ -524,6 +536,11 @@ def generate_elastic_query(self, args): if 'size' in query.keys(): es_query['source']['size'] = query['size'] + page = query.get('page') or 1 + es_query['source']['from'] = (page - 1) * query['size'] + es_query['page'] = query.get('page') or 1 + es_query['max_results'] = query['size'] + if 'sort' in query.keys(): es_query['source']['sort'] = query['sort'] diff --git a/server/analytics/base_report/base_report_test.py b/server/analytics/base_report/base_report_test.py index 2e6066650..ad9dae797 100644 --- a/server/analytics/base_report/base_report_test.py +++ b/server/analytics/base_report/base_report_test.py @@ -77,8 +77,11 @@ def assert_bool_query(self, query, result): } }, 'sort': [{'versioncreated': 'desc'}], - 'size': 0 - } + 'size': 0, + 'from': 0 + }, + 'max_results': 0, + 'page': 1 }) def test_get_aggregation_buckets(self): @@ -301,9 +304,12 @@ def test_generate_elastic_query_for_repos(self): 'must_not': [] }}}}, 'sort': [{'versioncreated': 'desc'}], - 'size': 0 + 'size': 0, + 'from': 0 }, - 'repo': 'archived,published' + 'repo': 'archived,published', + 'max_results': 0, + 'page': 1 }) def test_generate_elastic_query_for_desks(self): @@ -584,8 +590,11 @@ def test_generate_elastic_query_size(self): 'must_not': [] }}}}, 'sort': [{'versioncreated': 'desc'}], - 'size': 200 - } + 'size': 200, + 'from': 0 + }, + 'max_results': 200, + 'page': 1 }) def test_get_with_request(self): diff --git a/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py b/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py index 75ce30eb6..caf95b712 100644 --- a/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py +++ b/server/analytics/featuremedia_updates_report/featuremedia_updates_report.py @@ -11,7 +11,7 @@ from superdesk.resource import Resource from superdesk.utc import utc_to_local -from analytics.base_report import BaseReportService +from analytics.stats.stats_report_service import StatsReportService from analytics.chart_config import ChartConfig from flask import current_app as app @@ -26,7 +26,7 @@ class FeaturemdiaUpdatesReportResource(Resource): privileges = {'GET': 'featuremedia_updates_report'} -class FeaturemediaUpdatesTimeReportService(BaseReportService): +class FeaturemediaUpdatesTimeReportService(StatsReportService): aggregations = None repos = ['archive_statistics'] date_filter_field = 'versioncreated' @@ -35,45 +35,9 @@ def get_request_aggregations(self, params, args): """Disable generating aggregations""" return None - def _get_filters(self, repos, invisible_stages): - return None - - def get_elastic_index(self, types): - return 'statistics' - def _es_set_size(self, query, params): """Disable setting the size""" - pass - - def _es_filter_desks(self, query, desks, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.timeline', - 'query': { - 'terms': {'stats.timeline.task.desk': desks} - } - } - }) - - def _es_filter_users(self, query, users, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.timeline', - 'query': { - 'terms': {'stats.timeline.task.user': users} - } - } - }) - - def _es_filter_stages(self, query, stages, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.timeline', - 'query': { - 'terms': {'stats.timeline.task.stage': stages} - } - } - }) + query['size'] = 200 def generate_elastic_query(self, args): query = super().generate_elastic_query(args) @@ -176,23 +140,21 @@ def gen_date_str(date): user_translations.get(update.get('user')) or '' )) - row = [ + rows.append([ gen_date_str(item.get('versioncreated')), user_translations.get(item.get('original_creator')) or '', item.get('slugline') or '', original_image.get('headline') or original_image.get('alt_text') or '', '
  • {}
'.format('
  • '.join(updates)) - ] - - rows.append(row) - - report['highcharts'] = [{ - 'id': 'featuremedia_updates', - 'type': 'table', - 'title': title, - 'subtitle': subtitle, - 'headers': ['Date', 'Creator', 'Slugline', 'Original Image', 'Featuremedia History'], - 'rows': rows - }] - - return report + ]) + + return { + 'highcharts': [{ + 'id': 'featuremedia_updates', + 'type': 'table', + 'title': title, + 'subtitle': subtitle, + 'headers': ['Date', 'Creator', 'Slugline', 'Original Image', 'Featuremedia History'], + 'rows': rows + }] + } diff --git a/server/analytics/production_time_report/production_time_report.py b/server/analytics/production_time_report/production_time_report.py index d35a5e10f..913955fb7 100644 --- a/server/analytics/production_time_report/production_time_report.py +++ b/server/analytics/production_time_report/production_time_report.py @@ -10,7 +10,7 @@ from superdesk.resource import Resource -from analytics.base_report import BaseReportService +from analytics.stats.stats_report_service import StatsReportService from analytics.chart_config import SDChart, ChartConfig from analytics.common import seconds_to_human_readable @@ -23,7 +23,7 @@ class ProductionTimeReportResource(Resource): privileges = {'GET': 'production_time_report'} -class ProductionTimeReportService(BaseReportService): +class ProductionTimeReportService(StatsReportService): aggregations = { 'operations': { 'terms': { @@ -74,90 +74,6 @@ def get_request_aggregations(self, params, args): } } - def _get_filters(self, repos, invisible_stages): - return None - - def get_elastic_index(self, types): - return 'statistics' - - def _get_es_query_funcs(self): - funcs = super()._get_es_query_funcs() - - funcs['desk_transitions'] = { - 'query': self._es_filter_desk_transitions - } - - return funcs - - def _es_filter_desks(self, query, desks, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.desk_transitions', - 'query': { - 'terms': {'stats.desk_transitions.desk': desks} - } - } - }) - - def _es_filter_users(self, query, users, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.desk_transitions', - 'query': { - 'terms': {'stats.desk_transitions.user': users} - } - } - }) - - def _es_filter_stages(self, query, stages, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.desk_transitions', - 'query': { - 'terms': {'stats.desk_transitions.stage': stages} - } - } - }) - - def _es_filter_desk_transitions(self, query, value, must, params): - if not isinstance(value, dict): - return - - if value.get('min') or value.get('max'): - ranges = {} - - if value.get('min'): - ranges['gte'] = value['min'] - - if value.get('max'): - ranges['lte'] = value['ax'] - - query[must].append({ - 'range': { - 'num_desk_transitions': ranges - } - }) - - if value.get('enter'): - query[must].append({ - 'nested': { - 'path': 'stats.desk_transitions', - 'query': { - 'terms': {'stats.desk_transitions.entered_operation': value['enter']} - } - } - }) - - if value.get('exit'): - query[must].append({ - 'nested': { - 'path': 'stats.desk_transitions', - 'query': { - 'terms': {'stats.desk_transitions.exited_operation': value['exit']} - } - } - }) - def generate_report(self, docs, args): aggregations = getattr(docs, 'hits', {}).get('aggregations') or {} diff --git a/server/analytics/stats/archive_statistics.py b/server/analytics/stats/archive_statistics.py index a5eb6dc11..d4b6dd0d2 100644 --- a/server/analytics/stats/archive_statistics.py +++ b/server/analytics/stats/archive_statistics.py @@ -237,6 +237,10 @@ class ArchiveStatisticsResource(Resource): 'original_par_count': metadata_schema['word_count'], 'par_count': metadata_schema['word_count'], 'time_to_first_publish': {'type': 'integer'}, + 'time_to_next_update_publish': { + 'type': 'integer', + 'default': 0 + }, 'num_desk_transitions': { 'type': 'integer', 'default': 0 diff --git a/server/analytics/stats/gen_archive_statistics.py b/server/analytics/stats/gen_archive_statistics.py index 642859dcd..69285b66a 100644 --- a/server/analytics/stats/gen_archive_statistics.py +++ b/server/analytics/stats/gen_archive_statistics.py @@ -297,6 +297,7 @@ def set_metadata_updates(self, item, history): def process_timelines(self, items, failed_ids): statistics_service = get_resource_service('archive_statistics') items_to_create = [] + rewrites = [] for item_id, item in items.items(): try: @@ -306,6 +307,10 @@ def process_timelines(self, items, failed_ids): failed_ids.append(item_id) continue + if item['updates'].get('rewrite_of') and \ + (item['updates'].get('time_to_first_publish') or 0) > 0: + rewrites.append(item_id) + if not item['item'].get(config.ID_FIELD): item['updates'][config.ID_FIELD] = item_id item['updates']['stats_type'] = 'archive' @@ -333,6 +338,34 @@ def process_timelines(self, items, failed_ids): )) failed_ids.extend(failed_ids) + for item_id in rewrites: + item = items[item_id] + + updated_at = item['updates'].get('firstpublished') + if not updated_at: + logger.warning('Failed {}, updated_at not defined'.format(item_id)) + continue + + original_id = item['updates'].get('rewrite_of') + if not original_id: + logger.warning('Failed {}, original_id not defined'.format(item_id)) + continue + + original = statistics_service.find_one(req=None, _id=original_id) + if not original: + logger.warning('Failed {}, original not found'.format(item_id)) + continue + + published_at = original.get('firstpublished') + if not published_at: + logger.warning('Failed {}, published_at not defined'.format(original_id)) + continue + + statistics_service.patch( + original_id, + {'time_to_next_update_publish': (updated_at - published_at).total_seconds()} + ) + def _store_update_fields(self, entry): update = {} diff --git a/server/analytics/stats/stats_report_service.py b/server/analytics/stats/stats_report_service.py new file mode 100644 index 000000000..1dac0aa86 --- /dev/null +++ b/server/analytics/stats/stats_report_service.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8; -*- +# +# This file is part of Superdesk. +# +# Copyright 2013-2019 Sourcefabric z.u. and contributors. +# +# For the full copyright and license information, please see the +# AUTHORS and LICENSE files distributed with this source code, or +# at https://www.sourcefabric.org/superdesk/license + +from analytics.base_report import BaseReportService + + +class StatsReportService(BaseReportService): + def get_elastic_index(self, types): + return 'statistics' + + def _get_filters(self, repos, invisible_stages): + return None + + def _get_es_query_funcs(self): + funcs = super()._get_es_query_funcs() + + funcs.update({ + 'desk_transitions': {'query': self._es_filter_desk_transitions}, + 'user_locks': {'query': self._es_filter_user_locks}, + 'publish_pars': {'query': self._es_filter_publish_pars} + }) + + return funcs + + def _es_filter_desks(self, query, desks, must, params): + query[must].append({ + 'nested': { + 'path': 'stats.timeline', + 'query': { + 'terms': {'stats.timeline.task.desk': desks} + } + } + }) + + def _es_filter_users(self, query, users, must, params): + query[must].append({ + 'nested': { + 'path': 'stats.timeline', + 'query': { + 'terms': {'stats.timeline.task.user': users} + } + } + }) + + def _es_filter_stages(self, query, stages, must, params): + query[must].append({ + 'nested': { + 'path': 'stats.timeline', + 'query': { + 'terms': {'stats.timeline.task.stage': stages} + } + } + }) + + def _es_filter_desk_transitions(self, query, value, must, params): + if not isinstance(value, dict): + return + + if value.get('min') or value.get('max'): + ranges = {} + + if value.get('min'): + ranges['gte'] = value['min'] + + if value.get('max'): + ranges['lte'] = value['ax'] + + query[must].append({ + 'range': { + 'num_desk_transitions': ranges + } + }) + + if value.get('enter'): + query[must].append({ + 'nested': { + 'path': 'stats.desk_transitions', + 'query': { + 'terms': {'stats.desk_transitions.entered_operation': value['enter']} + } + } + }) + + if value.get('exit'): + query[must].append({ + 'nested': { + 'path': 'stats.desk_transitions', + 'query': { + 'terms': {'stats.desk_transitions.exited_operation': value['exit']} + } + } + }) + + def _es_filter_user_locks(self, query, user, must, params): + lt, gte, time_zone = self._es_get_date_filters(params) + + query[must].append({ + 'nested': { + 'path': 'stats.timeline', + 'query': { + 'bool': { + 'must': [ + {'term': {'stats.timeline.task.user': user}}, + {'terms': { + 'stats.timeline.operation': [ + 'item_lock', + 'item_unlock' + ] + }}, + {'range': { + 'stats.timeline.operation_created': { + 'gte': gte, + 'lt': lt, + 'time_zone': time_zone + } + }} + ] + } + } + } + }) + + def _es_filter_publish_pars(self, query, value, must, params): + if value: + query[must].append({ + 'nested': { + 'path': 'stats.timeline', + 'query': { + 'filtered': { + 'filter': { + 'and': [{ + 'term': {'stats.timeline.operation': 'publish'}, + }, { + 'term': {'stats.timeline.par_count': value} + }] + } + } + } + } + }) diff --git a/server/analytics/update_time_report/__init__.py b/server/analytics/update_time_report/__init__.py new file mode 100644 index 000000000..ef82f859a --- /dev/null +++ b/server/analytics/update_time_report/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8; -*- +# +# This file is part of Superdesk. +# +# Copyright 2018 Sourcefabric z.u. and contributors. +# +# For the full copyright and license information, please see the +# AUTHORS and LICENSE files distributed with this source code, or +# at https://www.sourcefabric.org/superdesk/license + +import superdesk +from .update_time_report import UpdateTimeReportResource, UpdateTimeReportService +from analytics.common import register_report + + +def init_app(app): + # Don't register this endpoint if archive stats aren't being generated + # Generating stats with PUBLISH_ASSOCIATED_ITEMS=True is currently not supported + if not app.config.get('ANALYTICS_ENABLE_ARCHIVE_STATS', False) or \ + app.config.get('PUBLISH_ASSOCIATED_ITEMS', False): + return + + endpoint_name = 'update_time_report' + service = UpdateTimeReportService(endpoint_name, backend=superdesk.get_backend()) + UpdateTimeReportResource(endpoint_name, app=app, service=service) + + register_report(endpoint_name, endpoint_name) + + superdesk.privilege( + name=endpoint_name, + label='Update Time Report', + description='User can view Update Time Report' + ) diff --git a/server/analytics/update_time_report/update_time_report.py b/server/analytics/update_time_report/update_time_report.py new file mode 100644 index 000000000..91d17306c --- /dev/null +++ b/server/analytics/update_time_report/update_time_report.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8; -*- +# +# This file is part of Superdesk. +# +# Copyright 2013-2018 Sourcefabric z.u. and contributors. +# +# For the full copyright and license information, please see the +# AUTHORS and LICENSE files distributed with this source code, or +# at https://www.sourcefabric.org/superdesk/license + +from superdesk.resource import Resource +from superdesk.utc import utc_to_local, utcnow + +from analytics.stats.stats_report_service import StatsReportService +from analytics.chart_config import ChartConfig + +from flask import current_app as app +from datetime import timedelta + + +class UpdateTimeReportResource(Resource): + """Update Time Report schema""" + + item_methods = ['GET'] + resource_methods = ['GET'] + privileges = {'GET': 'update_time_report'} + + +class UpdateTimeReportService(StatsReportService): + aggregations = None + repos = ['archive_statistics'] + date_filter_field = 'firstpublished' + + def get_request_aggregations(self, params, args): + """Disable generating aggregations""" + return None + + def generate_elastic_query(self, args): + query = super().generate_elastic_query(args) + + query['source']['query']['filtered']['filter']['bool']['must'].extend([ + {'range': {'time_to_next_update_publish': {'gt': 0}}}, + {'exists': {'field': 'rewritten_by'}}, + ]) + + query['source']['query']['filtered']['filter']['bool']['must_not'].extend([ + {'exists': {'field': 'rewrite_of'}} + ]) + + return query + + def generate_report(self, docs, args): + for doc in docs: + doc.pop('stats', None) + return docs + + def generate_highcharts_config(self, docs, args): + items = list(self.generate_report(docs, args)) + + params = args.get('params') or {} + chart_params = params.get('chart') or {} + + title = chart_params.get('title') or 'Update Time' + subtitle = chart_params.get('subtitle') or ChartConfig.gen_subtitle_for_dates(params) + + rows = [] + + def gen_date_str(date): + return utc_to_local( + app.config['DEFAULT_TIMEZONE'], + date + ).strftime('%d/%m/%Y %H:%M') + + def gen_update_string(seconds): + times = (utcnow().replace(minute=0, hour=0, second=0)) + timedelta(seconds=seconds) + times = times.strftime('%H:%M').split(':') + + if int(times[0]) > 0: + return '{} hours, {} minutes'.format(times[0], times[1]) + + return '{} minutes'.format(times[1]) + + for item in items: + publish_time = item.get('firstpublished') + update_time = publish_time + timedelta(seconds=item.get('time_to_next_update_publish')) + updated = '{} ({})'.format( + gen_update_string(item.get('time_to_next_update_publish')), + gen_date_str(update_time) + ) + + rows.append([ + gen_date_str(publish_time), + item.get('slugline') or '', + item.get('headline') or '', + updated + ]) + + return { + 'highcharts': [{ + 'id': 'update_time_report', + 'type': 'table', + 'title': title, + 'subtitle': subtitle, + 'headers': ['Published', 'Slugline', 'Headline', 'Updated In'], + 'rows': rows + }] + } diff --git a/server/analytics/user_activity_report/user_acitivity_report.py b/server/analytics/user_activity_report/user_acitivity_report.py index 560cd2c5f..762bfaa13 100644 --- a/server/analytics/user_activity_report/user_acitivity_report.py +++ b/server/analytics/user_activity_report/user_acitivity_report.py @@ -10,7 +10,7 @@ from superdesk.resource import Resource -from analytics.base_report import BaseReportService +from analytics.stats.stats_report_service import StatsReportService from eve_elastic.elastic import parse_date @@ -23,7 +23,7 @@ class UserActivityReportResource(Resource): privileges = {'GET': 'user_activity_report'} -class UserActivityReportService(BaseReportService): +class UserActivityReportService(StatsReportService): repos = ['archive_statistics'] date_filter_field = 'versioncreated' @@ -31,84 +31,10 @@ def get_request_aggregations(self, params, args): """Disable generating aggregations""" return None - def _get_filters(self, repos, invisible_stages): - return None - - def get_elastic_index(self, types): - return 'statistics' - - def _get_es_query_funcs(self): - funcs = super()._get_es_query_funcs() - - funcs['user_locks'] = { - 'query': self._es_filter_user_locks - } - - return funcs - def _es_set_size(self, query, params): """Disable setting the size""" pass - def _es_filter_user_locks(self, query, user, must, params): - lt, gte, time_zone = self._es_get_date_filters(params) - - query[must].append({ - 'nested': { - 'path': 'stats.timeline', - 'query': { - 'bool': { - 'must': [ - {'term': {'stats.timeline.task.user': user}}, - {'terms': { - 'stats.timeline.operation': [ - 'item_lock', - 'item_unlock' - ] - }}, - {'range': { - 'stats.timeline.operation_created': { - 'gte': gte, - 'lt': lt, - 'time_zone': time_zone - } - }} - ] - } - } - } - }) - - def _es_filter_desks(self, query, desks, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.timeline', - 'query': { - 'terms': {'stats.timeline.task.desk': desks} - } - } - }) - - def _es_filter_users(self, query, users, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.timeline', - 'query': { - 'terms': {'stats.timeline.task.user': users} - } - } - }) - - def _es_filter_stages(self, query, stages, must, params): - query[must].append({ - 'nested': { - 'path': 'stats.timeline', - 'query': { - 'terms': {'stats.timeline.task.stage': stages} - } - } - }) - def generate_report(self, docs, args): report = { 'items': [], diff --git a/server/features/base_report.feature b/server/features/base_report.feature index b846889fb..42ee71ab0 100644 --- a/server/features/base_report.feature +++ b/server/features/base_report.feature @@ -278,3 +278,65 @@ Feature: Base Analytics Report Service }] } """ + + @auth + Scenario: Paginate response + Given "archived" + """ + [ + { + "_id": "archive1", "_type": "archived", "source": "AAP", + "task": {"stage": "5b501a511d41c84c0bfced4b", "desk": "5b501a501d41c84c0bfced4a"}, + "anpa_category": [{"qcode": "A", "name": "Advisories"}, {"qcode": "T", "name": "Transport"}] + }, + { + "_id": "archive2", "_type": "archived", "source": "AAP", + "task": {"stage": "5b501a6f1d41c84c0bfced4c", "desk": "5b501a501d41c84c0bfced4a"}, + "anpa_category": [{"qcode": "A", "name": "Advisories"}] + }, + { + "_id": "archive3", "_type": "archived", "source": "AAP", + "task": {"stage": "5b501a511d41c84c0bfced4b", "desk": "5b501a501d41c84c0bfced4a"}, + "anpa_category": [{"qcode": "A", "name": "Advisories"}, {"qcode": "T", "name": "Transport"}] + }, + { + "_id": "archive4", "_type": "archived", "source": "AAP", + "task": {"stage": "5b501a6f1d41c84c0bfced4c", "desk": "5b501a501d41c84c0bfced4a"}, + "anpa_category": [{"qcode": "A", "name": "Advisories"}] + }, + { + "_id": "archive5", "_type": "archived", "source": "AAP", + "task": {"stage": "5b501a511d41c84c0bfced4b", "desk": "5b501a501d41c84c0bfced4a"}, + "anpa_category": [{"qcode": "A", "name": "Advisories"}, {"qcode": "T", "name": "Transport"}] + }, + { + "_id": "archive6", "_type": "archived", "source": "AAP", + "task": {"stage": "5b501a6f1d41c84c0bfced4c", "desk": "5b501a501d41c84c0bfced4a"}, + "anpa_category": [{"qcode": "A", "name": "Advisories"}] + } + ] + """ + When we get "/analytics_test_report?params={"must": {}, "size": 2, "page": 1, "sort": [{"_id": "asc"}]}&aggs=0&max_results=2" + Then we get list with 6 items + """ + { + "_items": [{"_id": "archive1"}, {"_id": "archive2"}], + "_meta": {"total": 6, "page": 1, "max_results": 2} + } + """ + When we get "/analytics_test_report?params={"must": {}, "size": 2, "page": 2, "sort": [{"_id": "asc"}]}&aggs=0&max_results=2&page=2" + Then we get list with 6 items + """ + { + "_items": [{"_id": "archive3"}, {"_id": "archive4"}], + "_meta": {"total": 6, "page": 2, "max_results": 2} + } + """ + When we get "/analytics_test_report?params={"must": {}, "size": 2, "page": 3, "sort": [{"_id": "asc"}]}&aggs=0&max_results=2&page=3" + Then we get list with 6 items + """ + { + "_items": [{"_id": "archive5"}, {"_id": "archive6"}], + "_meta": {"total": 6, "page": 3, "max_results": 2} + } + """