Update Content publishing reports Group to support custom vocabularie…

…s fields [SDESK-7268] (#151)

* update backend to support custom vocabularies fields

* udpate frontend

* address comment

* minor updatation

client/content_publishing_report/controllers/ContentPublishingReportController.js

@@ -3,6 +3,7 @@ import {appConfig} from 'appConfig';
 import {getErrorMessage, gettext} from '../../utils';
 import {CHART_TYPES} from '../../charts/directives/ChartOptions';
 import {searchReportService} from '../../search/services/SearchReport';
+import {superdeskApi} from '../../superdeskApi';
 
 ContentPublishingReportController.$inject = [
     '$scope',
@@ -85,7 +86,7 @@ export function ContentPublishingReportController(
         );
 
         $scope.report_groups = searchReportService.filterDataFields(
-            ['anpa_category.qcode', 'genre.qcode', 'source', 'urgency', 'subject.qcode']
+            ['anpa_category.qcode', 'genre.qcode', 'source', 'urgency', 'subject.qcode', 'authors.parent', 'language']
         );
 
         $scope.currentParams = {
@@ -118,7 +119,8 @@ export function ContentPublishingReportController(
 
         $scope.defaultReportParams = _.cloneDeep($scope.currentParams);
 
-        $scope.group_by = _.cloneDeep($scope.report_groups);
+        $scope.group_by = $scope.group_by = _.cloneDeep($scope.report_groups);
+
         $scope.updateGroupOptions();
     };
 
@@ -177,6 +179,14 @@ export function ContentPublishingReportController(
      * @description Updates the available group/subgroup options when the user changes the group
      */
     $scope.updateGroupOptions = () => {
+        const excludedSchemes = new Set(['languages', 'author_roles', 'genre']);
+
+        const customVocab = getCustomVocabulariesData().filter(
+            (item) => !excludedSchemes.has(item.qcode.scheme));
+
+        $scope.group_by = [...$scope.report_groups, ...customVocab.filter(
+            (item) => !new Set($scope.report_groups.map((item) => item.name)).has(item.name))];
+
         $scope.subgroup_by = _.filter(
             $scope.report_groups,
             (group) => group.qcode !== $scope.currentParams.params.aggs.group.field
@@ -342,7 +352,15 @@ export const generateTitle = (chart, params) => {
         return params.chart.title;
     }
 
-    const parentField = _.get(params, 'aggs.group.field');
+    let parentField = _.get(params, 'aggs.group.field');
+
+    if (parentField.startsWith('{"scheme')) {
+        const obj = JSON.parse(parentField);
+
+        parentField = getCustomVocabulariesData().find(
+            (value) => value.qcode.scheme == obj.scheme)?.name;
+    }
+
     const parentName = chart.getSourceName(parentField);
 
     if (_.get(params, 'aggs.subgroup.field.length', 0) > 0) {
@@ -357,3 +375,9 @@ export const generateTitle = (chart, params) => {
 
     return gettext('Published Stories per {{group}}', {group: parentName});
 };
+
+export function getCustomVocabulariesData() {
+    return superdeskApi.entities.vocabulary.getCustomVocabulary().map((data) => {
+        return {'name': data['display_name'], 'qcode': {'scheme': data['_id']}};
+    });
+}

client/interfaces.ts

@@ -40,6 +40,7 @@ export enum DATA_FIELD {
     SUBJECT = 'subject.qcode',
     AUTHOR = 'authors.parent',
     STATE = 'state',
+    LANGUAGE = 'language'
 }
 
 export enum REPORT_RESPONSE_TYPE {
@@ -125,6 +126,7 @@ export interface IReportParams extends IBaseReportParams {
     };
     repos?: IReposFilter;
     return_type?: REPORT_RESPONSE_TYPE;
+    show_all_desks?: number;
 }
 
 export interface IReportPayload {
@@ -136,6 +138,7 @@ export interface IReportPayload {
     max_results?: number;
     page?: number;
     sort?: Array<{ [key: string]: string }>;
+    show_all_desks?: number;
 }
 
 export interface IAnalyticsConfig extends ISuperdeskGlobalConfig {

client/search/services/SearchReport.ts

@@ -121,6 +121,9 @@ function getSupportedDataFields(): Array<IDataFilter> {
     }, {
         qcode: DATA_FIELD.STATE,
         name: gettext('State'),
+    }, {
+        qcode: DATA_FIELD.LANGUAGE,
+        name: gettext('Language'),
     }];
 }
 

server/analytics/base_report/__init__.py

@@ -102,6 +102,12 @@ def generate_html(self, docs, args):
         """
         return {}
 
+    def get_custom_aggs_query(self, query, aggs):
+        """
+        Overwrite this method to update the query
+        """
+        return {}
+
     def get_aggregation_buckets(self, docs, aggregation_ids=None):
         """
         Retrieves the aggregation buckets from the documents provided
@@ -274,6 +280,8 @@ def run_query(self, params, args):
 
         index = self.get_elastic_index(types)
 
+        self.get_custom_aggs_query(query, aggs)
+
         docs = self.elastic.search(query, types, params={})
 
         for resource in types:

server/analytics/content_publishing_report/content_publishing_report.py

@@ -11,6 +11,7 @@
 from analytics.base_report import BaseReportService, BaseReportResource
 from analytics.chart_config import ChartConfig
 from analytics.common import MAX_TERMS_SIZE
+import json
 
 
 class ContentPublishingReportResource(BaseReportResource):
@@ -90,7 +91,11 @@ def generate_report(self, docs, args):
 
             report["groups"][parent_key] = {}
 
-            for child in (parent.get("child") or {}).get("buckets") or []:
+            for child in (
+                (parent.get("child") or {}).get("buckets")
+                or (parent.get("child_aggs", {}).get("child", {}).get("buckets"))
+                or []
+            ):
                 child_key = child.get("key")
 
                 if not child_key:
@@ -151,3 +156,50 @@ def gen_subtitle():
         report["highcharts"] = [chart_config.gen_config()]
 
         return report
+
+    def get_aggregation_buckets(self, docs, aggregation_ids=None):
+        buckets = {}
+        filtered_subjects = docs.get("aggregations", {}).get("parent", {}).get("qcode_filter")
+        if filtered_subjects:
+            buckets[aggregation_ids[0]] = filtered_subjects.get("qcode_terms", {}).get("buckets") or []
+            return buckets
+
+        return super().get_aggregation_buckets(docs, aggregation_ids)
+
+    def get_custom_aggs_query(self, query, aggs):
+        field = self.parse_field_param(aggs.get("parent", {}).get("terms", {}).get("field"))
+        if field:
+            # Construct the nested aggregation query
+            qcode_terms_agg = {"terms": {"field": "subject.qcode", "size": 1000}}
+
+            # Retrieve child aggregations if any
+            child = self.get_child_aggs(query)
+            if child:
+                qcode_terms_agg["aggs"] = {"child_aggs": {"reverse_nested": {}, "aggs": child}}
+
+            query["aggs"]["parent"] = {
+                "nested": {"path": "subject"},
+                "aggs": {
+                    "qcode_filter": {
+                        "filter": {"term": {"subject.scheme": field}},
+                        "aggs": {"qcode_terms": qcode_terms_agg},
+                    }
+                },
+            }
+
+    def get_child_aggs(self, query):
+        parent_aggs = query.get("aggs", {}).get("parent", {})
+        if "aggs" in parent_aggs:
+            return parent_aggs["aggs"]
+        return {}
+
+    def parse_field_param(self, field_param):
+        try:
+            # Check if the field_param is a JSON string and parse it
+            field_dict = json.loads(field_param)
+            if isinstance(field_dict, dict) and "scheme" in field_dict:
+                return field_dict["scheme"]
+        except json.JSONDecodeError:
+            pass
+
+        return None