Merge pull request #6504 from bshankar/feature/6500-project-summary-t…

…able-view

Project summary table view

backend/api/projects/resources.py

@@ -10,6 +10,7 @@
     ProjectSearchDTO,
     ProjectSearchBBoxDTO,
 )
+from backend.models.postgis.statuses import UserRole
 from backend.services.project_search_service import (
     ProjectSearchService,
     ProjectSearchServiceError,
@@ -481,6 +482,10 @@ def setup_search_dto(self) -> ProjectSearchDTO:
         search_dto.last_updated_lte = request.args.get("lastUpdatedTo")
         search_dto.created_gte = request.args.get("createdFrom")
         search_dto.created_lte = request.args.get("createdTo")
+        search_dto.partner_id = request.args.get("partnerId")
+        search_dto.partnership_from = request.args.get("partnershipFrom")
+        search_dto.partnership_to = request.args.get("partnershipTo")
+        search_dto.download_as_csv = request.args.get("downloadAsCSV")
 
         # See https://github.com/hotosm/tasking-manager/pull/922 for more info
         try:
@@ -550,7 +555,7 @@ def get(self):
               name: orderBy
               type: string
               default: priority
-              enum: [id,difficulty,priority,status,last_updated,due_date]
+              enum: [id,difficulty,priority,status,last_updated,due_date,percent_mapped,percent_validated]
             - in: query
               name: orderByType
               type: string
@@ -655,9 +660,33 @@ def get(self):
               type: boolean
               description: If true, it will not return the project centroid's geometries.
               default: false
+            - in: query
+              name: partnerId
+              type: int
+              description: Limit to projects currently linked to a specific partner ID
+              default: 1
+            - in: query
+              name: partnershipFrom
+              type: date
+              description: Limit to projects with partners that began greater than or equal to a date
+              default: "2017-04-11"
+            - in: query
+              name: partnershipTo
+              type: date
+              description: Limit to projects with partners that ended less than or equal to a date
+              default: "2018-04-11"
+            - in: query
+              name: downloadAsCSV
+              type: boolean
+              description: Set to true to download search results as a CSV
+              default: false
         responses:
             200:
                 description: Projects found
+            400:
+                description: Bad input.
+            401:
+                description: Search parameters partnerId, partnershipFrom, partnershipTo are not allowed for this user.
             404:
                 description: No projects found
             500:
@@ -668,7 +697,57 @@ def get(self):
             user_id = token_auth.current_user()
             if user_id:
                 user = UserService.get_user_by_id(user_id)
+
             search_dto = self.setup_search_dto()
+
+            if search_dto.omit_map_results and search_dto.download_as_csv:
+                return {
+                    "Error": "omitMapResults and downloadAsCSV cannot be both set to true"
+                }, 400
+
+            if (
+                search_dto.partnership_from is not None
+                or search_dto.partnership_to is not None
+            ) and search_dto.partner_id is None:
+                return {
+                    "Error": "partnershipFrom or partnershipTo cannot be provided without partnerId"
+                }, 400
+
+            if (
+                search_dto.partner_id is not None
+                and search_dto.partnership_from is not None
+                and search_dto.partnership_to is not None
+                and search_dto.partnership_from > search_dto.partnership_to
+            ):
+                return {
+                    "Error": "partnershipFrom cannot be greater than partnershipTo"
+                }, 400
+
+            if any(
+                map(
+                    lambda x: x is not None,
+                    [
+                        search_dto.partner_id,
+                        search_dto.partnership_from,
+                        search_dto.partnership_to,
+                    ],
+                )
+            ) and (user is None or not user.role == UserRole.ADMIN.value):
+                error_msg = "Only admins can search projects by partnerId, partnershipFrom, partnershipTo"
+                return {"Error": error_msg}, 401
+
+            if search_dto.download_as_csv:
+                all_results_csv = ProjectSearchService.search_projects_as_csv(
+                    search_dto, user
+                )
+
+                return send_file(
+                    io.BytesIO(all_results_csv.encode()),
+                    mimetype="text/csv",
+                    as_attachment=True,
+                    download_name="projects_search_result.csv",
+                )
+
             results_dto = ProjectSearchService.search_projects(search_dto, user)
             return results_dto.to_primitive(), 200
         except NotFound:

backend/models/dtos/project_dto.py

@@ -330,6 +330,10 @@ class ProjectSearchDTO(Model):
     last_updated_gte = StringType(required=False)
     created_lte = StringType(required=False)
     created_gte = StringType(required=False)
+    partner_id = IntType(required=False)
+    partnership_from = StringType(required=False)
+    partnership_to = StringType(required=False)
+    download_as_csv = BooleanType(required=False)
 
     def __hash__(self):
         """Make object hashable so we can cache user searches"""
@@ -402,6 +406,11 @@ class ListSearchResultDTO(Model):
     total_contributors = IntType(serialized_name="totalContributors")
     country = StringType(serialize_when_none=False)
 
+    creation_date = UTCDateTimeType(serialized_name="creationDate", required=True)
+    author = StringType(serialize_when_none=False)
+    partner_names = ListType(StringType, serialized_name="partnerNames")
+    total_area = FloatType(required=True, serialized_name="totalAreaSquareKilometers")
+
 
 class ProjectSearchResultsDTO(Model):
     """Contains all results for the search criteria"""

backend/models/postgis/project.py

@@ -12,6 +12,8 @@
 from sqlalchemy import desc, func, Time, orm, literal
 from shapely.geometry import shape
 from sqlalchemy.dialects.postgresql import ARRAY
+from sqlalchemy.ext.hybrid import hybrid_property
+
 import requests
 
 from backend import db
@@ -199,6 +201,19 @@ class Project(db.Model):
     tasks_validated = db.Column(db.Integer, default=0, nullable=False)
     tasks_bad_imagery = db.Column(db.Integer, default=0, nullable=False)
 
+    # Total tasks are always >= 1
+    @hybrid_property
+    def percent_mapped(self):
+        return (
+            (self.tasks_mapped + self.tasks_validated)
+            / (self.total_tasks - self.tasks_bad_imagery)
+            * 100
+        )
+
+    @hybrid_property
+    def percent_validated(self):
+        return self.tasks_validated / (self.total_tasks - self.tasks_bad_imagery) * 100
+
     # Mapped Objects
     tasks = db.relationship(
         Task, backref="projects", cascade="all, delete, delete-orphan", lazy="dynamic"
@@ -224,6 +239,7 @@ class Project(db.Model):
     interests = db.relationship(
         Interest, secondary=project_interests, backref="projects"
     )
+    partnerships = db.relationship("ProjectPartnership", backref="project")
 
     def create_draft_project(self, draft_project_dto: DraftProjectDTO):
         """

backend/services/project_search_service.py

@@ -1,3 +1,4 @@
+import pandas as pd
 from flask import current_app
 import math
 import geojson
@@ -17,6 +18,7 @@
     ProjectSearchBBoxDTO,
 )
 from backend.models.postgis.project import Project, ProjectInfo, ProjectTeams
+from backend.models.postgis.partner import Partner
 from backend.models.postgis.statuses import (
     ProjectStatus,
     MappingLevel,
@@ -28,6 +30,7 @@
     MappingPermission,
     ProjectDifficulty,
 )
+from backend.models.postgis.project_partner import ProjectPartnership
 from backend.models.postgis.campaign import Campaign
 from backend.models.postgis.organisation import Organisation
 from backend.models.postgis.task import TaskHistory
@@ -40,8 +43,8 @@
 from backend.models.postgis.interests import project_interests
 from backend.services.users.user_service import UserService
 
-
 search_cache = TTLCache(maxsize=128, ttl=300)
+csv_download_cache = TTLCache(maxsize=16, ttl=600)
 
 # max area allowed for passed in bbox, calculation shown to help future maintenance
 # client resolution (mpp)* arbitrary large map size on a large screen in pixels * 50% buffer, all squared
@@ -116,7 +119,13 @@ def create_search_query(user=None):
         return query
 
     @staticmethod
-    def create_result_dto(project, preferred_locale, total_contributors):
+    def create_result_dto(
+        project: Project,
+        preferred_locale: str,
+        total_contributors: int,
+        with_partner_names: bool = False,
+        with_author_name: bool = True,
+    ) -> ListSearchResultDTO:
         project_info_dto = ProjectInfo.get_dto_for_locale(
             project.id, preferred_locale, project.default_locale
         )
@@ -144,6 +153,27 @@ def create_result_dto(project, preferred_locale, total_contributors):
         list_dto.organisation_logo = project.organisation_logo
         list_dto.campaigns = Project.get_project_campaigns(project.id)
 
+        list_dto.creation_date = project_obj.created
+
+        if with_author_name:
+            list_dto.author = project_obj.author.name or project_obj.author.username
+
+        if with_partner_names:
+            list_dto.partner_names = list(
+                set(
+                    map(
+                        lambda p: Partner.get_by_id(p.partner_id).name,
+                        project_obj.partnerships,
+                    )
+                )
+            )
+
+        # Use postgis to compute the total area of the geometry in square kilometers
+        list_dto.total_area = project_obj.query.with_entities(
+            func.coalesce(func.sum(func.ST_Area(project_obj.geometry, True) / 1000000))
+        ).scalar()
+        list_dto.total_area = round(list_dto.total_area, 3)
+
         return list_dto
 
     @staticmethod
@@ -169,6 +199,40 @@ def get_total_contributions(paginated_results):
 
         return [p.total for p in project_contributors_count]
 
+    @staticmethod
+    @cached(csv_download_cache)
+    def search_projects_as_csv(search_dto: ProjectSearchDTO, user) -> str:
+        all_results, _ = ProjectSearchService._filter_projects(search_dto, user)
+        is_user_admin = user is not None and user.role == UserRole.ADMIN.value
+        results_as_dto = [
+            ProjectSearchService.create_result_dto(
+                p,
+                search_dto.preferred_locale,
+                Project.get_project_total_contributions(p[0]),
+                with_partner_names=is_user_admin,
+                with_author_name=False,
+            ).to_primitive()
+            for p in all_results
+        ]
+
+        df = pd.json_normalize(results_as_dto)
+        columns_to_drop = [
+            "locale",
+            "shortDescription",
+            "organisationLogo",
+            "campaigns",
+        ]
+        if not is_user_admin:
+            columns_to_drop.append("partnerNames")
+
+        df.drop(
+            columns=columns_to_drop,
+            inplace=True,
+            axis=1,
+        )
+
+        return df.to_csv(index=False)
+
     @staticmethod
     @cached(search_cache)
     def search_projects(search_dto: ProjectSearchDTO, user) -> ProjectSearchResultsDTO:
@@ -185,6 +249,10 @@ def search_projects(search_dto: ProjectSearchDTO, user) -> ProjectSearchResultsD
                 p,
                 search_dto.preferred_locale,
                 Project.get_project_total_contributions(p[0]),
+                with_partner_names=(
+                    user is not None and user.role == UserRole.ADMIN.value
+                ),
+                with_author_name=True,
             )
             for p in paginated_results.items
         ]
@@ -344,11 +412,40 @@ def _filter_projects(search_dto: ProjectSearchDTO, user):
             created_lte = validate_date_input(search_dto.created_lte)
             query = query.filter(Project.created <= created_lte)
 
+        if search_dto.partner_id:
+            query = query.join(
+                ProjectPartnership, ProjectPartnership.project_id == Project.id
+            ).filter(ProjectPartnership.partner_id == search_dto.partner_id)
+
+            if search_dto.partnership_from:
+                partnership_from = validate_date_input(search_dto.partnership_from)
+                query = query.filter(ProjectPartnership.started_on <= partnership_from)
+
+            if search_dto.partnership_to:
+                partnership_to = validate_date_input(search_dto.partnership_to)
+                query = query.filter(
+                    (ProjectPartnership.ended_on.is_(None))
+                    | (ProjectPartnership.ended_on >= partnership_to)
+                )
+
         order_by = search_dto.order_by
-        if search_dto.order_by_type == "DESC":
-            order_by = desc(search_dto.order_by)
 
-        query = query.order_by(order_by).distinct(search_dto.order_by, Project.id)
+        if search_dto.order_by == "percent_mapped":
+            if search_dto.order_by_type == "DESC":
+                order_by = Project.percent_mapped.desc()
+            else:
+                order_by = Project.percent_mapped.asc()
+            query = query.order_by(order_by)
+        elif search_dto.order_by == "percent_validated":
+            if search_dto.order_by_type == "DESC":
+                order_by = Project.percent_validated.desc()
+            else:
+                order_by = Project.percent_validated.asc()
+            query = query.order_by(order_by)
+        else:
+            if search_dto.order_by_type == "DESC":
+                order_by = desc(search_dto.order_by)
+            query = query.order_by(order_by).distinct(search_dto.order_by, Project.id)
 
         if search_dto.managed_by and user.role != UserRole.ADMIN.value:
             # Get all the projects associated with the user and team.

frontend/package.json

@@ -15,6 +15,7 @@
     "@sentry/react": "^7.102.0",
     "@tanstack/react-query": "^4.29.7",
     "@tanstack/react-query-devtools": "^4.29.7",
+    "@tanstack/react-table": "^8.20.1",
     "@tmcw/togeojson": "^5.8.1",
     "@turf/area": "^6.5.0",
     "@turf/bbox": "^6.5.0",

frontend/src/api/projects.js

@@ -193,6 +193,21 @@ export const submitValidationTask = (projectId, payload, token, locale) => {
   );
 };
 
+export const downloadAsCSV = (allQueryParams, action, token) => {
+  const paramsRemapped = remapParamsToAPI(allQueryParams, backendToQueryConversion);
+  // it's needed in order to query by action
+  if (paramsRemapped.action === undefined && action) {
+    paramsRemapped.action = action;
+  }
+
+  if (paramsRemapped.lastUpdatedTo) {
+    paramsRemapped.lastUpdatedTo = format(subMonths(Date.now(), 6), 'yyyy-MM-dd');
+  }
+  return api(token).get('projects/', {
+    params: paramsRemapped,
+  });
+};
+
 export const useAvailableCountriesQuery = () => {
   const fetchGeojsonData = () => {
     return axios.get(`${UNDERPASS_URL}/availability`);
@@ -239,4 +254,7 @@ const backendToQueryConversion = {
   stale: 'lastUpdatedTo',
   createdFrom: 'createdFrom',
   basedOnMyInterests: 'basedOnMyInterests',
+  partnerId: 'partnerId',
+  partnershipFrom: 'partnershipFrom',
+  partnershipTo: 'partnershipTo',
 };