Add new flag to filter findings in Turbinia reports according to the …

…finding priority. Pump turbinia-client to 1.0.7 and fix some mypy and linter findings (#904)

* Update turbinia_gcp.py

* Update turbinia_base.py

* Update aws_turbinia_ts.json

* Update gcp_turbinia_ts.json

* Update gcp_turbinia_disk_copy_ts.json

* Update turbinia_gcp.py

* Update turbinia_base.py

* Update turbinia_gcp.py

* Update turbinia_base.py

* Update turbinia_base.py

* Update turbinia_gcp.py

* pump turbinia-client to version 1.0.7

* some linter and mypy fixes

data/recipes/aws_turbinia_ts.json

@@ -72,7 +72,8 @@
         "turbinia_zone": "@turbinia_zone",
         "turbinia_api": "@turbinia_api",
         "incident_id": "@incident_id",
-        "sketch_id": "@sketch_id"
+        "sketch_id": "@sketch_id",
+        "priority_filter": "@priority_filter"
       }
     }, {
       "wants": ["TurbiniaGCPProcessor"],
@@ -109,6 +110,7 @@
     ["--turbinia_zone", "Zone Turbinia is located in", "us-central1-f", {"format": "gcp_zone"}],
     ["--turbinia_auth", "Flag to indicate whether Turbinia API server requires authentication.", false],
     ["--turbinia_api", "Turbinia API server endpoint.", "http://127.0.0.1:8000"],
-    ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true]
+    ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
+    ["--priority_filter", "Filter report findings, range from 0 to 100, 0 is the highest.", "100", {"format": "integer"}]
   ]
 }

data/recipes/gcp_turbinia_disk_copy_ts.json

@@ -63,7 +63,8 @@
             "turbinia_zone": "@turbinia_zone",
             "turbinia_api": "@turbinia_api",
             "incident_id": "@incident_id",
-            "sketch_id": "@sketch_id"
+            "sketch_id": "@sketch_id",
+            "priority_filter": "@priority_filter"
         }
     }, {
         "wants": ["TurbiniaGCPProcessor"],
@@ -104,6 +105,7 @@
         ["--boot_disk_size", "The size of the analysis VM boot disk (in GB).", "50", {"format": "integer"}],
         ["--boot_disk_type", "Disk type to use [pd-standard, pd-ssd]", "pd-standard", {"format": "regex", "regex": "^pd-((ssd)|(standard))$"}],
         ["--image_project", "Name of the project where the analysis VM image is hosted.", "ubuntu-os-cloud"],
-        ["--image_family", "Name of the image to use to create the analysis VM.", "ubuntu-2204-lts"]
+        ["--image_family", "Name of the image to use to create the analysis VM.", "ubuntu-2204-lts"],
+        ["--priority_filter", "Filter report findings, range from 0 to 100, 0 is the highest.", "100", {"format": "integer"}]
     ]
 }

data/recipes/gcp_turbinia_ts.json

@@ -22,7 +22,8 @@
             "turbinia_zone": "@turbinia_zone",
             "turbinia_api": "@turbinia_api",
             "incident_id": "@incident_id",
-            "sketch_id": "@sketch_id"
+            "sketch_id": "@sketch_id",
+            "priority_filter": "@priority_filter"
         }
     }, {
         "wants": ["TurbiniaGCPProcessor"],
@@ -52,6 +53,7 @@
         ["--turbinia_recipe", "The Turbinia recipe name to use for evidence processing.", null],
         ["--turbinia_auth", "Flag to indicate whether Turbinia API server requires authentication.", false],
         ["--turbinia_api", "Turbinia API server endpoint.", "http://127.0.0.1:8000"],
-        ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true]
+        ["--wait_for_timelines", "Whether to wait for Timesketch to finish processing all timelines.", true],
+        ["--priority_filter", "Filter report findings, range from 0 to 100, 0 is the highest.", "100", {"format": "integer"}]
     ]
 }

dftimewolf/lib/collectors/gcp_logging.py

@@ -23,7 +23,7 @@
 
 def _CustomToAPIRepr(self: entries.ProtobufEntry) -> Dict[str, Any]:
   """API repr (JSON format) for entry."""
-  info = super(entries.ProtobufEntry, self).to_api_repr()  # type: ignore
+  info = super(entries.ProtobufEntry, self).to_api_repr()
   info['protoPayload'] = self.payload  # type: ignore
   return info  # type: ignore
 
@@ -62,9 +62,9 @@ def SetupLoggingClient(self) -> Any:
       logging.Client: A GCP logging client
     """
     if self._project_name:
-      return logging.Client(_use_grpc=False,  # type: ignore
+      return logging.Client(_use_grpc=False,
                                         project=self._project_name)
-    return logging.Client(_use_grpc=False)  # type: ignore
+    return logging.Client(_use_grpc=False)
 
   def ListPages(self, logging_client: Any) -> Any:
     """Returns pages based on a Cloud Logging filter

dftimewolf/lib/collectors/gsheets.py

@@ -39,7 +39,7 @@ def __init__(
     super(GoogleSheetsCollector, self).__init__(
         state, name=name, critical=critical)
     self._sheets_resource = None
-    self._credentials = None
+    self._credentials: Optional[Credentials] = None
     self._spreadsheet_id = ''
     self._sheet_names: List[str] = []
     # These are mandatory columns required by Timesketch.
@@ -127,13 +127,13 @@ def Process(self) -> None:
           '$ gcloud auth application-default login')
       self.ModuleError(str(exception), critical=True)
 
-  def _GetCredentials(self) -> Credentials:
+  def _GetCredentials(self) -> Optional[Credentials]:
     """Obtains API credentials for accessing the Google Sheets API.
 
     Returns:
       google.oauth2.credentials.Credentials: Google API credentials.
     """
-    credentials = None
+    credentials: Optional[Credentials] = None
 
     # The credentials file stores the user's access and refresh tokens, and is
     # created automatically when the authorization flow completes for the first
@@ -169,8 +169,9 @@ def _GetCredentials(self) -> Credentials:
           credentials = flow.run_console()
 
         # Save the credentials for the next run
-        with open(credentials_path, 'w') as token_file:
-          token_file.write(credentials.to_json())
+        if credentials:
+          with open(credentials_path, 'w') as token_file:
+            token_file.write(credentials.to_json())
 
     return credentials
 

dftimewolf/lib/collectors/workspace_audit.py

@@ -40,14 +40,15 @@ def __init__(self,
     """Initializes a Workspace Audit Log collector."""
     super(WorkspaceAuditCollector, self).__init__(state, name=name,
         critical=critical)
-    self._credentials = None
+    self._credentials: Optional[Credentials] = None
     self._application_name = ''
     self._filter_expression = ''
     self._user_key = 'all'
     self._start_time = None  # type: Optional[datetime.datetime]
     self._end_time = None # type: Optional[datetime.datetime]
 
-  def _BuildAuditResource(self, credentials: Credentials) -> discovery.Resource:
+  def _BuildAuditResource(self, credentials: Optional[Credentials]
+                          ) -> discovery.Resource:
     """Builds a reports resource object to use to request logs.
 
     Args:
@@ -59,13 +60,13 @@ def _BuildAuditResource(self, credentials: Credentials) -> discovery.Resource:
     service = discovery.build('admin', 'reports_v1', credentials=credentials)
     return service
 
-  def _GetCredentials(self) -> Credentials:
+  def _GetCredentials(self) -> Optional[Credentials]:
     """Obtains API credentials for accessing the Workspace audit API.
 
     Returns:
       google.oauth2.credentials.Credentials: Google API credentials.
     """
-    credentials = None
+    credentials: Optional[Credentials] = None
 
     # The credentials file stores the user's access and refresh tokens, and is
     # created automatically when the authorization flow completes for the first
@@ -103,8 +104,9 @@ def _GetCredentials(self) -> Credentials:
           credentials = flow.run_local_server()
 
         # Save the credentials for the next run
-        with open(credentials_path, 'w') as token_file:
-          token_file.write(credentials.to_json())
+        if credentials:
+          with open(credentials_path, 'w') as token_file:
+            token_file.write(credentials.to_json())
 
     return credentials
 

dftimewolf/lib/processors/turbinia_base.py

@@ -53,6 +53,8 @@ class TurbiniaProcessorBase(module.BaseModule):
     turbinia_zone (str): GCP zone in which the Turbinia server is running.
     turbinia_api (str): Turbinia API endpoint.
     turbinia_auth (bool): Turbinia auth flag.
+    priority_filter (int): Filter report findings, range from 0 to 100,
+        0 is the highest.
     parallel_count (int): Number of threads to use.
   """
 
@@ -102,6 +104,7 @@ def __init__(
     self.sketch_id = int()
     self.state = state
     self.turbinia_auth: bool = False
+    self.priority_filter = int()
     self.turbinia_recipe = str()  # type: Any
     self.turbinia_region = None
     self.turbinia_zone = str()
@@ -281,7 +284,7 @@ def GetCredentials(self, credentials_path: str,
                      client_secrets_path: str) -> Optional[Credentials]:
     """Authenticates the user using Google OAuth services."""
     scopes = ['openid', 'https://www.googleapis.com/auth/userinfo.email']
-    credentials = None
+    credentials: Optional[Credentials] = None
 
     # Load credentials file if it exists
     if os.path.exists(credentials_path):
@@ -375,7 +378,7 @@ def InitializeTurbiniaApiClient(
 
   def TurbiniaSetUp(
       self, project: str, turbinia_recipe: Union[str, None], turbinia_zone: str,
-      turbinia_api: str, incident_id: str, sketch_id: int,
+      turbinia_api: str, incident_id: str, sketch_id: int, priority_filter: int,
       turbinia_auth: bool = False) -> None:
     """Sets up the object attributes.
 
@@ -386,10 +389,13 @@ def TurbiniaSetUp(
       turbinia_zone (str): GCP zone in which the Turbinia server is running.
       turbinia_api (str): URL of the Turbinia API server.
       incident_id (str): The incident ID.
+      priority_filter (int): Filter report findings, range from 0 to 100,
+          0 is the highest.
       sketch_id (int): The sketch ID.
     """
     self.project = project
     self.turbinia_auth = turbinia_auth
+    self.priority_filter = priority_filter
     self.turbinia_api = turbinia_api
     self.turbinia_recipe = turbinia_recipe
     self.turbinia_zone = turbinia_zone
@@ -573,7 +579,9 @@ def TurbiniaWait(self,
         self.logger.warning(f'Retrying after 3 seconds: {exception.body}')
         time.sleep(3)
 
-  def TurbiniaFinishReport(self, request_id: str) -> Optional[str]:
+  def TurbiniaFinishReport(self,
+                           request_id: str,
+                           priority_filter: int) -> Optional[str]:
     """This method generates a report for a Turbinia request."""
     # Refresh token if needed
     if self.RefreshClientCredentials():
@@ -587,6 +595,6 @@ def TurbiniaFinishReport(self, request_id: str) -> Optional[str]:
     if request_data:
       report: str = turbinia_formatter.RequestMarkdownReport(
           request_data=request_data
-      ).generate_markdown()
+      ).generate_markdown(priority_filter)
       return report
     return None

dftimewolf/lib/processors/turbinia_gcp.py

@@ -142,6 +142,7 @@ def GetTelemetryEntry(self) -> Dict[str, str]:
     return telemetry_entry
 
   # pylint: disable=arguments-differ
+  # pylint: disable=too-many-arguments
   def SetUp(
       self,
       project: str,
@@ -152,7 +153,8 @@ def SetUp(
       sketch_id: int,
       request_ids: str = '',
       disk_names: str = '',
-      turbinia_auth: bool = False) -> None:
+      turbinia_auth: bool = False,
+      priority_filter: int = 100) -> None:
     """Sets up the object attributes.
 
     Args:
@@ -165,7 +167,8 @@ def SetUp(
       incident_id (str): The incident ID.
       sketch_id (int): The sketch ID.
       request_ids (str): Turbinia requests for jobs being processed.
-      disk_names (str): Names of the disks to process.
+      priority_filter (int): Filter report findings, range from 0 to 100,
+          0 is the highest.
     """
 
     if (disk_names and request_ids):
@@ -193,7 +196,8 @@ def SetUp(
 
     self.TurbiniaSetUp(
         project, turbinia_recipe, turbinia_zone, turbinia_api,
-        incident_id, int(sketch_id) if sketch_id else 0, turbinia_auth)
+        incident_id, int(sketch_id) if sketch_id else 0, priority_filter,
+        turbinia_auth)
 
   def PreProcess(self) -> None:
     """Ensures containers from previous modules are processed.
@@ -271,7 +275,7 @@ def Process(self, request_container: containers.TurbiniaRequest) -> None:
               f'additional information. {exception}')
           self.logger.error(message)
     # Generate a Turbinia report and store it in the state.
-    report = self.TurbiniaFinishReport(request_id)
+    report = self.TurbiniaFinishReport(request_id, self.priority_filter)
 
     # Stop profiler
     self.profiler.disable()

mypy.ini

@@ -4,3 +4,4 @@ disable_error_code = override
 ignore_missing_imports = True
 strict = True
 exclude = _pb2.py|tests
+disallow_untyped_calls = False