Refresh rules at runtime for Yara Analyser Job (#1550)

* Add yara dynamic update rules
* Add tests for yara dynamic update rules
* Add linux/amd64 spec to docker build fules.

docker/api_server/Dockerfile

@@ -7,7 +7,7 @@ COPY web/. .
 RUN npm run build
 
 # Build Turbinia API Server, copy from build, and setup rest of requirements
-FROM ubuntu:22.04 as build-stage2
+FROM --platform=linux/amd64 ubuntu:22.04 as build-stage2
 
 ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 

docker/server/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 
 ENV DEBIAN_FRONTEND=noninteractive 
 ENV PIP_NO_CACHE_DIR=1

docker/tests/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1 
 ARG PPA_TRACK=stable
@@ -57,9 +57,10 @@ RUN echo "Defaults secure_path=\"/home/turbinia/.venv/bin:/usr/local/sbin:/usr/l
 
 # Install yara rules and fraken binary.
 RUN cd /opt \
-    && git clone https://github.com/Neo23x0/signature-base.git \
+    && git clone https://github.com/Neo23x0/signature-base.git --depth=1\
     && sudo chown -R turbinia:turbinia /opt/signature-base \
-    && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
+    && sudo git config --global --add safe.directory /opt/signature-base \
+    && find /opt/signature-base -type f -not -path '*.git/*' -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
 COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
 RUN mkdir -p /opt/fraken && chown -R turbinia:turbinia /opt/fraken
 COPY --from=us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest --chown=turbinia:turbinia /bin/fraken /opt/fraken/fraken

docker/worker/Dockerfile

@@ -1,5 +1,5 @@
 # Build 1 - Turbinia Worker
-FROM ubuntu:22.04
+FROM --platform=linux/amd64 ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive \
     PIP_NO_CACHE_DIR=1
 ARG PPA_TRACK=stable
@@ -52,9 +52,10 @@ RUN echo "Defaults secure_path=\"/home/turbinia/.venv/bin:/opt/fraken:/usr/local
 
 # Install yara rules and fraken binary.
 RUN cd /opt \
-    && git clone https://github.com/Neo23x0/signature-base.git \
+    && git clone https://github.com/Neo23x0/signature-base.git --depth=1\
     && sudo chown -R turbinia:turbinia /opt/signature-base \
-    && find /opt/signature-base -type f -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
+    && sudo git config --global --add safe.directory /opt/signature-base \
+    && find /opt/signature-base -type f -not -path '*.git/*' -not -iname '*.yar' -not -iname '*.yara' -not -iname 'file-type-signatures.txt' -delete
 COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
 RUN mkdir -p /opt/fraken && chown -R turbinia:turbinia /opt/fraken
 COPY --from=us-docker.pkg.dev/osdfir-registry/turbinia/release/fraken:latest --chown=turbinia:turbinia /bin/fraken /opt/fraken/fraken

pyproject.toml

@@ -41,6 +41,7 @@ urllib3 = [
 ]
 debugpy = "^1.8.0"
 jurigged = "^0.5.7"
+gitpython = "^3.1.43"
 
 [tool.poetry.group.test]
 optional = true

turbinia/workers/analysis/yara.py

@@ -14,7 +14,9 @@
 # limitations under the License.
 """Task for running Yara on drives & directories."""
 
+import git
 import json
+import logging
 import os
 import re
 
@@ -28,6 +30,8 @@
 from turbinia.workers import Priority
 from turbinia.workers import TurbiniaTask
 
+log = logging.getLogger(__name__)
+
 
 class YaraAnalysisTask(TurbiniaTask):
   """Task to use Yara to analyse files."""
@@ -43,6 +47,39 @@ class YaraAnalysisTask(TurbiniaTask):
       'minscore': None
   }
 
+  RULES = {
+      'https://github.com/Neo23x0/signature-base.git': '/opt/signature-base'
+  }
+
+  def update_rules(self, rules):
+    """Update the Yara rules.
+
+    Args: 
+        rules (dict): dict with repo url -> path mapping
+        
+    Returns:  
+        bool: True if success, False if error
+    """
+    if rules is None:
+      rules = self.RULES
+
+    log.debug('Updating Yara rules')
+    for repo, path in rules.items():
+      try:
+        repository = git.Repo(path)
+        origin = repository.remotes.origin
+        origin.pull(depth=1)
+        log.debug('Successfully updated rules from %s in %s', repo, path)
+      except git.exc.InvalidGitRepositoryError as e:
+        log.debug(
+            'InvalidGitRepositoryError updating rules in %s: %s', path, str(e))
+        return False
+      except Exception as e:
+        log.debug('Unknown error updating rules in %s: %s', path, str(e))
+        return False
+
+    return True
+
   def run(self, evidence, result):
     """Run the Yara worker.
 
@@ -52,6 +89,9 @@ def run(self, evidence, result):
     Returns:
         TurbiniaTaskResult object.
     """
+    # Let's update the Yara rules
+    self.update_rules(rules=self.RULES)
+
     # Where to store the resulting output file.
     output_file_name = 'yara_analysis.txt'
     output_file_path = os.path.join(self.output_dir, output_file_name)

turbinia/workers/analysis/yara_test.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 """Tests for the Yara analysis task."""
 
+import git
 import logging
 import os
 import mock
@@ -74,6 +75,16 @@ def test_yara_no_stderr(self):
         TurbiniaException, '.*Unknown \(no stderr\).*', self.task.runFraken,
         self.result, self.evidence)
 
+  def test_update_rules(self):
+    """Tests the update_rules method"""
+    ret = self.task.update_rules(None)
+    self.assertEqual(ret, True)
+
+    error_rules = {'http://dummy': '/'}
+    ret = self.task.update_rules(error_rules)
+    self.assertRaises(git.exc.InvalidGitRepositoryError)
+    self.assertEqual(ret, False)
+
 
 if __name__ == '__main__':
   unittest.main()