Demo 4 PR into Main

.github/workflows/frontendTests.yml

@@ -35,3 +35,12 @@ jobs:
     - run: |
         cd gnd-app
         npm test -- --watch=false --browsers=ChromeHeadless
+
+    # - name: Generate frontend coverage report
+    #   run: |
+    #     cd gnd-app
+    #     npm run test -- --watch=false --browsers=ChromeHeadless --code-coverage
+
+    # - name: Move frontend coverage to backend
+    #   run: |
+    #     mv gnd-app/coverage/lcov.info backend/.coverage.frontend

.github/workflows/lint-tests.yaml

@@ -48,6 +48,13 @@
             VALIDATE_JAVASCRIPT_STANDARD: false
             VALIDATE_TYPESCRIPT_STANDARD: false
             VALIDATE_CHECKOV: false
+            VALIDATE_GITLEAKS: false
+            VALIDATE_BASH: false
+            VALIDATE_BASH_EXEC: false
+            VALIDATE_DOCKERFILE_HADOLINT: false
+            VALIDATE_GROOVY: false
+            VALIDATE_SHELL_SHFMT: false
+            VALIDATE_ENV: false
 
     # doing this to use the config
     run-flake8-lint:

.github/workflows/python-app-test.yml

@@ -60,13 +60,51 @@ jobs:
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
       - name: Test with pytest
+        env:
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+        run: |
+          ls -a
+          cd backend/backend_tests
+          pytest --cov=. --cov-report=xml --cov-append
+        continue-on-error: true
+
+      - name: test gnd email monitor 
+        env:
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
+        run: |
+          ls -a
+          cd backend/GND_Email_Monitor
+          pytest --cov=. --cov-report=xml --cov-append
+        continue-on-error: true
+
+      - name: test document parser 
+        env:
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
         run: |
           ls -a
           cd backend/Document_parser
-          # cd backend
-          pytest --cov=. --cov-report=xml
+          pytest --cov=. --cov-report=xml --cov-append
+        continue-on-error: true
+
+      - name: Move Coverage Files to Root
+        run: |
+          mv backend/backend_tests/.coverage backend/.coverage.backend_tests
+          mv backend/GND_Email_Monitor/.coverage backend/.coverage.GND_Email_Monitor_tests
+          mv backend/Document_parser/.coverage backend/.coverage.Document_parser_tests
+
+      - name: Combine Coverage Reports
+        run: |
+          cd backend
+          coverage combine .coverage.*
+          coverage xml -o coverage.xml
 
       - name: Integration test
+        env:
+          SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
+          SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
         run: |
           ls -a
           cd backend
@@ -75,4 +113,5 @@ jobs:
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v4.0.1
         with:
+          files: ./backend/coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -5,8 +5,13 @@ obj/
 **/__pycache__/
 venv/
 myenv/
+env
+.env
 **/results/
 __pycache__/
 .DS_Store
 **/.DS_Store
 env
+.env
+.vs/
+.pyc

README.md

@@ -1,4 +1,4 @@
-<img src="https://github.com/COS301-SE-2024/GDPR-data-noncompliance-detector/blob/develop/documentation/headshots/GND.png" alt="GND Logo">
+![image](https://github.com/user-attachments/assets/98c03af9-5f20-4d6e-8140-b4a69f7ff72f)<img src="https://github.com/COS301-SE-2024/GDPR-data-noncompliance-detector/blob/develop/documentation/headshots/GND.png" alt="GND Logo">
 
 [![codecov](https://codecov.io/github/COS301-SE-2024/GDPR-data-noncompliance-detector/graph/badge.svg?token=nEPpXWGssM)](https://codecov.io/github/COS301-SE-2024/GDPR-data-noncompliance-detector)
 ![GitHub issues](https://img.shields.io/github/issues/COS301-SE-2024/GDPR-data-noncompliance-detector)
@@ -36,6 +36,17 @@ The GDPR Data Noncompliance Detector is a software tool designed to identify ins
 <br></br>
 
 ### Demo 4
+[Demo 4 SRS Documentation](https://github.com/COS301-SE-2024/GDPR-data-noncompliance-detector/blob/develop/documentation/Demo_3___SRS.pdf)
+<br></br>
+[Demo 4 Testing Specification](https://github.com/COS301-SE-2024/GDPR-data-noncompliance-detector/blob/develop/documentation/Demo_4___Testing_Specification.pdf)
+<br></br>
+# Download
+
+[EXE Download](https://drive.google.com/drive/folders/1LANx27MNDwmfvDtj09NFvJ05s91Ya-7n?usp=sharing)
+
+#Demo 4 Input
+
+[Submit Personal Data](https://docs.google.com/spreadsheets/d/1_c1JbehTnfNju1iyqR2vt_2np4OUfFGkX9SmGOiEvco/edit?usp=sharing)
 
 # Documentation
 
@@ -54,7 +65,7 @@ The GDPR Data Noncompliance Detector is a software tool designed to identify ins
 [Coding Standards](https://github.com/COS301-SE-2024/GDPR-data-noncompliance-detector/blob/develop/documentation/Coding_Standards.pdf)
 
 ## User Manual
-[GND User Manual](https://github.com/COS301-SE-2024/GDPR-data-noncompliance-detector/blob/develop/documentation/GND%20Manual.pdf)
+[GND User Manual](https://github.com/COS301-SE-2024/GDPR-data-noncompliance-detector/blob/develop/documentation/GND%20Manual_V2.pdf)
 
 ## Project Management Tools
 GitHub Issues and GitHub Boards
@@ -189,7 +200,7 @@ Angular with Electron
 
 ### Backend
 Python
-FastAPI
+FlaskAPI
 
 ### Testing
 PyTest

backend/Detection_Engine/CA_model_access.py

@@ -25,6 +25,7 @@ def predict(self, input_text):
         truncated_text = self.classifier.tokenizer.decode(tokens, skip_special_tokens=True)
 
         result = self.classifier(truncated_text)
+        # print(result)
         return result[0]['label']
 
 if __name__ == '__main__':

backend/Detection_Engine/EM_model_access.py

@@ -1,6 +1,15 @@
-import os
+import os, sys
 from transformers import BertTokenizer, BertForSequenceClassification, pipeline
 
+def resource_path(relative_path):
+
+    try:
+        base_path = sys._MEIPASS
+
+    except Exception:
+        base_path = os.path.abspath(".")
+
+    return os.path.join(base_path, relative_path)
 class EM:
 
     def __init__(self):
@@ -13,19 +22,30 @@ def __init__(self):
     def run_EM(self, text):
         return self.predict(text)
 
+    def sliding_window(self, text, window_size, overlap):
+        tokens = self.classifier.tokenizer.encode(text, truncation=False)
+        chunks = []
+        for i in range(0, len(tokens), window_size - overlap):
+            chunk = tokens[i:i + window_size]
+            if len(chunk) < window_size:
+                break
+            chunks.append(chunk)
+        return chunks
+
     def predict(self, input_text):
-        count = 0
-        tokens = self.classifier.tokenizer.encode(input_text, truncation=True, max_length=self.max_length_)
-        truncated_text = self.classifier.tokenizer.decode(tokens, skip_special_tokens=True)
-
-        result = self.classifier(truncated_text)
-        label = result[0]['label']
-        if label != 'LABEL_2':
-            count += 1
-        else:
-            count += 0
-
-        return count
+        window_size = 128  
+        overlap = 64       
+
+        chunks = self.sliding_window(input_text, window_size, overlap)
+        labels = []
+
+        for chunk in chunks:
+            truncated_text = self.classifier.tokenizer.decode(chunk, skip_special_tokens=True)
+            result = self.classifier(truncated_text)
+            label = result[0]['label']
+            labels.append(label)
+
+        return labels
 
 if __name__ == '__main__':
     em = EM()

backend/Detection_Engine/GDPR_model_access.py

@@ -1,6 +1,15 @@
-import os
+import os, sys
 from transformers import DebertaV2Tokenizer, DebertaV2ForSequenceClassification, pipeline
 
+def resource_path(relative_path):
+
+    try:
+        base_path = sys._MEIPASS
+
+    except Exception:
+        base_path = os.path.abspath(".")
+
+    return os.path.join(base_path, relative_path)
 
 class GDPR:
 
@@ -12,22 +21,31 @@ def __init__(self):
     def run_GDPR(self, text):
         return self.predict(text)
 
+
+    def sliding_window(self, text, window_size, overlap):
+        tokens = self.classifier.tokenizer.encode(text, truncation=False)
+        chunks = []
+        for i in range(0, len(tokens), window_size - overlap):
+            chunk = tokens[i:i + window_size]
+            if len(chunk) < window_size:
+                break
+            chunks.append(chunk)
+        return chunks
+
     def predict(self, input_text):
-        count  = 0
-        tokens = self.classifier.tokenizer.encode(input_text, truncation=True, max_length=self.max_length_)
-        truncated_text = self.classifier.tokenizer.decode(tokens, skip_special_tokens=True)
-
-        result = self.classifier(truncated_text)
-        # return result[0]['label']
+        window_size = 128
+        overlap = 64
 
-        label = result[0]['label']
-        if label != 'LABEL_0':
-            count += 1
-        else:
-            count += 0
+        chunks = self.sliding_window(input_text, window_size, overlap)
+        labels = []
 
-        return count
+        for chunk in chunks:
+            truncated_text = self.classifier.tokenizer.decode(chunk, skip_special_tokens=True)
+            result = self.classifier(truncated_text)
+            label = result[0]['label']
+            labels.append(label)
 
+        return labels
 
 if __name__ == '__main__':
     gdpr = GDPR()

backend/Detection_Engine/GF_model_access.py

@@ -0,0 +1,72 @@
+import os
+# from transformers import BertTokenizer, BertForSequenceClassification, pipeline
+import re
+
+# base_dir = os.path.dirname(__file__)
+# model_path = os.path.join(base_dir, 'trained_model')
+# tokenizer_path = os.path.join(base_dir,'ca_tokenizer')
+
+# tokenizer_ = AutoTokenizer.from_pretrained("rdhinaz/gdpr_consent_agreement")
+# mode+l = AutoModelForSequenceClassification.from_pretrained("rdhinaz/gdpr_consent_agreement")
+
+class GF:
+
+    def __init__(self):
+        self.patterns = [
+            re.compile("CGA"),
+            re.compile("TCAGGTCA"),
+            re.compile(r"ATG([ATGC]{3})*?(TAA|TAG|TGA)"),
+            re.compile(r"GATATC"),
+            re.compile(r"VI\w{2}L\w{2}"),
+            re.compile(r"SR[AG]"),
+            re.compile(r"F[LIV]{2}[KR]{1}\w{2}D"),
+            re.compile(r"TG\w{6}CA"),
+            re.compile(r"[WG]{1}\w{1,2}[ED]{1}\w{1,2}[WG]{1}"),
+            re.compile(r"[ACGT]{6}G[AG]N[AG]T[AC][AG][ACGT]{3}G[ACGT]{3}[ACGT]{2}"),
+            re.compile(r"(?:G|C){3,}|(?:A|U){3,}"),
+            re.compile(r"L.{1,3}L.{1,3}L")
+        ]
+        # self.model_ = model
+        # self.tokenizer_ = tokenizer
+        # self.max_length_ = 512
+        # self.classifier = pipeline("text-classification", model="rdhinaz/genome-finder")
+        # self.max_length = self.max_length_
+
+    def run_GF(self, text):
+        match_count = 0
+        for pattern in self.patterns:
+            matches = pattern.findall(text)
+            match_count += len(matches)  # Count the number of matches
+        return match_count
+        # return self.predict(text)
+
+    def sliding_window(self, text, window_size, overlap):
+        tokens = self.classifier.tokenizer.encode(text, truncation=False)
+        chunks = []
+        for i in range(0, len(tokens), window_size - overlap):
+            chunk = tokens[i:i + window_size]
+            if len(chunk) < window_size:
+                break
+            chunks.append(chunk)
+        return chunks
+
+    # def predict(self, input_text):
+        # window_size = 128  
+        # overlap = 64       
+
+        # chunks = self.sliding_window(input_text, window_size, overlap)
+        # labels = []
+
+        # for chunk in chunks:
+        # tokenized_text = self.classifier.tokenizer.encode(input_text, truncation=True, padding=True, return_tensors='pt')
+
+        # Pass the tokenized text to the classifier
+        # result = self.classifier(tokenized_text)
+            # label = result[0]['label']
+            # labels.append(label)
+
+        # return result
+
+if __name__ == '__main__':
+    GF = GF()
+    print(GF.run_CA("Sure, you can use my data"))

backend/Detection_Engine/MD_model_access.py

@@ -1,6 +1,17 @@
+import os, sys
 from transformers import AutoTokenizer, AutoModelForTokenClassification
 from transformers import pipeline
 
+def resource_path(relative_path):
+
+    try:
+        base_path = sys._MEIPASS
+
+    except Exception:
+        base_path = os.path.abspath(".")
+
+    return os.path.join(base_path, relative_path)
+
 class MD:
 
     def __init__(self):