Merge remote-tracking branch 'origin/master' into #2170_CallsSubcateg…

…ories
sepinf-inc · May 29, 2024 · db350d3 · db350d3
2 parents 4ff1b06 + b56fd6e
commit db350d3
Show file tree

Hide file tree

Showing 118 changed files with 3,980 additions and 841 deletions.
diff --git a/iped-app/pom.xml b/iped-app/pom.xml
@@ -125,7 +125,7 @@
                                 <artifactItem>
                                     <groupId>org.python</groupId>
                                     <artifactId>python-jep-dlib</artifactId>
-                                    <version>3.9.12-4.0.3-19.23.1</version>
+                                    <version>3.9.12-4.0.3-19.23.1-2</version>
                                     <type>zip</type>
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${release.dir}</outputDirectory>
@@ -315,7 +315,7 @@
                                 <artifactItem>
                                     <groupId>tesseract</groupId>
                                     <artifactId>tesseract-zip</artifactId>
-                                    <version>5.3.2-24-g3922</version>
+                                    <version>5.3.2-24-g3922_1</version>
                                     <type>zip</type>
                                     <overWrite>false</overWrite>
                                     <outputDirectory>${tools.dir}</outputDirectory>

diff --git a/iped-app/resources/config/IPEDConfig.txt b/iped-app/resources/config/IPEDConfig.txt
@@ -97,10 +97,12 @@ enableMinIO = false
 enableOCR = false
 
 # Enable audio transcription.
-# Default implementation uses VOSK transcription on local CPU (slow and not good accuracy).
-# You can change it to a local Facebook Wav2Vec2 implementation (slower on CPU, faster on GPU and good accuracy)
-# or remote Microsoft Azure or Google Cloud services (faster and good accuracy).
-# Configure it in conf/AudioTranscriptConfig.txt
+# Default implementation uses VOSK transcription on local CPU (faster but bad accuracy).
+# You can change the algorithm into conf/AudioTranscriptConfig.txt:
+# - Wav2Vec2 algorithm (slower and good accuracy)
+# - Whisper algorithm (much slower but better accuracy)
+# - Google Cloud (about $1.00 per hour cost)
+# - Microsoft Azure (about $1.00 per hour cost)
 enableAudioTranscription = false
 
 # Enables carving. "addUnallocated" must be enabled to scan unallocated space.

diff --git a/iped-app/resources/config/LocalConfig.txt b/iped-app/resources/config/LocalConfig.txt
@@ -2,7 +2,7 @@
 # Local environment configuration
 ########################################################################
 
-# Defines program localization/language. Currently there are localizations for 'en', 'pt-BR', 'it-IT', 'de-DE' & 'es-AR'
+# Defines program localization/language. Currently there are localizations for 'en', 'pt-BR', 'it-IT', 'de-DE', 'es-AR' and 'fr-FR'.
 locale = en
 
 # Temporary directory for processing: "default" uses the system temporary folder.

diff --git a/iped-app/resources/config/conf/AudioTranscriptConfig.txt b/iped-app/resources/config/conf/AudioTranscriptConfig.txt
@@ -9,14 +9,23 @@
 # you should download it from https://alphacephei.com/vosk/models and put in 'models/vosk/[lang]' folder.
 implementationClass = iped.engine.task.transcript.VoskTranscriptTask
 
-# Uses a local/remote wav2vec2 implementation for transcription. Accuracy is much better than most Vosk models.
-# The local impl is AT LEAST 1 order of magnitude slower than Vosk on high end CPUs. Using a good GPU is highly recommended!
-# The remote impl is useful if you have a central server/cluster with many GPUs to be shared among processing nodes.
-# For both the local or remote options, please check the installation steps: https://github.com/sepinf-inc/IPED/wiki/User-Manual#wav2vec2
-# If you use the local implementation, you must set 'huggingFaceModel' param below.
-# If you use the remote implementation, you must set 'wav2vec2Service' param below.
+# Uses a local wav2vec2 implementation for transcription. Accuracy is much better than most Vosk models.
+# This is up to 10x slower than Vosk on high end CPUs. Using a good GPU is highly recommended!
+# Please check the installation steps: https://github.com/sepinf-inc/IPED/wiki/User-Manual#wav2vec2
+# If you enable this, you must set 'huggingFaceModel' param below.
 #implementationClass = iped.engine.task.transcript.Wav2Vec2TranscriptTask
-#implementationClass = iped.engine.task.transcript.RemoteWav2Vec2TranscriptTask
+
+# Uses a local Whisper implementation for transcription. Accuracy is better than wav2vec2 depending on the model.
+# This is up to 4x slower than wav2vec2 depending on compared models. Using a high end GPU is strongly recommended!
+# Please check the installation steps: https://github.com/sepinf-inc/IPED/wiki/User-Manual#whisper
+# If you enable this, you must set 'whisperModel' param below.
+#implementationClass = iped.engine.task.transcript.WhisperTranscriptTask
+
+# Uses a remote service for transcription.
+# The remote service is useful if you have a central server/cluster with many GPUs to be shared among processing nodes.
+# Please check steps on https://github.com/sepinf-inc/IPED/wiki/User-Manual#remote-transcription
+# If you enable this, you must set 'remoteServiceAddress' param below.
+#implementationClass = iped.engine.task.transcript.RemoteTranscriptionTask
 
 # If you want to use the Microsoft Azure service implementation, comment above and uncomment below.
 # You MUST include Microsoft client-sdk.jar into plugins folder.
@@ -91,11 +100,32 @@ minWordScore = 0.5
 # huggingFaceModel = jonatasgrosman/wav2vec2-xls-r-1b-french
 
 #########################################
-# RemoteWav2Vec2TranscriptTask options
+# Local WhisperTranscriptTask options
+#########################################
+
+# Possible values: tiny, base, small, medium, large-v3, dwhoelz/whisper-large-pt-cv11-ct2
+# large-v3 is much better than medium, but 2x slower and uses 2x more memory.
+# If you know the language you want to transcribe, please set the 'language' option above.
+# 'language = auto' uses the 'locale' set on LocalConfig.txt
+# 'language = detect' uses auto detection, but it can cause mistakes
+whisperModel = medium
+
+# Compute type precision. This affects accuracy, speed and memory usage.
+# Possible values: float32 (better), float16 (recommended for GPU), int8 (faster)
+precision = int8
+
+# Batch size (number of parallel transcriptions). If you have a GPU with enough memory,
+# increasing this value to e.g. 16 can speed up transcribing long audios up to 10x.
+# Test what is the better value for your GPU before hitting OOM.
+# This works just if you are using whisperx library instead of faster_whisper
+batchSize = 1
+
+#########################################
+# RemoteAudioTranscriptTask options
 #########################################
 
 # IP:PORT of the service/central node used by the RemoteWav2Vec2TranscriptTask implementation.
-# wav2vec2Service = 127.0.0.1:11111
+# remoteServiceAddress = 127.0.0.1:11111
 
 #########################################
 # MicrosoftTranscriptTask options

diff --git a/iped-app/resources/config/conf/CategoriesConfig.json b/iped-app/resources/config/conf/CategoriesConfig.json
@@ -108,7 +108,7 @@
       {"name": "Ares Galaxy", "mimes": ["application/x-ares-galaxy","application/x-ares-galaxy-entry"]},
       {"name": "E-Mule", "mimes": ["application/x-emule", "application/x-emule-part-met", "application/x-emule-searches", "application/x-emule-preferences-ini", "application/x-emule-preferences-dat", "application/x-emule-known-met-entry", "application/x-emule-part-met-entry"]},
       {"name": "Shareaza", "mimes": ["application/x-shareaza-searches-dat", "application/x-shareaza-library-dat", "application/x-shareaza-library-dat-entry", "application/x-shareaza-download"]},
-      {"name": "Torrent", "mimes": ["application/x-bittorrent-resume-dat", "application/x-bittorrent"]},
+      {"name": "Torrent", "mimes": ["application/x-bittorrent-resume-dat","application/x-bittorrent-settings-dat", "application/x-bittorrent"]},
       {"name": "Other Peer-to-peer", "mimes": ["application/x-p2p"]}
     ]},
     {"name": "Browser Artifacts", "categories":[

diff --git a/iped-app/resources/config/conf/CustomSignatures.xml b/iped-app/resources/config/conf/CustomSignatures.xml
@@ -441,13 +441,25 @@
         <_comment>BitTorrent Client Resume.dat file</_comment>
         <sub-class-of type="application/x-p2p"/>
         <magic priority="50">
-    		<match value="d10:.fileguard" type="string" offset="0"/>
+    		<match value="d10:.fileguard" type="string" offset="0">
+	    		<match value="e4:info" type="string" offset="16:1024"/>
+    		</match>
 		</magic>
         <glob pattern="resume.dat" />
         <glob pattern="resume.dat.old" />
         <glob pattern="resume.dat.new" />
     </mime-type>
 
+    <mime-type type="application/x-bittorrent-settings-dat">
+        <_comment>BitTorrent Client Settings.dat file</_comment>
+        <sub-class-of type="application/x-p2p"/>
+        <magic priority="50">
+    		<match value="d10:.fileguard" type="string" offset="0">
+	    		<match value="e9:autostart" type="string" offset="16:1024"/>
+    		</match>
+		</magic>
+    </mime-type>
+
 	<mime-type type="application/x-bittorrent">
 		<sub-class-of type="application/x-p2p"/>
 		<magic priority="50">

diff --git a/iped-app/resources/config/conf/MakePreviewConfig.txt b/iped-app/resources/config/conf/MakePreviewConfig.txt
@@ -7,7 +7,7 @@
 supportedMimes = application/x-msaccess; application/x-lnk; application/x-firefox-savedsession
 supportedMimes = application/x-sqlite3; application/sqlite-skype; application/x-win10-timeline; application/x-gdrive-cloud-graph; application/x-gdrive-snapshot
 supportedMimes = application/x-whatsapp-db; application/x-whatsapp-db-f; application/x-whatsapp-chatstorage; application/x-whatsapp-chatstorage-f; application/x-threema-chatstorage; application/x-shareaza-searches-dat; application/x-msie-cache
-supportedMimes = application/x-prefetch; text/x-vcard; application/x-bittorrent-resume-dat; application/x-bittorrent; application/x-emule-preferences-dat; application/vnd.android.package-archive
+supportedMimes = application/x-prefetch; text/x-vcard; application/x-emule-preferences-dat; application/vnd.android.package-archive; application/x-bittorrent-settings-dat
 
 # List of mimetypes which parsers insert links to other case items into preview
-supportedMimesWithLinks = application/x-emule; application/x-emule-part-met; application/x-ares-galaxy; application/x-shareaza-library-dat; application/x-shareaza-download
+supportedMimesWithLinks = application/x-emule; application/x-emule-part-met; application/x-ares-galaxy; application/x-shareaza-library-dat; application/x-shareaza-download; application/x-bittorrent-resume-dat; application/x-bittorrent
diff --git a/iped-app/resources/config/conf/OCRConfig.txt b/iped-app/resources/config/conf/OCRConfig.txt
@@ -5,7 +5,7 @@
 # You can use the parameter -ocr "bookmark_name" to restrict the OCR to a specific bookmark (just works when creating reports from cmd line).
 
 # Dictionary language to be used for OCR. You can specify multiple languages e.g. por+eng but you will pay an important processing cost.
-# Languages supported: eng, por, ita, deu, spa
+# Languages supported: eng, por, ita, deu, spa, fra
 OCRLanguage = por
 
 # Skip known files found in the hash lookup database.

diff --git a/iped-app/resources/config/conf/ParserConfig.xml b/iped-app/resources/config/conf/ParserConfig.xml
@@ -351,6 +351,7 @@
             </params>
         </parser>
         <parser class="iped.parsers.bittorrent.BitTorrentResumeDatParser"></parser>
+        <parser class="iped.parsers.bittorrent.BitTorrentGenericDatParser"></parser>
         <parser class="iped.parsers.bittorrent.TorrentFileParser"></parser>
         <parser class="iped.parsers.lnk.LNKShortcutParser"></parser>
         <parser class="iped.parsers.misc.GenericOLEParser"></parser>

diff --git a/iped-app/resources/config/conf/ParsingTaskConfig.txt b/iped-app/resources/config/conf/ParsingTaskConfig.txt
@@ -32,6 +32,10 @@ minRawStringSize = 4
 # Possible values: internal, external, all
 phoneParsersToUse = internal
 
+# List of UFED sources names for which IPED has a specific internal parser.
+# This avoids duplicated decoding results when 'phoneParsersToUse = internal'
+sourcesWithInternalParsers = WhatsApp, WhatsApp (Dual App), WhatsApp Business, Telegram
+
 # If true, takes into account the characters' position on the page when reconstructing the extracted text.
 # Needed to properly index rotated PDFs but doubles the processing time of PDFs.
 sortPDFChars = false

diff --git a/iped-app/resources/config/conf/metadataTypes.txt b/iped-app/resources/config/conf/metadataTypes.txt
@@ -3392,6 +3392,8 @@ og:type = java.lang.String
 og:url = java.lang.String
 ontent-Type" = java.lang.String
 p2pHistoryEntries = java.lang.Integer
+childPornHashHits = java.lang.Integer
+p2p:torrentFilesFoundInCase = java.lang.Integer
 containerTrackId = java.lang.String
 parentTrackId = java.lang.String
 trackId = java.lang.String

diff --git a/iped-app/resources/config/profiles/forensic/conf/ParserConfig.xml b/iped-app/resources/config/profiles/forensic/conf/ParserConfig.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- this config is merged with default config XML -->
+<properties>
+
+    <parsers>
+        <parser class="iped.parsers.usnjrnl.UsnJrnlParser">
+            <params>
+                <param name="extractEntries" type="bool">true</param>
+            </params>
+        </parser>
+    </parsers>
+
+</properties>
diff --git a/iped-app/resources/config/profiles/pedo/conf/ParserConfig.xml b/iped-app/resources/config/profiles/pedo/conf/ParserConfig.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- this config is merged with default config XML -->
+<properties>
+
+    <parsers>
+        <parser class="iped.parsers.usnjrnl.UsnJrnlParser">
+            <params>
+                <param name="extractEntries" type="bool">true</param>
+            </params>
+        </parser>
+    </parsers>
+
+</properties>
diff --git a/iped-app/resources/localization/iped-categories.properties b/iped-app/resources/localization/iped-categories.properties
@@ -28,8 +28,8 @@ Windows\ Artifacts=Windows\ Artifacts
 Event\ Files=Event\ Files
 Event\ Records=Event\ Records
 Prefetch\ and\ Superfetch=Prefetch\ and\ Superfetch
-User\ Activities=Windows\ 10\ User\ Timeline
-User\ Activities\ Entries=Windows\ 10\ User\ Timeline\ Records
+User\ Activities=User\ Timeline
+User\ Activities\ Entries=User\ Timeline\ Records
 Windows\ Registry=Windows\ Registry
 Main\ Registry\ Files=Main\ Registry\ Files
 Other\ Registry\ Files=Other\ Registry\ Files

diff --git a/iped-app/resources/localization/iped-categories_de_DE.properties b/iped-app/resources/localization/iped-categories_de_DE.properties
@@ -16,7 +16,7 @@ Other\ Documents=sonstige\ Dokumente
 Emails\ and\ Mailboxes=E-Mails\ und\ Postfächer
 Emails=E-Mails
 Mailboxes=Postfächer
-Appointments=Appointments[TBT]
+Appointments=Termine
 GDrive\ Synced\ Files=GDrive\ synchronisierte\ Dateien
 GDrive\ File\ Entries=GDrive\ Datei\ Einträge
 Databases=Datenbanken
@@ -28,8 +28,8 @@ Windows\ Artifacts=Windows\ Artefakte
 Event\ Files=Ereignis\ Dateien
 Event\ Records=Ereignis\ Aufzeichnungen
 Prefetch\ and\ Superfetch=Prefetch\ und\ Superfetch
-User\ Activities=Windows\ 10\ Benutzer\ Zeitlinie
-User\ Activities\ Entries=Windows\ 10\ Benutzer\ Zeitlinienaufzeichnungen
+User\ Activities=Benutzer\ Zeitlinie
+User\ Activities\ Entries=Benutzer\ Zeitlinienaufzeichnungen
 Windows\ Registry=Windows\ Registrierung
 Main\ Registry\ Files=wesentliche\ Registry-Dateien
 Other\ Registry\ Files=andere\ Registry-Dateien

diff --git a/iped-app/resources/localization/iped-categories_es_AR.properties b/iped-app/resources/localization/iped-categories_es_AR.properties
@@ -28,8 +28,8 @@ Windows\ Artifacts=Windows\ Artefactos
 Event\ Files=Eventos\ Archivos
 Event\ Records=Eventos\ Grabados
 Prefetch\ and\ Superfetch=Prefetch\ y\ Superfetch
-User\ Activities=Windows\ 10\ Usuario\ Línea de tiempo
-User\ Activities\ Entries=Windows\ 10\ Usuario\ Línea de tiempo\ Registros
+User\ Activities=Usuario\ Línea de tiempo
+User\ Activities\ Entries=Usuario\ Línea de tiempo\ Registros
 Windows\ Registry=Registros\ Windows
 Main\ Registry\ Files=Registro\ Principal\ Archivos
 Other\ Registry\ Files=Otros\ Registros\ de\ Archivos