Hikvision, WFS and DHFS File System parser - DVR videos

iped-api/src/main/java/iped/properties/MediaTypes.java

@@ -34,6 +34,9 @@ public class MediaTypes {
     public static final MediaType VHDX = MediaType.application("x-vhdx"); //$NON-NLS-1$
     public static final MediaType VDI = MediaType.application("x-vdi"); //$NON-NLS-1$
     public static final MediaType MS_PUBLISHER = MediaType.application("x-mspublisher"); //$NON-NLS-1$
+    public static final MediaType HIKVISIONFS = MediaType.application("hikvisionfs"); //$NON-NLS-1$    
+    public static final MediaType WFS = MediaType.application("wfs"); //$NON-NLS-1$    
+    public static final MediaType DHFS = MediaType.application("dhfs"); //$NON-NLS-1$    
 
     public static final String UFED_MIME_PREFIX = "x-ufed-"; //$NON-NLS-1$
 

iped-app/resources/config/IPEDConfig.txt

@@ -144,4 +144,7 @@ enableDocThumbs = false
 
 # Enables HTML report generation on automatic extractions or from selected items.
 # Generation settings can be modified in file "conf/HTMLReportConfig.txt"
-enableHTMLReport = true
+enableHTMLReport = true
+
+# Enables extraction of video files from DVR (Digital Video Recorder) File Systems
+enableDVR = true

iped-app/resources/config/conf/CustomSignatures.xml

@@ -1643,4 +1643,33 @@
         <glob pattern="vlc-qt-interface.ini"/>
     </mime-type>
 
+	<mime-type type="application/hikvisionfs">
+    	<magic priority="60">
+      		<match value="HIKVISION@HANGZHOU" type="string" offset="0:1024"/>
+    	</magic>
+    	<sub-class-of type="application/octet-stream"/>
+  	</mime-type>
+
+	<mime-type type="application/wfs">
+    	<magic priority="50">
+      		<match value="WFS0.4" type="string" offset="0:1024"/>
+    	</magic>
+    	<sub-class-of type="application/octet-stream"/>
+  	</mime-type>
+
+	<mime-type type="application/wfs">
+        <sub-class-of type="application/wfs"/>
+    	<magic priority="50">
+      		<match value="WFS0.5" type="string" offset="0:1024"/>
+    	</magic>
+    	<sub-class-of type="application/octet-stream"/>
+  	</mime-type>
+
+	<mime-type type="application/dhfs">
+    	<magic priority="50">
+      		<match value="DHFS4.1" type="string" offset="0:1024"/>
+    	</magic>
+    	<sub-class-of type="application/octet-stream"/>
+  	</mime-type>
+
 </mime-info>

iped-app/resources/config/conf/DVRConfig.txt

@@ -0,0 +1,24 @@
+###############################################
+# DVR Task Configuration   #
+###############################################
+
+#It is recommended to disable 'parseUnknownFiles' to speed up the processing time of this task
+
+###############################################
+# Hikvision FS Configuration   #
+###############################################
+
+#Video files are usually saved in 1GB blocks. The task will separate these videos based on the metadata data
+#To also index the entire video block, this option must be activated
+extractDataBlockHVFS = false
+
+# Parse and extract system logs as log files in text mode
+extractSystemLogsHVFS = true
+
+###############################################
+# WFS Configuration   #
+###############################################
+
+###############################################
+# DHFS Configuration   #
+###############################################

iped-app/resources/config/conf/TaskInstaller.xml

@@ -38,6 +38,7 @@
     <task class="iped.engine.task.similarity.ImageSimilarityTask"></task>
     <task class="iped.engine.task.PhotoDNATask"></task>
     <task class="iped.engine.task.PhotoDNALookup"></task>
+    <task class="iped.engine.task.dvr.DVRTask"></task>
     <task script="NSFWNudityDetectTask.py"></task>
     <task script="FaceRecognitionTask.py"></task>
     <task script="SearchHardwareWallets.py"></task>

iped-engine/src/main/java/iped/engine/config/DVRTaskConfig.java

@@ -0,0 +1,67 @@
+package iped.engine.config;
+
+import iped.utils.UTF8Properties;
+
+/*
+
+ * WFSExtractor.
+ *
+ * @author guilherme.dutra
+
+*/
+
+public class DVRTaskConfig extends AbstractTaskPropertiesConfig {
+
+    private static final long serialVersionUID = 1L;
+
+
+    private static final String CONFIG_FILE = "DVRConfig.txt";
+
+    private static final String ENABLED_PROP = "enableDVR";
+
+    private static final String EXTRACT_DATA_BLOCK_HVFS = "extractDataBlockHVFS";
+
+    private static final String EXTRACT_SYSTEM_LOG_HVFS = "extractSystemLogsHVFS";
+
+
+    private boolean extractDataBlockHVFS = false;
+
+    private boolean extractSystemLogsHVFS = false;
+
+
+    public Boolean getExtractDataBlockHVFS() {
+        return extractDataBlockHVFS;
+    }
+
+    public Boolean getExtractSystemLogsHVFS() {
+        return extractSystemLogsHVFS;
+    }
+
+
+    @Override
+    public String getTaskEnableProperty() {
+        return ENABLED_PROP;
+    }
+
+    @Override
+    public String getTaskConfigFileName() {
+        return CONFIG_FILE;
+    }
+
+    @Override
+    public void processProperties(UTF8Properties properties) {
+
+        String value = properties.getProperty(EXTRACT_DATA_BLOCK_HVFS);
+        if (value != null && value.trim().equalsIgnoreCase("true")) {
+            this.extractDataBlockHVFS = true;
+        }
+
+        value = properties.getProperty(EXTRACT_SYSTEM_LOG_HVFS);
+        if (value != null && value.trim().equalsIgnoreCase("true")) {
+            this.extractSystemLogsHVFS = true;
+        }
+
+
+    }
+
+}

iped-engine/src/main/java/iped/engine/datasource/SleuthkitReader.java

@@ -83,6 +83,7 @@
 import iped.engine.sleuthkit.SleuthkitClient;
 import iped.engine.sleuthkit.SleuthkitInputStreamFactory;
 import iped.engine.task.carver.BaseCarveTask;
+import iped.engine.task.ThumbTask;
 import iped.engine.task.index.IndexItem;
 import iped.engine.util.UIPropertyListenerProvider;
 import iped.engine.util.Util;
@@ -92,6 +93,10 @@
 import iped.utils.IOUtil;
 import iped.utils.UTF8Properties;
 
+import iped.io.SeekableInputStream;
+import java.util.Arrays;
+import iped.engine.sleuthkit.SpecialFileSystem;
+
 public class SleuthkitReader extends DataSourceReader {
 
     private static Logger LOGGER = LoggerFactory.getLogger(SleuthkitReader.class);
@@ -1097,9 +1102,18 @@ private IItem addEvidenceFile(Content content) throws Exception {
             evidence.setPath(inheritedPath + path);
         }
 
+        evidence.setIdInDataSource(Long.toString(content.getId()));
+        SleuthkitInputStreamFactory objSleuthkitInputStreamFactory = new SleuthkitInputStreamFactory(sleuthCase, null,false);
+        evidence.setInputStreamFactory(objSleuthkitInputStreamFactory);
+        boolean isSpecialFileSystem = SpecialFileSystem.isSpecialFileSystem(evidence.getSeekableInputStream());
+
+        if (isSpecialFileSystem) //Workaround for not call fragmentLargeBinaryTask task
+            objSleuthkitInputStreamFactory.setPassthroughContent(true);
+
         if (content instanceof Image) {
             evidence.setRoot(true);
-            evidence.setMediaType(getMediaType(evidence.getExt()));
+            if (!isSpecialFileSystem)
+                evidence.setMediaType(getMediaType(evidence.getExt()));
         } else {
             evidence.setIsDir(true);
         }
@@ -1108,9 +1122,12 @@ private IItem addEvidenceFile(Content content) throws Exception {
 
         // evidence.setSleuthFile(content);
         evidence.setHash(""); //$NON-NLS-1$
-        evidence.setIdInDataSource(Long.toString(content.getId()));
-        // below is used to don't process images, partitions or file systems raw data
-        evidence.setInputStreamFactory(new SleuthkitInputStreamFactory(sleuthCase, null));
+        if (!isSpecialFileSystem){
+            evidence.setIdInDataSource(Long.toString(content.getId()));
+            // below is used to don't process images, partitions or file systems raw data
+            evidence.setInputStreamFactory(new SleuthkitInputStreamFactory(sleuthCase, null));
+        }
+
 
         boolean first = true;
         Integer tskId = (int) content.getId();
@@ -1130,6 +1147,9 @@ private IItem addEvidenceFile(Content content) throws Exception {
 
         addToProcessingQueue(caseData, evidence);
 
+        if (isSpecialFileSystem) //Workaround for not call entropy task
+            evidence.setExtraAttribute(ThumbTask.HAS_THUMB, true);
+
         return evidence;
     }
 

iped-engine/src/main/java/iped/engine/io/WFSInputStreamFactory.java

@@ -0,0 +1,42 @@
+package iped.engine.io;
+
+import iped.utils.SeekableInputStreamFactory;
+import iped.engine.task.dvr.wfs.WFSExtractor;
+import iped.utils.SeekableFileInputStream;
+import iped.io.SeekableInputStream;
+import java.io.IOException;
+
+/*
+
+ * WFSExtractor.
+ *
+ * @author guilherme.dutra
+
+*/
+
+public class WFSInputStreamFactory extends SeekableInputStreamFactory {
+
+    WFSExtractor wfs;
+    SeekableInputStream is;
+
+    public WFSInputStreamFactory(SeekableInputStream is) {
+
+        super(null);
+        this.is = is;
+    }
+
+    private synchronized void init() throws IOException {
+        if (wfs == null){
+            wfs = new WFSExtractor();
+            wfs.init(is);
+        }
+    }
+
+    @Override
+    public SeekableInputStream getSeekableInputStream(String identifier) throws IOException {
+        if (wfs == null)
+            init();
+        return wfs.getDescriptorFromIdentifier(identifier);
+    }
+
+}

iped-engine/src/main/java/iped/engine/sleuthkit/SleuthkitInputStreamFactory.java

@@ -29,6 +29,7 @@ public class SleuthkitInputStreamFactory extends SeekableInputStreamFactory {
     private SleuthkitCase sleuthkitCase;
     private Content content;
     private boolean emptyContent = false;
+    private boolean passthroughContent = false;
 
     public SleuthkitInputStreamFactory(Path dataSource) {
         super(dataSource.toUri());
@@ -56,9 +57,26 @@ public SleuthkitInputStreamFactory(SleuthkitCase sleuthkitCase, Content content)
         }
     }
 
+    public SleuthkitInputStreamFactory(SleuthkitCase sleuthkitCase, Content content, boolean emptyContent) {
+        this(sleuthkitCase);
+        if (content != null) {
+            this.content = content;
+        } else {
+            this.emptyContent = emptyContent;
+        }
+    }
+
+    public void setPassthroughContent(boolean value){
+        this.passthroughContent = value;
+    }
+
     @Override
     public boolean returnsEmptyInputStream() {
-        return this.emptyContent;
+
+        if (passthroughContent)
+            return true;
+        else
+            return this.emptyContent;
     }
 
     public SleuthkitCase getSleuthkitCase() {
@@ -116,14 +134,36 @@ public SeekableInputStream getSeekableInputStream(String identifier) throws IOEx
             return new EmptyInputStream();
         }
         FileSystemConfig fsConfig = ConfigurationManager.get().findObject(FileSystemConfig.class);
-        long tskId = Long.valueOf(identifier);
+
+        boolean isSpecialFileSystem = false;
+        long identifier_number = Long.valueOf(identifier);
+        long type = (identifier_number >> 63) & 1L;
+        String address = "0";
+        long tskId = 0;
+
+        if (type == 0){
+            tskId = identifier_number;
+        }else{
+            tskId = (identifier_number & (8191L << 50)) >> 50;
+            address = Long.toString((identifier_number & (1125899906842623L)));
+            isSpecialFileSystem = true;
+        }      
+
         Content tskContent = getContentById(tskId);
         if (SleuthkitReader.sleuthCase == null || !fsConfig.isRobustImageReading()) {
-            return new SleuthkitInputStream(tskContent);
+            if (!isSpecialFileSystem){
+                return new SleuthkitInputStream(tskContent);
+            }else{
+                    return SpecialFileSystem.getSeekableInputStream(new SleuthkitInputStream(tskContent),tskId,address);
+            }
         } else {
             SleuthkitClient sleuthProcess = SleuthkitClient.get();
             try {
-                return sleuthProcess.getInputStream((int) tskId, tskContent.getUniquePath());
+                if (!isSpecialFileSystem){
+                    return sleuthProcess.getInputStream((int) tskId, tskContent.getUniquePath());
+                }else{
+                    return SpecialFileSystem.getSeekableInputStream(sleuthProcess.getInputStream((int) tskId, tskContent.getUniquePath()),tskId,address);
+                }
             } catch (TskCoreException e) {
                 throw new IOException(e);
             }