Integrate tlshc library, calculate telfhash and import hash in ELF mo…

…dule (#1624)

* Add simple_str string implementation

* Integrate tlshc library, calculate telfhash and import hash in the ELF module.

* Hide usage of MD5 into #ifdef crypto

bazel/yara.bzl

@@ -6,6 +6,8 @@ YARA_CONFIG_OPTS = [
     "-DHAVE_STDBOOL_H=1",
     # "-DHAVE__MKGMTIME=1",
     "-DHAVE_TIMEGM=1",
+    "-DBUCKETS_128=1", # Defining TLSH function
+    "-DCHECKSUM_1B=1", # Defining TLSH function
 ]
 
 YARA_COPTS = YARA_CONFIG_OPTS + [
@@ -126,6 +128,7 @@ def yara_library(
             "libyara/include/yara/dex.h",
             "libyara/include/yara/dotnet.h",
             "libyara/include/yara/elf.h",
+            "libyara/include/yara/elf_utils.h",
             "libyara/include/yara/endian.h",
             "libyara/include/yara/error.h",
             "libyara/include/yara/exec.h",
@@ -152,6 +155,7 @@ def yara_library(
             "libyara/include/yara/rules.h",
             "libyara/include/yara/scan.h",
             "libyara/include/yara/scanner.h",
+            "libyara/include/yara/simple_str.h",
             "libyara/include/yara/sizedstr.h",
             "libyara/include/yara/stack.h",
             "libyara/include/yara/stopwatch.h",
@@ -161,6 +165,7 @@ def yara_library(
             "libyara/include/yara/types.h",
             "libyara/include/yara/unaligned.h",
             "libyara/include/yara/utils.h",
+            "libyara/include/tlshc/tlsh.h",
             "libyara/lexer.c",
             "libyara/libyara.c",
             "libyara/mem.c",
@@ -182,12 +187,18 @@ def yara_library(
             "libyara/rules.c",
             "libyara/scan.c",
             "libyara/scanner.c",
+            "libyara/simple_str.c",
             "libyara/sizedstr.c",
             "libyara/stack.c",
             "libyara/stopwatch.c",
             "libyara/stream.c",
             "libyara/strutils.c",
             "libyara/threading.c",
+            "libyara/tlshc/tlsh.c",
+            "libyara/tlshc/tlsh_impl.c",
+            "libyara/tlshc/tlsh_impl.h",
+            "libyara/tlshc/tlsh_util.c",
+            "libyara/tlshc/tlsh_util.h",
         ],
         hdrs = [
             "libyara/include/yara.h",

configure.ac

@@ -367,6 +367,9 @@ AS_IF(
   [test x$proc_interface != xnone],[AC_DEFINE([HAVE_SCAN_PROC_IMPL],[1])],
   [test x$proc_interface = xnone],[AC_DEFINE([HAVE_SCAN_PROC_IMPL],[0])])
 
+# Configure TLSH function
+CFLAGS="$CFLAGS -DBUCKETS_128=1 -DCHECKSUM_1B=1"
+
 AC_SUBST([PC_REQUIRES_PRIVATE])
 AC_SUBST([PC_LIBS_PRIVATE])
 

docs/modules/elf.rst

@@ -371,6 +371,17 @@ Reference
 
         The section index which the symbol is associated with.
 
+.. c:function:: telfhash()
+
+    Function returning Telfhash - TLSH hash of the ELF export and import symbols.
+
+     *Example: elf.telfhash() == "t166a00284751084526486df8b5df5b2fccb3f511dbc188c37156f5e714a11bc5d71014d"*
+
+.. c:function:: import_md5()
+
+    Function returning Import Hash - MD5 hash of the ELF imported symbols.
+
+     *Example: elf.import_md5() == "c3eca50cbb03400a6e91b9fe48da0c0c"*
 
 
 

libyara/Makefile.am

@@ -157,6 +157,7 @@ yarainclude_HEADERS = \
 	include/yara/rules.h \
 	include/yara/scan.h \
 	include/yara/scanner.h \
+	include/yara/simple_str.h \
 	include/yara/sizedstr.h \
 	include/yara/stack.h \
 	include/yara/stopwatch.h \
@@ -226,11 +227,15 @@ libyara_la_SOURCES = \
 	rules.c \
 	scan.c \
 	scanner.c \
+	simple_str.c \
 	sizedstr.c \
 	stack.c \
 	stopwatch.c \
 	strutils.c \
 	stream.c \
+	tlshc/tlsh.c \
+	tlshc/tlsh_impl.c \
+	tlshc/tlsh_util.c \
 	threading.c
 
 

libyara/include/tlshc/tlsh.h

@@ -0,0 +1,64 @@
+#ifndef __TLSH_TLSH_H__
+#define __TLSH_TLSH_H__
+
+#include <stdbool.h>
+
+#define TLSH_OPTION_CONSERVATIVE 2
+#define TLSH_OPTION_KEEP_BUCKET  4
+#define TLSH_OPTION_PRIVATE      8
+#define TLSH_OPTION_THREADED     16
+
+// Define TLSH_STRING_LEN_REQ, which is the string length of "T1" + the hex
+// value of the Tlsh hash. BUCKETS_256 & CHECKSUM_3B are compiler switches
+// defined in CMakeLists.txt
+#if defined BUCKETS_256
+    #define TLSH_STRING_LEN_REQ          136
+    // changed the minimum data length to 256 for version 3.3
+    #define MIN_DATA_LENGTH              50
+    // added the -force option for version 3.5
+    // added the -conservatibe option for version 3.17
+    #define MIN_CONSERVATIVE_DATA_LENGTH 256
+#endif
+
+#if defined BUCKETS_128
+    #define TLSH_STRING_LEN_REQ          72
+    // changed the minimum data length to 256 for version 3.3
+    #define MIN_DATA_LENGTH              50
+    // added the -force option for version 3.5
+    // added the -conservatibe option for version 3.17
+    #define MIN_CONSERVATIVE_DATA_LENGTH 256
+#endif
+
+#if defined BUCKETS_48
+    // No 3 Byte checksum option for 48 Bucket min hash
+    #define TLSH_STRING_LEN              30
+    // changed the minimum data length to 256 for version 3.3
+    #define MIN_DATA_LENGTH              10
+    // added the -force option for version 3.5
+    #define MIN_CONSERVATIVE_DATA_LENGTH 10
+#endif
+
+#define TLSH_STRING_BUFFER_LEN (TLSH_STRING_LEN_REQ + 1)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct TlshImpl TlshImpl;
+
+typedef struct {
+    TlshImpl* impl;
+} Tlsh;
+
+Tlsh* tlsh_new();
+void tlsh_free(Tlsh* tlsh);
+void tlsh_reset(Tlsh* tlsh);
+int tlsh_update(Tlsh* tlsh, const unsigned char* data, unsigned int len);
+int tlsh_final(Tlsh* tlsh, const unsigned char* data, unsigned int len, int tlsh_option);
+const char* tlsh_get_hash(Tlsh* tlsh, bool showvers);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // __TLSH_TLSH_H__

libyara/include/yara/elf.h

@@ -152,6 +152,11 @@ typedef uint64_t elf64_xword_t;
 #define ELF_STT_COMMON  5  // Symbol is a common data object
 #define ELF_STT_TLS     6  // Symbol is thread-local data object
 
+#define ELF_STV_DEFAULT   0  // Visibility by binding
+#define ELF_STV_INTERNAL  1  // Reserved
+#define ELF_STV_HIDDEN    2  // Not visible to other components
+#define ELF_STV_PROTECTED 3  // Visible in other but cannot be preempted.
+
 #define ELF_STB_LOCAL  0  // Local symbol
 #define ELF_STB_GLOBAL 1  // Global symbol
 #define ELF_STB_WEAK   2  // Weak symbol
@@ -162,6 +167,10 @@ typedef uint64_t elf64_xword_t;
 
 #define ELF_PN_XNUM 0xffff
 
+#define ELF_SHN_UNDEF  0       // Missing, undefined section index
+#define ELF_SHN_ABS    0xFFF1  // Absolute references, not affected by relocs
+#define ELF_SHN_COMMON 0xFFF2  // Symbols relative to this are common
+
 #pragma pack(push, 1)
 
 typedef struct

libyara/include/yara/elf_utils.h

@@ -0,0 +1,34 @@
+#ifndef YR_ELF_UTILS_H
+#define YR_ELF_UTILS_H
+
+#include <yara/elf.h>
+
+typedef struct _ELF_SYMBOL
+{
+  char *name;
+  int value;
+  int size;
+  int type;
+  int bind;
+  int shndx;
+  int visibility;
+
+  struct _ELF_SYMBOL *next; // Next symbol in the list
+} ELF_SYMBOL;
+
+// Linked list of symbols
+typedef struct _ELF_SYMBOL_LIST
+{
+  int count;
+  ELF_SYMBOL *symbols;
+} ELF_SYMBOL_LIST;
+
+typedef struct _ELF
+{
+  ELF_SYMBOL_LIST *symtab;
+  ELF_SYMBOL_LIST *dynsym;
+  char *telfhash;
+  char *import_hash;
+} ELF;
+
+#endif //YR_ELF_UTILS_H

libyara/include/yara/simple_str.h

@@ -0,0 +1,21 @@
+#ifndef _SIMPLESTR_H
+#define _SIMPLESTR_H
+
+#include <yara/types.h>
+
+/* Simple dynamic string implementation for more readable/maintainable code
+   Can be further optimized */
+typedef struct _SIMPLE_STR
+{
+  uint32_t len;
+  uint32_t cap;
+  char* str;
+} SIMPLE_STR, *PSIMPLE_STR;
+
+SIMPLE_STR* sstr_new(const char* s);
+SIMPLE_STR* sstr_newf(const char* fmt, ...);
+void sstr_free(SIMPLE_STR* ss);
+bool sstr_appendf(SIMPLE_STR* ss, const char* fmt, ...);
+char* sstr_move(SIMPLE_STR* ss);
+
+#endif