From b8028bf386b498ff2639c1b75262b39037e4b693 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 9 Jan 2024 16:00:08 +0100 Subject: [PATCH] thresholds: use dedicated storage Instead of a Host and IPPair table thresholding layer, use a dedicated THash to store both. This allows hashing on host+sid+tracker or ippair+sid+tracker, to create more unique hash keys. This allows for fewer hash collisions. The per rule tracking also uses this, so that the single big lock is no longer a single point of contention. Reimplement storage for flow thresholds to reuse as much logic as possible from the host/ippair/rule thresholds. Ticket: #426. --- src/detect-detection-filter.c | 12 +- src/detect-engine-build.c | 2 - src/detect-engine-threshold.c | 811 +++++++++++++--------------------- src/detect-engine-threshold.h | 16 +- src/detect-engine.c | 2 - src/detect-threshold.c | 163 ++----- src/detect-threshold.h | 15 - src/detect.h | 12 - src/flow-manager.c | 2 + src/host-timeout.c | 2 - src/ippair-timeout.c | 10 +- src/suricata.c | 1 + src/util-threshold-config.c | 61 +-- 13 files changed, 403 insertions(+), 706 deletions(-) diff --git a/src/detect-detection-filter.c b/src/detect-detection-filter.c index 47ce00829880..dbc73dd59aee 100644 --- a/src/detect-detection-filter.c +++ b/src/detect-detection-filter.c @@ -378,7 +378,7 @@ static int DetectDetectionFilterTestSig1(void) ThreadVars th_v; DetectEngineThreadCtx *det_ctx; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -418,7 +418,7 @@ static int DetectDetectionFilterTestSig1(void) DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -435,7 +435,7 @@ static int DetectDetectionFilterTestSig2(void) ThreadVars th_v; DetectEngineThreadCtx *det_ctx; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -480,7 +480,7 @@ static int DetectDetectionFilterTestSig2(void) DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -493,7 +493,7 @@ static int DetectDetectionFilterTestSig3(void) ThreadVars th_v; DetectEngineThreadCtx *det_ctx; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -556,7 +556,7 @@ static int DetectDetectionFilterTestSig3(void) DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index f69def1c7d54..82ef66898f1e 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -2219,8 +2219,6 @@ int SigGroupBuild(DetectEngineCtx *de_ctx) SCProfilingRuleInitCounters(de_ctx); #endif - ThresholdHashAllocate(de_ctx); - if (!DetectEngineMultiTenantEnabled()) { VarNameStoreActivate(); } diff --git a/src/detect-engine-threshold.c b/src/detect-engine-threshold.c index 1100a877ad0a..768b7375e14a 100644 --- a/src/detect-engine-threshold.c +++ b/src/detect-engine-threshold.c @@ -71,34 +71,150 @@ #include "action-globals.h" #include "util-validate.h" -static HostStorageId host_threshold_id = { .id = -1 }; /**< host storage id for thresholds */ -static IPPairStorageId ippair_threshold_id = { .id = -1 }; /**< ip pair storage id for thresholds */ +#include "util-thash.h" +#include "util-hash-lookup3.h" -HostStorageId ThresholdHostStorageId(void) +struct Thresholds { + THashTableContext *thash; +} ctx; + +static int ThresholdsInit(struct Thresholds *t); +static void ThresholdsDestroy(struct Thresholds *t); + +void ThresholdInit(void) { - return host_threshold_id; + ThresholdsInit(&ctx); } -void ThresholdInit(void) +void ThresholdDestroy(void) +{ + ThresholdsDestroy(&ctx); +} + +typedef struct ThresholdEntry_ { + uint32_t sid; /**< Signature id */ + uint32_t gid; /**< Signature group id */ + int track; /**< Track type: by_src, by_dst, etc */ + + uint32_t tv_timeout; /**< Timeout for new_action (for rate_filter) + its not "seconds", that define the time interval */ + uint32_t seconds; /**< Event seconds */ + uint32_t current_count; /**< Var for count control */ + + SCTime_t tv1; /**< Var for time control */ + + Address addr; /* used for src/dst/either tracking */ + Address addr2; /* used for both tracking */ +} ThresholdEntry; + +static int ThresholdEntrySet(void *dst, void *src) +{ + const ThresholdEntry *esrc = src; + ThresholdEntry *edst = dst; + memset(edst, 0, sizeof(*edst)); + *edst = *esrc; + return 0; +} + +static void ThresholdEntryFree(void *ptr) +{ + // nothing to free, base data is part of hash +} + +static inline uint32_t HashAddress(const Address *a) +{ + uint32_t key; + + if (a->family == AF_INET) { + key = a->addr_data32[0]; + } else if (a->family == AF_INET6) { + key = hashword(a->addr_data32, 4, 0); + } else + key = 0; + + return key; +} + +static inline int CompareAddress(const Address *a, const Address *b) +{ + if (a->family == b->family) { + switch (a->family) { + case AF_INET: + return (a->addr_data32[0] == b->addr_data32[0]); + case AF_INET6: + return CMP_ADDR(a, b); + } + } + return 0; +} + +static uint32_t ThresholdEntryHash(void *ptr) +{ + const ThresholdEntry *e = ptr; + uint32_t hash = e->sid + e->gid + e->track; + switch (e->track) { + case TRACK_BOTH: + hash += HashAddress(&e->addr2); + /* fallthrough */ + case TRACK_SRC: + case TRACK_DST: + hash += HashAddress(&e->addr); + break; + } + return hash; +} + +static bool ThresholdEntryCompare(void *a, void *b) { - host_threshold_id = HostStorageRegister("threshold", sizeof(void *), NULL, ThresholdListFree); - if (host_threshold_id.id == -1) { - FatalError("Can't initiate host storage for thresholding"); + const ThresholdEntry *e1 = a; + const ThresholdEntry *e2 = b; + SCLogDebug("sid1: %u sid2: %u", e1->sid, e2->sid); + if (!(e1->sid == e2->sid && e1->gid == e2->gid && e1->track == e2->track)) + return false; + switch (e1->track) { + case TRACK_BOTH: + if (!(CompareAddress(&e1->addr2, &e2->addr2))) + return false; + /* fallthrough */ + case TRACK_SRC: + case TRACK_DST: + if (!(CompareAddress(&e1->addr, &e2->addr))) + return false; + break; } - ippair_threshold_id = IPPairStorageRegister("threshold", sizeof(void *), NULL, ThresholdListFree); - if (ippair_threshold_id.id == -1) { - FatalError("Can't initiate IP pair storage for thresholding"); + return true; +} + +static bool ThresholdEntryExpire(void *data, const SCTime_t ts) +{ + const ThresholdEntry *e = data; + const SCTime_t entry = SCTIME_ADD_SECS(e->tv1, e->seconds); + if (SCTIME_CMP_GT(ts, entry)) { + return true; } + return false; } -int ThresholdHostHasThreshold(Host *host) +static int ThresholdsInit(struct Thresholds *t) { - return HostGetStorageById(host, host_threshold_id) ? 1 : 0; + uint64_t memcap = 16 * 1024 * 1024; + uint32_t hashsize = 16384; + t->thash = THashInit("thresholds", sizeof(ThresholdEntry), ThresholdEntrySet, + ThresholdEntryFree, ThresholdEntryHash, ThresholdEntryCompare, ThresholdEntryExpire, 0, + memcap, hashsize); + BUG_ON(t->thash == NULL); + return 0; +} +static void ThresholdsDestroy(struct Thresholds *t) +{ + if (t->thash) { + THashShutdown(t->thash); + } } -int ThresholdIPPairHasThreshold(IPPair *pair) +uint32_t ThresholdsExpire(const SCTime_t ts) { - return IPPairGetStorageById(pair, ippair_threshold_id) ? 1 : 0; + return THashExpire(ctx.thash, ts); } #include "util-hash.h" @@ -234,7 +350,9 @@ static int SetupCache(const Packet *p, const int8_t track, const int8_t retval, n->expires_at = expires; if (HashTableAdd(threshold_cache_ht, n, 0) == 0) { - (void)THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, n); + ThresholdCacheItem *r = THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, n); + DEBUG_VALIDATE_BUG_ON(r != NULL); // duplicate; should be impossible + (void)r; // only used by DEBUG_VALIDATE_BUG_ON return 1; } SCFree(n); @@ -358,98 +476,18 @@ const DetectThresholdData *SigGetThresholdTypeIter( return NULL; } -/** - * \brief Remove timeout threshold hash elements - * - * \param head Current head element of storage - * \param tv Current time - * - * \retval DetectThresholdEntry Return new head element or NULL if all expired - * - */ - -static DetectThresholdEntry *ThresholdTimeoutCheck(DetectThresholdEntry *head, SCTime_t ts) -{ - DetectThresholdEntry *tmp = head; - DetectThresholdEntry *prev = NULL; - DetectThresholdEntry *new_head = head; - - while (tmp != NULL) { - /* check if the 'check' timestamp is not before the creation ts. - * This can happen due to the async nature of the host timeout - * code that also calls this code from a management thread. */ - SCTime_t entry = SCTIME_ADD_SECS(tmp->tv1, (time_t)tmp->seconds); - if (SCTIME_CMP_LTE(ts, entry)) { - prev = tmp; - tmp = tmp->next; - continue; - } - - /* timed out */ - - DetectThresholdEntry *tde = tmp; - if (prev != NULL) { - prev->next = tmp->next; - } - else { - new_head = tmp->next; - } - tmp = tde->next; - SCFree(tde); - } - - return new_head; -} - -int ThresholdHostTimeoutCheck(Host *host, SCTime_t ts) -{ - DetectThresholdEntry* head = HostGetStorageById(host, host_threshold_id); - DetectThresholdEntry *new_head = ThresholdTimeoutCheck(head, ts); - if (new_head != head) { - HostSetStorageById(host, host_threshold_id, new_head); - } - return new_head == NULL; -} - -int ThresholdIPPairTimeoutCheck(IPPair *pair, SCTime_t ts) -{ - DetectThresholdEntry* head = IPPairGetStorageById(pair, ippair_threshold_id); - DetectThresholdEntry *new_head = ThresholdTimeoutCheck(head, ts); - if (new_head != head) { - IPPairSetStorageById(pair, ippair_threshold_id, new_head); - } - return new_head == NULL; -} +typedef struct FlowThresholdEntryList_ { + struct FlowThresholdEntryList_ *next; + ThresholdEntry threshold; +} FlowThresholdEntryList; -static DetectThresholdEntry *DetectThresholdEntryAlloc( - const DetectThresholdData *td, uint32_t sid, uint32_t gid) +static void FlowThresholdEntryListFree(FlowThresholdEntryList *list) { - SCEnter(); - - DetectThresholdEntry *ste = SCCalloc(1, sizeof(DetectThresholdEntry)); - if (unlikely(ste == NULL)) { - SCReturnPtr(NULL, "DetectThresholdEntry"); + for (FlowThresholdEntryList *i = list; i != NULL;) { + FlowThresholdEntryList *next = i->next; + SCFree(i); + i = next; } - - ste->sid = sid; - ste->gid = gid; - ste->track = td->track; - ste->seconds = td->seconds; - - SCReturnPtr(ste, "DetectThresholdEntry"); -} - -static DetectThresholdEntry *ThresholdHostLookupEntry(Host *h, - uint32_t sid, uint32_t gid) -{ - DetectThresholdEntry *e; - - for (e = HostGetStorageById(h, host_threshold_id); e != NULL; e = e->next) { - if (e->sid == sid && e->gid == gid) - break; - } - - return e; } /** struct for storing per flow thresholds. This will be stored in the Flow::flowvar list, so it @@ -458,13 +496,13 @@ typedef struct FlowVarThreshold_ { uint8_t type; uint8_t pad[7]; struct GenericVar_ *next; - DetectThresholdEntry *thresholds; + FlowThresholdEntryList *thresholds; } FlowVarThreshold; void FlowThresholdVarFree(void *ptr) { FlowVarThreshold *t = ptr; - ThresholdListFree(t->thresholds); + FlowThresholdEntryListFree(t->thresholds); SCFree(t); } @@ -481,21 +519,21 @@ static FlowVarThreshold *FlowThresholdVarGet(Flow *f) return NULL; } -static DetectThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid) +static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid) { FlowVarThreshold *t = FlowThresholdVarGet(f); if (t == NULL) return NULL; - for (DetectThresholdEntry *e = t->thresholds; e != NULL; e = e->next) { - if (e->sid == sid && e->gid == gid) { - return e; + for (FlowThresholdEntryList *e = t->thresholds; e != NULL; e = e->next) { + if (e->threshold.sid == sid && e->threshold.gid == gid) { + return &e->threshold; } } return NULL; } -static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time) +static int AddEntryToFlow(Flow *f, FlowThresholdEntryList *e, SCTime_t packet_time) { DEBUG_VALIDATE_BUG_ON(e == NULL); @@ -509,27 +547,11 @@ static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time GenericVarAppend(&f->flowvar, (GenericVar *)t); } - e->current_count = 1; - e->tv1 = packet_time; - e->tv_timeout = 0; e->next = t->thresholds; t->thresholds = e; return 0; } -static DetectThresholdEntry *ThresholdIPPairLookupEntry(IPPair *pair, - uint32_t sid, uint32_t gid) -{ - DetectThresholdEntry *e; - - for (e = IPPairGetStorageById(pair, ippair_threshold_id); e != NULL; e = e->next) { - if (e->sid == sid && e->gid == gid) - break; - } - - return e; -} - static int ThresholdHandlePacketSuppress(Packet *p, const DetectThresholdData *td, uint32_t sid, uint32_t gid) { @@ -591,285 +613,208 @@ static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action) } } -/** -* \brief Check if the entry reached threshold count limit -* -* \param lookup_tsh Current threshold entry -* \param td Threshold settings -* \param packet_time used to compare against previous detection and to set timeouts -* -* \retval int 1 if threshold reached for this entry -* -*/ -static int IsThresholdReached( - DetectThresholdEntry *lookup_tsh, const DetectThresholdData *td, SCTime_t packet_time) +static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te, + const SCTime_t packet_time, const uint32_t sid, const uint32_t gid) { - int ret = 0; - - /* Check if we have a timeout enabled, if so, - * we still matching (and enabling the new_action) */ - if (lookup_tsh->tv_timeout != 0) { - if ((SCTIME_SECS(packet_time) - lookup_tsh->tv_timeout) > td->timeout) { - /* Ok, we are done, timeout reached */ - lookup_tsh->tv_timeout = 0; - } else { - /* Already matching */ - ret = 1; - } /* else - if ((packet_time - lookup_tsh->tv_timeout) > td->timeout) */ - - } - else { - /* Update the matching state with the timeout interval */ - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(packet_time, entry)) { - lookup_tsh->current_count++; - if (lookup_tsh->current_count > td->count) { - /* Then we must enable the new action by setting a - * timeout */ - lookup_tsh->tv_timeout = SCTIME_SECS(packet_time); - ret = 1; - } - } else { - lookup_tsh->tv1 = packet_time; - lookup_tsh->current_count = 1; - } - } /* else - if (lookup_tsh->tv_timeout != 0) */ - - return ret; -} + te->sid = sid; + te->gid = gid; + te->track = td->track; + te->seconds = td->seconds; -static void AddEntryToHostStorage(Host *h, DetectThresholdEntry *e, SCTime_t packet_time) -{ - if (h && e) { - e->current_count = 1; - e->tv1 = packet_time; - e->tv_timeout = 0; - e->next = HostGetStorageById(h, host_threshold_id); - HostSetStorageById(h, host_threshold_id, e); - } -} + te->current_count = 1; + te->tv1 = packet_time; + te->tv_timeout = 0; -static void AddEntryToIPPairStorage(IPPair *pair, DetectThresholdEntry *e, SCTime_t packet_time) -{ - if (pair && e) { - e->current_count = 1; - e->tv1 = packet_time; - e->tv_timeout = 0; - e->next = IPPairGetStorageById(pair, ippair_threshold_id); - IPPairSetStorageById(pair, ippair_threshold_id, e); + switch (td->type) { + case TYPE_LIMIT: + case TYPE_RATE: + return 1; + case TYPE_THRESHOLD: + case TYPE_BOTH: + if (td->count == 1) + return 1; + return 0; + case TYPE_DETECTION: + return 0; } + return 0; } -/** - * \retval 2 silent match (no alert but apply actions) - * \retval 1 normal match - * \retval 0 no match - * - * If a new DetectThresholdEntry is generated to track the threshold - * for this rule, then it will be returned in new_tsh. - */ -static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh, - DetectThresholdEntry **new_tsh, const DetectThresholdData *td, - uint32_t sid, uint32_t gid, PacketAlert *pa) +static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *te, + const Packet *p, // ts only? - cache too + const uint32_t sid, PacketAlert *pa) { int ret = 0; - - switch(td->type) { + const SCTime_t packet_time = p->ts; + const SCTime_t entry = SCTIME_ADD_SECS(te->tv1, td->seconds); + switch (td->type) { case TYPE_LIMIT: - { SCLogDebug("limit"); - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - lookup_tsh->current_count++; - - if (lookup_tsh->current_count <= td->count) { - ret = 1; - } else { - ret = 2; + if (SCTIME_CMP_LTE(p->ts, entry)) { + te->current_count++; - if (PacketIsIPv4(p)) { - SetupCache(p, td->track, (int8_t)ret, sid, entry); - } - } + if (te->current_count <= td->count) { + ret = 1; } else { - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; + ret = 2; - ret = 1; + if (PacketIsIPv4(p)) { + SetupCache(p, td->track, (int8_t)ret, sid, entry); + } } } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); - + /* entry expired, reset */ + te->tv1 = p->ts; + te->current_count = 1; ret = 1; } break; - } case TYPE_THRESHOLD: - { - SCLogDebug("threshold"); - - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - lookup_tsh->current_count++; + if (SCTIME_CMP_LTE(p->ts, entry)) { + te->current_count++; - if (lookup_tsh->current_count >= td->count) { - ret = 1; - lookup_tsh->current_count = 0; - } - } else { - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; - } - } else { - if (td->count == 1) { + if (te->current_count >= td->count) { ret = 1; - } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + te->current_count = 0; } + } else { + te->tv1 = p->ts; + te->current_count = 1; } break; - } case TYPE_BOTH: - { - SCLogDebug("both"); - - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - /* within time limit */ - - lookup_tsh->current_count++; - if (lookup_tsh->current_count == td->count) { - ret = 1; - } else if (lookup_tsh->current_count > td->count) { - /* silent match */ - ret = 2; - - if (PacketIsIPv4(p)) { - SetupCache(p, td->track, (int8_t)ret, sid, entry); - } - } - } else { - /* expired, so reset */ - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; + if (SCTIME_CMP_LTE(p->ts, entry)) { + /* within time limit */ - /* if we have a limit of 1, this is a match */ - if (lookup_tsh->current_count == td->count) { - ret = 1; + te->current_count++; + if (te->current_count == td->count) { + ret = 1; + } else if (te->current_count > td->count) { + /* silent match */ + ret = 2; + + if (PacketIsIPv4(p)) { + SetupCache(p, td->track, (int8_t)ret, sid, entry); } } } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + /* expired, so reset */ + te->tv1 = p->ts; + te->current_count = 1; - /* for the first match we return 1 to - * indicate we should alert */ - if (td->count == 1) { + /* if we have a limit of 1, this is a match */ + if (te->current_count == td->count) { ret = 1; } } break; - } - /* detection_filter */ case TYPE_DETECTION: - { SCLogDebug("detection_filter"); - if (lookup_tsh != NULL) { - SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds); - if (SCTIME_CMP_LTE(p->ts, entry)) { - /* within timeout */ - lookup_tsh->current_count++; - if (lookup_tsh->current_count > td->count) { - ret = 1; - } - } else { - /* expired, reset */ - lookup_tsh->tv1 = p->ts; - lookup_tsh->current_count = 1; + if (SCTIME_CMP_LTE(p->ts, entry)) { + /* within timeout */ + te->current_count++; + if (te->current_count > td->count) { + ret = 1; } } else { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + /* expired, reset */ + te->tv1 = p->ts; + te->current_count = 1; } break; - } - /* rate_filter */ case TYPE_RATE: - { SCLogDebug("rate_filter"); ret = 1; - if (lookup_tsh && IsThresholdReached(lookup_tsh, td, p->ts)) { - RateFilterSetAction(pa, td->new_action); - } else if (!lookup_tsh) { - *new_tsh = DetectThresholdEntryAlloc(td, sid, gid); + /* Check if we have a timeout enabled, if so, + * we still matching (and enabling the new_action) */ + if (te->tv_timeout != 0) { + if ((SCTIME_SECS(packet_time) - te->tv_timeout) > td->timeout) { + /* Ok, we are done, timeout reached */ + te->tv_timeout = 0; + } else { + /* Already matching */ + RateFilterSetAction(pa, td->new_action); + } + } else { + /* Update the matching state with the timeout interval */ + if (SCTIME_CMP_LTE(packet_time, entry)) { + te->current_count++; + if (te->current_count > td->count) { + /* Then we must enable the new action by setting a + * timeout */ + te->tv_timeout = SCTIME_SECS(packet_time); + RateFilterSetAction(pa, td->new_action); + } + } else { + te->tv1 = packet_time; + te->current_count = 1; + } } break; - } - /* case TYPE_SUPPRESS: is not handled here */ - default: - SCLogError("type %d is not supported", td->type); } return ret; } -static int ThresholdHandlePacketIPPair(IPPair *pair, Packet *p, const DetectThresholdData *td, - uint32_t sid, uint32_t gid, PacketAlert *pa) -{ - int ret = 0; - - DetectThresholdEntry *lookup_tsh = ThresholdIPPairLookupEntry(pair, sid, gid); - SCLogDebug("ippair lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid); - - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa); - if (new_tsh != NULL) { - AddEntryToIPPairStorage(pair, new_tsh, p->ts); - } - - return ret; -} +#include "detect-engine-address-ipv6.h" -/** - * \retval 2 silent match (no alert but apply actions) - * \retval 1 normal match - * \retval 0 no match - */ -static int ThresholdHandlePacketHost(Host *h, Packet *p, const DetectThresholdData *td, - uint32_t sid, uint32_t gid, PacketAlert *pa) +static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const Signature *s, + const DetectThresholdData *td, PacketAlert *pa) { - int ret = 0; - DetectThresholdEntry *lookup_tsh = ThresholdHostLookupEntry(h, sid, gid); - SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid); - - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa); - if (new_tsh != NULL) { - AddEntryToHostStorage(h, new_tsh, p->ts); + /* fast track for count 1 threshold */ + if (td->count == 1 && td->type == TYPE_THRESHOLD) { + return 1; } - return ret; -} -static int ThresholdHandlePacketRule(DetectEngineCtx *de_ctx, Packet *p, - const DetectThresholdData *td, const Signature *s, PacketAlert *pa) -{ - int ret = 0; + ThresholdEntry lookup; + memset(&lookup, 0, sizeof(lookup)); + lookup.sid = s->id; + lookup.gid = s->gid; + lookup.track = td->track; + if (td->track == TRACK_SRC) { + COPY_ADDRESS(&p->src, &lookup.addr); + } else if (td->track == TRACK_DST) { + COPY_ADDRESS(&p->dst, &lookup.addr); + } else if (td->track == TRACK_BOTH) { + /* make sure lower ip address is first */ + if (PacketIsIPv4(p)) { + if (SCNtohl(p->src.addr_data32[0]) < SCNtohl(p->dst.addr_data32[0])) { + COPY_ADDRESS(&p->src, &lookup.addr); + COPY_ADDRESS(&p->dst, &lookup.addr2); + } else { + COPY_ADDRESS(&p->dst, &lookup.addr); + COPY_ADDRESS(&p->src, &lookup.addr2); + } + } else { + if (AddressIPv6Lt(&p->src, &p->dst)) { + COPY_ADDRESS(&p->src, &lookup.addr); + COPY_ADDRESS(&p->dst, &lookup.addr2); + } else { + COPY_ADDRESS(&p->dst, &lookup.addr); + COPY_ADDRESS(&p->src, &lookup.addr2); + } + } + } - DetectThresholdEntry* lookup_tsh = (DetectThresholdEntry *)de_ctx->ths_ctx.th_entry[s->num]; - SCLogDebug("by_rule lookup_tsh %p num %u", lookup_tsh, s->num); + struct THashDataGetResult res = THashGetFromHash(tctx->thash, &lookup); + if (res.data) { + SCLogDebug("found %p, is_new %s", res.data, BOOL2STR(res.is_new)); + int r; + ThresholdEntry *te = res.data->data; + if (res.is_new) { + // new threshold, set up + r = ThresholdSetup(td, te, p->ts, s->id, s->gid); + } else { + // existing, check/update + r = ThresholdCheckUpdate(td, te, p, s->id, pa); + } - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, s->id, s->gid, pa); - if (new_tsh != NULL) { - new_tsh->tv1 = p->ts; - new_tsh->current_count = 1; - new_tsh->tv_timeout = 0; - de_ctx->ths_ctx.th_entry[s->num] = new_tsh; + (void)THashDecrUsecnt(res.data); + THashDataUnlock(res.data); + return r; } - - return ret; + return 0; // TODO error? } /** @@ -881,15 +826,24 @@ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdDa uint32_t sid, uint32_t gid, PacketAlert *pa) { int ret = 0; - DetectThresholdEntry *lookup_tsh = ThresholdFlowLookupEntry(f, sid, gid); - SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid); - - DetectThresholdEntry *new_tsh = NULL; - ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa); - if (new_tsh != NULL) { - if (AddEntryToFlow(f, new_tsh, p->ts) == -1) { - SCFree(new_tsh); + ThresholdEntry *found = ThresholdFlowLookupEntry(f, sid, gid); + SCLogDebug("found %p sid %u gid %u", found, sid, gid); + + if (found == NULL) { + FlowThresholdEntryList *new = SCCalloc(1, sizeof(*new)); + if (new == NULL) + return 0; + + // new threshold, set up + ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid); + + if (AddEntryToFlow(f, new, p->ts) == -1) { + SCFree(new); + return 0; } + } else { + // existing, check/update + ret = ThresholdCheckUpdate(td, found, p, sid, pa); } return ret; } @@ -925,11 +879,8 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx SCReturnInt(cache_ret); } } - Host *src = HostGetHostFromHash(&p->src); - if (src) { - ret = ThresholdHandlePacketHost(src,p,td,s->id,s->gid,pa); - HostRelease(src); - } + + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_DST) { if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) { int cache_ret = CheckCache(p, td->track, s->id); @@ -937,21 +888,12 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx SCReturnInt(cache_ret); } } - Host *dst = HostGetHostFromHash(&p->dst); - if (dst) { - ret = ThresholdHandlePacketHost(dst,p,td,s->id,s->gid,pa); - HostRelease(dst); - } + + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_BOTH) { - IPPair *pair = IPPairGetIPPairFromHash(&p->src, &p->dst); - if (pair) { - ret = ThresholdHandlePacketIPPair(pair, p, td, s->id, s->gid, pa); - IPPairRelease(pair); - } + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_RULE) { - SCMutexLock(&de_ctx->ths_ctx.threshold_table_lock); - ret = ThresholdHandlePacketRule(de_ctx,p,td,s,pa); - SCMutexUnlock(&de_ctx->ths_ctx.threshold_table_lock); + ret = ThresholdGetFromHash(&ctx, p, s, td, pa); } else if (td->track == TRACK_FLOW) { if (p->flow) { ret = ThresholdHandlePacketFlow(p->flow, p, td, s->id, s->gid, pa); @@ -961,125 +903,6 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx SCReturnInt(ret); } -/** - * \brief Init threshold context hash tables - * - * \param de_ctx Detection Context - * - */ -void ThresholdHashInit(DetectEngineCtx *de_ctx) -{ - if (SCMutexInit(&de_ctx->ths_ctx.threshold_table_lock, NULL) != 0) { - FatalError("Threshold: Failed to initialize hash table mutex."); - } -} - -/** - * \brief Allocate threshold context hash tables - * - * \param de_ctx Detection Context - */ -void ThresholdHashAllocate(DetectEngineCtx *de_ctx) -{ - const Signature *s = de_ctx->sig_list; - bool has_by_rule_tracking = false; - - /* Find the signature with the highest signature number that is using - thresholding with by_rule tracking. */ - uint32_t highest_signum = 0; - while (s != NULL) { - if (s->sm_arrays[DETECT_SM_LIST_SUPPRESS] != NULL) { - const SigMatchData *smd = NULL; - do { - const DetectThresholdData *td = - SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_SUPPRESS); - if (td == NULL) { - continue; - } - if (td->track != TRACK_RULE) { - continue; - } - if (s->num >= highest_signum) { - highest_signum = s->num; - has_by_rule_tracking = true; - } - } while (smd != NULL); - } - - if (s->sm_arrays[DETECT_SM_LIST_THRESHOLD] != NULL) { - const SigMatchData *smd = NULL; - do { - const DetectThresholdData *td = - SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_THRESHOLD); - if (td == NULL) { - continue; - } - if (td->track != TRACK_RULE) { - continue; - } - if (s->num >= highest_signum) { - highest_signum = s->num; - has_by_rule_tracking = true; - } - } while (smd != NULL); - } - - s = s->next; - } - - /* Skip allocating if by_rule tracking is not used */ - if (has_by_rule_tracking == false) { - return; - } - - de_ctx->ths_ctx.th_size = highest_signum + 1; - de_ctx->ths_ctx.th_entry = SCCalloc(de_ctx->ths_ctx.th_size, sizeof(DetectThresholdEntry *)); - if (de_ctx->ths_ctx.th_entry == NULL) { - FatalError( - "failed to allocate memory for \"by_rule\" thresholding (tried to allocate %" PRIu32 - " entries)", - de_ctx->ths_ctx.th_size); - } -} - -/** - * \brief Destroy threshold context hash tables - * - * \param de_ctx Detection Context - * - */ -void ThresholdContextDestroy(DetectEngineCtx *de_ctx) -{ - if (de_ctx->ths_ctx.th_entry != NULL) { - for (uint32_t i = 0; i < de_ctx->ths_ctx.th_size; i++) { - if (de_ctx->ths_ctx.th_entry[i] != NULL) { - SCFree(de_ctx->ths_ctx.th_entry[i]); - } - } - SCFree(de_ctx->ths_ctx.th_entry); - } - SCMutexDestroy(&de_ctx->ths_ctx.threshold_table_lock); -} - -/** - * \brief this function will free all the entries of a list - * DetectTagDataEntry - * - * \param td pointer to DetectTagDataEntryList - */ -void ThresholdListFree(void *ptr) -{ - if (ptr != NULL) { - DetectThresholdEntry *entry = ptr; - - while (entry != NULL) { - DetectThresholdEntry *next_entry = entry->next; - SCFree(entry); - entry = next_entry; - } - } -} - /** * @} */ diff --git a/src/detect-engine-threshold.h b/src/detect-engine-threshold.h index b7e53842433d..086dba497a5a 100644 --- a/src/detect-engine-threshold.h +++ b/src/detect-engine-threshold.h @@ -26,16 +26,12 @@ #define SURICATA_DETECT_ENGINE_THRESHOLD_H #include "detect.h" -#include "host.h" -#include "ippair.h" -#include "host-storage.h" +#include "detect-threshold.h" void ThresholdInit(void); +void ThresholdDestroy(void); -HostStorageId ThresholdHostStorageId(void); -int ThresholdHostHasThreshold(Host *); - -int ThresholdIPPairHasThreshold(IPPair *pair); +uint32_t ThresholdsExpire(const SCTime_t ts); const DetectThresholdData *SigGetThresholdTypeIter( const Signature *, const SigMatchData **, int list); @@ -43,12 +39,6 @@ int PacketAlertThreshold(DetectEngineCtx *, DetectEngineThreadCtx *, const DetectThresholdData *, Packet *, const Signature *, PacketAlert *); -void ThresholdHashInit(DetectEngineCtx *); -void ThresholdHashAllocate(DetectEngineCtx *); -void ThresholdContextDestroy(DetectEngineCtx *); - -int ThresholdHostTimeoutCheck(Host *, SCTime_t); -int ThresholdIPPairTimeoutCheck(IPPair *, SCTime_t); void ThresholdListFree(void *ptr); void ThresholdCacheThreadFree(void); diff --git a/src/detect-engine.c b/src/detect-engine.c index 3ec446168709..efd1b0f8850b 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2506,7 +2506,6 @@ static DetectEngineCtx *DetectEngineCtxInitReal( SigGroupHeadHashInit(de_ctx); MpmStoreInit(de_ctx); - ThresholdHashInit(de_ctx); DetectParseDupSigHashInit(de_ctx); DetectAddressMapInit(de_ctx); DetectMetadataHashInit(de_ctx); @@ -2620,7 +2619,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) MpmStoreFree(de_ctx); DetectParseDupSigHashFree(de_ctx); SCSigSignatureOrderingModuleCleanup(de_ctx); - ThresholdContextDestroy(de_ctx); SigCleanSignatures(de_ctx); if (de_ctx->sig_array) SCFree(de_ctx->sig_array); diff --git a/src/detect-threshold.c b/src/detect-threshold.c index d952b5ec36fc..0d5d01485514 100644 --- a/src/detect-threshold.c +++ b/src/detect-threshold.c @@ -509,7 +509,7 @@ static int DetectThresholdTestSig1(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -587,8 +587,8 @@ static int DetectThresholdTestSig1(void) UTHFreePackets(&p, 1); - HostShutdown(); end: + ThresholdDestroy(); return result; } @@ -610,7 +610,7 @@ static int DetectThresholdTestSig2(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -658,15 +658,12 @@ static int DetectThresholdTestSig2(void) goto cleanup; cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -681,94 +678,49 @@ static int DetectThresholdTestSig2(void) static int DetectThresholdTestSig3(void) { - Packet *p = NULL; - Signature *s = NULL; ThreadVars th_v; - DetectEngineThreadCtx *det_ctx; - int result = 0; - int alerts = 0; - DetectThresholdEntry *lookup_tsh = NULL; - - HostInitConfig(HOST_QUIET); - memset(&th_v, 0, sizeof(th_v)); - p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); + ThresholdInit(); + Packet *p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); - if (de_ctx == NULL) { - goto end; - } - + FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; - s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:10;)"); - if (s == NULL) { - goto end; - } + Signature *s = DetectEngineAppendSig(de_ctx, + "alert tcp any any -> any 80 (msg:\"Threshold limit\"; threshold: type limit, " + "track by_dst, count 5, seconds 60; sid:10;)"); + FAIL_IF_NULL(s); SigGroupBuild(de_ctx); + DetectEngineThreadCtx *det_ctx; DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); p->ts = TimeGet(); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); - - Host *host = HostLookupHostFromHash(&p->dst); - if (host == NULL) { - printf("host not found: "); - goto cleanup; - } - - if (!(ThresholdHostHasThreshold(host))) { - HostRelease(host); - printf("host has no threshold: "); - goto cleanup; - } - HostRelease(host); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); TimeSetIncrementTime(200); p->ts = TimeGet(); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); - - host = HostLookupHostFromHash(&p->dst); - if (host == NULL) { - printf("host not found: "); - goto cleanup; - } - HostRelease(host); - - lookup_tsh = HostGetStorageById(host, ThresholdHostStorageId()); - if (lookup_tsh == NULL) { - HostRelease(host); - printf("lookup_tsh is NULL: "); - goto cleanup; - } - - alerts = lookup_tsh->current_count; - - if (alerts == 3) - result = 1; - else { - printf("alerts %u != 3: ", alerts); - goto cleanup; - } - -cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); + FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); -end: UTHFreePackets(&p, 1); - HostShutdown(); - return result; + ThresholdDestroy(); + PASS; } /** @@ -789,7 +741,7 @@ static int DetectThresholdTestSig4(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -834,14 +786,11 @@ static int DetectThresholdTestSig4(void) goto cleanup; cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -863,7 +812,7 @@ static int DetectThresholdTestSig5(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -921,15 +870,12 @@ static int DetectThresholdTestSig5(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -942,7 +888,7 @@ static int DetectThresholdTestSig6Ticks(void) int result = 0; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -1004,15 +950,12 @@ static int DetectThresholdTestSig6Ticks(void) goto cleanup; cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1029,7 +972,7 @@ static int DetectThresholdTestSig7(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1095,14 +1038,11 @@ static int DetectThresholdTestSig7(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1119,7 +1059,7 @@ static int DetectThresholdTestSig8(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1185,14 +1125,11 @@ static int DetectThresholdTestSig8(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1209,7 +1146,7 @@ static int DetectThresholdTestSig9(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1275,14 +1212,11 @@ static int DetectThresholdTestSig9(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1299,7 +1233,7 @@ static int DetectThresholdTestSig10(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1365,14 +1299,11 @@ static int DetectThresholdTestSig10(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1389,7 +1320,7 @@ static int DetectThresholdTestSig11(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1455,14 +1386,11 @@ static int DetectThresholdTestSig11(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); return result; } @@ -1479,7 +1407,7 @@ static int DetectThresholdTestSig12(void) int alerts = 0; int drops = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); @@ -1545,14 +1473,12 @@ static int DetectThresholdTestSig12(void) } cleanup: - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); - DetectEngineThreadCtxDeinit(&th_v, (void*)det_ctx); DetectEngineCtxFree(de_ctx); end: UTHFreePackets(&p, 1); HostShutdown(); + ThresholdDestroy(); return result; } @@ -1573,7 +1499,7 @@ static int DetectThresholdTestSig13(void) DetectEngineThreadCtx *det_ctx; int alerts = 0; - HostInitConfig(HOST_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -1616,12 +1542,10 @@ static int DetectThresholdTestSig13(void) FAIL_IF(alerts != 4); - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); UTHFreePackets(&p, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1644,8 +1568,7 @@ static int DetectThresholdTestSig14(void) int alerts1 = 0; int alerts2 = 0; - HostInitConfig(HOST_QUIET); - IPPairInitConfig(IPPAIR_QUIET); + ThresholdInit(); memset(&th_v, 0, sizeof(th_v)); p1 = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80); @@ -1699,13 +1622,11 @@ static int DetectThresholdTestSig14(void) FAIL_IF(alerts1 != 3); FAIL_IF(alerts2 != 3); - SigGroupCleanup(de_ctx); - SigCleanSignatures(de_ctx); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); UTHFreePackets(&p1, 1); UTHFreePackets(&p2, 1); - HostShutdown(); + ThresholdDestroy(); PASS; } diff --git a/src/detect-threshold.h b/src/detect-threshold.h index 5321fd54bdac..e1ef71bd1089 100644 --- a/src/detect-threshold.h +++ b/src/detect-threshold.h @@ -62,21 +62,6 @@ typedef struct DetectThresholdData_ { DetectAddressHead addrs; } DetectThresholdData; -typedef struct DetectThresholdEntry_ { - uint32_t sid; /**< Signature id */ - uint32_t gid; /**< Signature group id */ - - uint32_t tv_timeout; /**< Timeout for new_action (for rate_filter) - its not "seconds", that define the time interval */ - uint32_t seconds; /**< Event seconds */ - uint32_t current_count; /**< Var for count control */ - int track; /**< Track type: by_src, by_src */ - - SCTime_t tv1; /**< Var for time control */ - struct DetectThresholdEntry_ *next; -} DetectThresholdEntry; - - /** * Registration function for threshold: keyword */ diff --git a/src/detect.h b/src/detect.h index a4f154523280..397c4ae8aadc 100644 --- a/src/detect.h +++ b/src/detect.h @@ -785,17 +785,6 @@ typedef struct DetectEngineLookupFlow_ { struct SigGroupHead_ *sgh[256]; } DetectEngineLookupFlow; -#include "detect-threshold.h" - -/** \brief threshold ctx */ -typedef struct ThresholdCtx_ { - SCMutex threshold_table_lock; /**< Mutex for hash table */ - - /** to support rate_filter "by_rule" option */ - DetectThresholdEntry **th_entry; - uint32_t th_size; -} ThresholdCtx; - typedef struct SigString_ { char *filename; char *sig_str; @@ -886,7 +875,6 @@ typedef struct DetectEngineCtx_ { HashListTable *dup_sig_hash_table; DetectEngineIPOnlyCtx io_ctx; - ThresholdCtx ths_ctx; /* maximum recursion depth for content inspection */ int inspection_recursion_limit; diff --git a/src/flow-manager.c b/src/flow-manager.c index bd547de95bd1..de55ed136f6a 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -47,6 +47,7 @@ #include "util-debug.h" #include "threads.h" +#include "detect-engine-threshold.h" #include "host-timeout.h" #include "defrag-hash.h" @@ -931,6 +932,7 @@ static TmEcode FlowManager(ThreadVars *th_v, void *thread_data) HostTimeoutHash(ts); IPPairTimeoutHash(ts); HttpRangeContainersTimeoutHash(ts); + ThresholdsExpire(ts); other_last_sec = (uint32_t)SCTIME_SECS(ts); } } diff --git a/src/host-timeout.c b/src/host-timeout.c index d1e358c1b1ca..35706c2cfd24 100644 --- a/src/host-timeout.c +++ b/src/host-timeout.c @@ -25,7 +25,6 @@ #include "host.h" #include "detect-engine-tag.h" -#include "detect-engine-threshold.h" #include "host-bit.h" #include "host-timeout.h" @@ -53,7 +52,6 @@ static int HostHostTimedOut(Host *h, SCTime_t ts) busy |= (h->iprep && SRepHostTimedOut(h) == 0); busy |= (TagHostHasTag(h) && TagTimeoutCheck(h, ts) == 0); - busy |= (ThresholdHostHasThreshold(h) && ThresholdHostTimeoutCheck(h, ts) == 0); busy |= (HostHasHostBits(h) && HostBitsTimedoutCheck(h, ts) == 0); SCLogDebug("host %p %s", h, busy ? "still active" : "timed out"); return !busy; diff --git a/src/ippair-timeout.c b/src/ippair-timeout.c index a3627c1ea911..02e88b7b3477 100644 --- a/src/ippair-timeout.c +++ b/src/ippair-timeout.c @@ -25,7 +25,6 @@ #include "ippair.h" #include "ippair-bit.h" #include "ippair-timeout.h" -#include "detect-engine-threshold.h" /** \internal * \brief See if we can really discard this ippair. Check use_cnt reference. @@ -39,8 +38,6 @@ static int IPPairTimedOut(IPPair *h, SCTime_t ts) { int vars = 0; - int thresholds = 0; - /** never prune a ippair that is used by a packet * we are currently processing in one of the threads */ if (SC_ATOMIC_GET(h->use_cnt) > 0) { @@ -50,12 +47,7 @@ static int IPPairTimedOut(IPPair *h, SCTime_t ts) if (IPPairHasBits(h) && IPPairBitsTimedoutCheck(h, ts) == 0) { vars = 1; } - - if (ThresholdIPPairHasThreshold(h) && ThresholdIPPairTimeoutCheck(h, ts) == 0) { - thresholds = 1; - } - - if (vars || thresholds) { + if (vars) { return 0; } diff --git a/src/suricata.c b/src/suricata.c index f7a03a208c6d..de60255199bd 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -369,6 +369,7 @@ void GlobalsInitPreConfig(void) void GlobalsDestroy(void) { SCInstance *suri = &suricata; + ThresholdDestroy(); HostShutdown(); HTPFreeConfig(); HTPAtExitPrintStats(); diff --git a/src/util-threshold-config.c b/src/util-threshold-config.c index 998dde56a9e5..6df1eebb0798 100644 --- a/src/util-threshold-config.c +++ b/src/util-threshold-config.c @@ -37,6 +37,7 @@ #include "detect.h" #include "detect-engine.h" #include "detect-engine-address.h" +#include "detect-engine-threshold.h" #include "detect-threshold.h" #include "detect-parse.h" #include "detect-engine-build.h" @@ -1534,7 +1535,7 @@ static int SCThresholdConfTest09(void) ThreadVars th_v; memset(&th_v, 0, sizeof(th_v)); - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP); FAIL_IF_NULL(p); @@ -1603,7 +1604,7 @@ static int SCThresholdConfTest09(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1615,7 +1616,7 @@ static int SCThresholdConfTest09(void) */ static int SCThresholdConfTest10(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); /* Create two different packets falling to the same rule, and * because count:3, we should drop on match #4. @@ -1685,15 +1686,15 @@ static int SCThresholdConfTest10(void) SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); FAIL_IF(PacketTestAction(p1, ACTION_DROP)); FAIL_IF(PacketAlertCheck(p1, 10) != 1); - +#if 0 /* Ensure that a Threshold entry was installed at the sig */ FAIL_IF_NULL(de_ctx->ths_ctx.th_entry[s->num]); - +#endif UTHFreePacket(p1); UTHFreePacket(p2); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1705,7 +1706,7 @@ static int SCThresholdConfTest10(void) */ static int SCThresholdConfTest11(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP); FAIL_IF_NULL(p); @@ -1798,7 +1799,7 @@ static int SCThresholdConfTest11(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1810,7 +1811,7 @@ static int SCThresholdConfTest11(void) */ static int SCThresholdConfTest12(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP); FAIL_IF_NULL(p); @@ -1903,7 +1904,7 @@ static int SCThresholdConfTest12(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -1948,7 +1949,7 @@ static int SCThresholdConfTest13(void) */ static int SCThresholdConfTest14(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p1 = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10", "192.168.0.100", 1234, 24); @@ -1997,7 +1998,7 @@ static int SCThresholdConfTest14(void) DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2009,7 +2010,7 @@ static int SCThresholdConfTest14(void) */ static int SCThresholdConfTest15(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10", "192.168.0.100", 1234, 24); @@ -2045,7 +2046,7 @@ static int SCThresholdConfTest15(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2057,7 +2058,7 @@ static int SCThresholdConfTest15(void) */ static int SCThresholdConfTest16(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.1.1", "192.168.0.100", 1234, 24); @@ -2092,7 +2093,7 @@ static int SCThresholdConfTest16(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2104,7 +2105,7 @@ static int SCThresholdConfTest16(void) */ static int SCThresholdConfTest17(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10", "192.168.0.100", 1234, 24); @@ -2140,7 +2141,7 @@ static int SCThresholdConfTest17(void) UTHFreePacket(p); DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2171,7 +2172,7 @@ static FILE *SCThresholdConfGenerateInvalidDummyFD12(void) */ static int SCThresholdConfTest18(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2192,7 +2193,7 @@ static int SCThresholdConfTest18(void) FAIL_IF_NOT(de->type == TYPE_SUPPRESS && de->track == TRACK_DST); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2223,7 +2224,7 @@ static FILE *SCThresholdConfGenerateInvalidDummyFD13(void) */ static int SCThresholdConfTest19(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2241,7 +2242,7 @@ static int SCThresholdConfTest19(void) FAIL_IF_NULL(de); FAIL_IF_NOT(de->type == TYPE_SUPPRESS && de->track == TRACK_DST); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2273,7 +2274,7 @@ static FILE *SCThresholdConfGenerateValidDummyFD20(void) */ static int SCThresholdConfTest20(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2306,7 +2307,7 @@ static int SCThresholdConfTest20(void) FAIL_IF_NOT(smd->is_last); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2319,7 +2320,7 @@ static int SCThresholdConfTest20(void) */ static int SCThresholdConfTest21(void) { - HostInitConfig(HOST_QUIET); + ThresholdInit(); DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); de_ctx->flags |= DE_QUIET; @@ -2351,7 +2352,7 @@ static int SCThresholdConfTest21(void) FAIL_IF_NOT(smd->is_last); DetectEngineCtxFree(de_ctx); - HostShutdown(); + ThresholdDestroy(); PASS; } @@ -2384,7 +2385,7 @@ static int SCThresholdConfTest22(void) ThreadVars th_v; memset(&th_v, 0, sizeof(th_v)); - IPPairInitConfig(IPPAIR_QUIET); + ThresholdInit(); /* This packet will cause rate_filter */ Packet *p1 = UTHBuildPacketSrcDst((uint8_t*)"lalala", 6, IPPROTO_TCP, "172.26.0.1", "172.26.0.10"); @@ -2487,7 +2488,7 @@ static int SCThresholdConfTest22(void) DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - IPPairShutdown(); + ThresholdDestroy(); PASS; } @@ -2521,7 +2522,7 @@ static int SCThresholdConfTest23(void) ThreadVars th_v; memset(&th_v, 0, sizeof(th_v)); - IPPairInitConfig(IPPAIR_QUIET); + ThresholdInit(); /* Create two packets between same addresses in opposite direction */ Packet *p1 = UTHBuildPacketSrcDst((uint8_t*)"lalala", 6, IPPROTO_TCP, "172.26.0.1", "172.26.0.10"); @@ -2568,7 +2569,7 @@ static int SCThresholdConfTest23(void) DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx); DetectEngineCtxFree(de_ctx); - IPPairShutdown(); + ThresholdDestroy(); PASS; } #endif /* UNITTESTS */