Skip to content

Commit

Permalink
More minemeld changes
Browse files Browse the repository at this point in the history
Second take on minemeld integration. Found a lot I missed from the previous push.

- Added lookup functions for domain and ip list.
- Added documentation.
- Modified the ThreatIntelPluginConfig for the spaumhaus plugin to rename from tor_enabled to spaumhaus_enabled.
- Modified the content pack to include minemeld.
  • Loading branch information
Chris Forsythe committed May 9, 2018
1 parent 5564e12 commit 8019b62
Show file tree
Hide file tree
Showing 14 changed files with 360 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ public void handleUpdatedClusterConfig(ClusterConfigChangedEvent clusterConfigCh
if (previous.abusechRansomEnabled() != currentVersion.abusechRansomEnabled()) {
adaptersToLoad.add("abuse-ch-ransomware-domains", "abuse-ch-ransomware-ip");
}
if (previous.minemeldEnabled() != currentVersion.minemeldEnabled()) {
adaptersToLoad.add("minemeld-domains", "minemeld-ip");
}
if (previous.torEnabled() != currentVersion.torEnabled()) {
adaptersToLoad.add("tor-exit-node");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,24 @@ public abstract class ThreatIntelPluginConfiguration {

@JsonProperty("abusech_ransom_enabled")
public abstract boolean abusechRansomEnabled();

@JsonProperty("minemeld_enabled")
public abstract boolean minemeldEnabled();

@JsonCreator
public static ThreatIntelPluginConfiguration create(@JsonProperty("otx_enabled") boolean otxEnabled,
@JsonProperty("otx_api_key") @Nullable String otxApiKey,
@JsonProperty("tor_enabled") boolean torEnabled,
@JsonProperty("spamhaus_enabled") boolean spamhausEnabled,
@JsonProperty("abusech_ransom_enabled") boolean abusechRansomEnabled) {
@JsonProperty("abusech_ransom_enabled") boolean abusechRansomEnabled,
@JsonProperty("minemeld_enabled") boolean ) {
return builder()
.otxEnabled(otxEnabled)
.otxApiKey(otxApiKey)
.torEnabled(torEnabled)
.spamhausEnabled(spamhausEnabled)
.abusechRansomEnabled(abusechRansomEnabled)
.minemeldEnabled(minemeldEnabled)
.build();
}

Expand All @@ -56,6 +61,7 @@ public static ThreatIntelPluginConfiguration defaults() {
.torEnabled(false)
.spamhausEnabled(false)
.abusechRansomEnabled(false)
.minemeldEnabled(false)
.build();
}

Expand All @@ -72,6 +78,8 @@ public static abstract class Builder {
public abstract Builder spamhausEnabled(boolean spamhausEnabled);

public abstract Builder abusechRansomEnabled(boolean abusechRansomEnabled);

public abstract Builder minemeldEnabled(boolean minemeldEnabled);

public abstract ThreatIntelPluginConfiguration build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.google.inject.multibindings.MapBinder;
import org.graylog.plugins.pipelineprocessor.ast.functions.Function;
import org.graylog.plugins.threatintel.adapters.abusech.AbuseChRansomAdapter;
import org.graylog.plugins.threatintel.adapters.minemeld.MineMeldBlockListAdapter;
import org.graylog.plugins.threatintel.adapters.otx.OTXDataAdapter;
import org.graylog.plugins.threatintel.functions.DomainFunctions;
import org.graylog.plugins.threatintel.functions.IPFunctions;
Expand All @@ -15,6 +16,8 @@
import org.graylog.plugins.threatintel.functions.GenericLookupResult;
import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomDomainLookupFunction;
import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomIpLookupFunction;
import org.graylog.plugins.threatintel.functions.minemeld.MineMeldDomainLookupFunction;
import org.graylog.plugins.threatintel.functions.minemeld.MineMeldIpLookupFunction;
import org.graylog.plugins.threatintel.functions.global.GlobalDomainLookupFunction;
import org.graylog.plugins.threatintel.functions.global.GlobalIpLookupFunction;
import org.graylog.plugins.threatintel.functions.otx.OTXDomainLookupFunction;
Expand Down Expand Up @@ -57,6 +60,11 @@ protected void configure() {
// abuse.ch Ransomware
addMessageProcessorFunction(AbuseChRansomDomainLookupFunction.NAME, AbuseChRansomDomainLookupFunction.class);
addMessageProcessorFunction(AbuseChRansomIpLookupFunction.NAME, AbuseChRansomIpLookupFunction.class);


// MineMeld Threat Feeds
addMessageProcessorFunction(MineMeldDomainLookupFunction.NAME, MineMeldDomainLookupFunction.class);
addMessageProcessorFunction(MineMeldIpLookupFunction.NAME, MineMeldIpLookupFunction.class);

// Global/combined lookup
addMessageProcessorFunction(GlobalIpLookupFunction.NAME, GlobalIpLookupFunction.class);
Expand All @@ -69,6 +77,7 @@ protected void configure() {
addMessageProcessorFunction(PrivateNetLookupFunction.NAME, PrivateNetLookupFunction.class);

installLookupDataAdapter(AbuseChRansomAdapter.NAME, AbuseChRansomAdapter.class, AbuseChRansomAdapter.Factory.class, AbuseChRansomAdapter.Config.class);
installLookupDataAdapter(MineMeldBlockListAdapter.NAME, MineMeldBlockListAdapter.class, MineMeldAdapter.Factory.class, MineMeldBlockListAdapter.Config.class);
installLookupDataAdapter(SpamhausEDROPDataAdapter.NAME, SpamhausEDROPDataAdapter.class, SpamhausEDROPDataAdapter.Factory.class, SpamhausEDROPDataAdapter.Config.class);
installLookupDataAdapter(TorExitNodeDataAdapter.NAME, TorExitNodeDataAdapter.class, TorExitNodeDataAdapter.Factory.class, TorExitNodeDataAdapter.Config.class);
installLookupDataAdapter(WhoisDataAdapter.NAME, WhoisDataAdapter.class, WhoisDataAdapter.Factory.class, WhoisDataAdapter.Config.class);
Expand All @@ -79,6 +88,8 @@ protected void configure() {

addDomainFunction("abusech_ransomware", AbuseChRansomDomainLookupFunction.class);
addIPFunction("abusech_ransomware", AbuseChRansomIpLookupFunction.class);
addDomainFunction("minemeld", MineMeldDomainLookupFunction.class);
addIPFunction("minemeld", MineMeldIpLookupFunction.class);
addIPFunction("spamhaus", SpamhausIpLookupFunction.class);
addIPFunction("tor", TorExitNodeLookupFunction.class);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.graylog.plugins.threatintel.functions.DomainFunctions;
import org.graylog.plugins.threatintel.functions.GenericLookupResult;
import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomDomainLookupFunction;
import org.graylog.plugins.threatintel.functions.minemeld.MineMeldDomainLookupFunction;
import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction;
import org.graylog2.plugin.cluster.ClusterConfigService;
import org.slf4j.Logger;
Expand Down Expand Up @@ -67,6 +68,15 @@ boolean isEnabled(LookupTableFunction<? extends GenericLookupResult> function) {
return true;
}

@Override
boolean isEnabled(LookupTableFunction<? extends GenericLookupResult> function) {
final ThreatIntelPluginConfiguration configuration = this.threatIntelPluginConfiguration();
if (function.getClass().equals(MineMeldDomainLookupFunction.class)) {
return configuration.minemeldEnabled();
}
return true;
}

@Override
public FunctionDescriptor<GlobalLookupResult> descriptor() {
return FunctionDescriptor.<GlobalLookupResult>builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.graylog.plugins.threatintel.functions.GenericLookupResult;
import org.graylog.plugins.threatintel.functions.IPFunctions;
import org.graylog.plugins.threatintel.functions.abusech.AbuseChRansomIpLookupFunction;
import org.graylog.plugins.threatintel.functions.minemeld.MineMeldIpLookupFunction;
import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction;
import org.graylog.plugins.threatintel.functions.otx.OTXIPLookupFunction;
import org.graylog.plugins.threatintel.functions.spamhaus.SpamhausIpLookupFunction;
Expand Down Expand Up @@ -73,6 +74,9 @@ boolean isEnabled(LookupTableFunction<? extends GenericLookupResult> function) {
if (function.getClass().equals(AbuseChRansomIpLookupFunction.class)) {
return configuration.abusechRansomEnabled();
}
if (function.getClass().equals(MineMeldIpLookupFunction.class)) {
return configuration.minemeldEnabled();
}
if (function.getClass().equals(OTXIPLookupFunction.class)) {
return configuration.otxEnabled();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package org.graylog.plugins.threatintel.functions.minemeld;

import org.graylog.plugins.pipelineprocessor.EvaluationContext;
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction;
import org.graylog.plugins.threatintel.functions.GenericLookupResult;
import org.graylog.plugins.threatintel.tools.Domain;
import org.graylog2.lookup.LookupTableService;
import org.graylog2.plugin.lookup.LookupResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;

public class MineMeldDomainLookupFunction extends LookupTableFunction<GenericLookupResult> {

private static final Logger LOG = LoggerFactory.getLogger(MineMeldDomainLookupFunction.class);

public static final String NAME = "minemeld_lookup_domain";
private static final String VALUE = "domain_name";
private static final String LOOKUP_TABLE_NAME = "minemeld-domains";

private final ParameterDescriptor<String, String> valueParam = ParameterDescriptor.string(VALUE).description("The domain to look up. Example: foo.example.org (A trailing dot ('.') will be ignored.)").build();

private final LookupTableService.Function lookupFunction;

@Inject
public MineMeldDomainLookupFunction(final LookupTableService lookupTableService) {
this.lookupFunction = lookupTableService.newBuilder().lookupTable(LOOKUP_TABLE_NAME).build();
}

@Override
public GenericLookupResult evaluate(FunctionArgs args, EvaluationContext context) {
String domain = valueParam.required(args, context);
if (domain == null) {
LOG.error("NULL parameter passed to abuse.ch Ransomware domain lookup.");
return null;
}

domain = Domain.prepareDomain(domain);

LOG.debug("Running abuse.ch Ransomware lookup for domain [{}].", domain);

final LookupResult lookupResult = this.lookupFunction.lookup(domain.trim());
if (lookupResult != null && !lookupResult.isEmpty() && lookupResult.singleValue() != null) {
if (lookupResult.singleValue() instanceof Boolean) {
return (Boolean)lookupResult.singleValue() ? GenericLookupResult.TRUE : GenericLookupResult.FALSE;
}
if (lookupResult.singleValue() instanceof String) {
return Boolean.valueOf((String) lookupResult.singleValue()) ? GenericLookupResult.TRUE : GenericLookupResult.FALSE;
}
}

return GenericLookupResult.FALSE;
}

@Override
public FunctionDescriptor<GenericLookupResult> descriptor() {
return FunctionDescriptor.<GenericLookupResult>builder()
.name(NAME)
.description("Match a domain name against the abuse.ch Ransomware Domain Blocklist. (RW_DOMBL)")
.params(valueParam)
.returnType(GenericLookupResult.class)
.build();
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
package org.graylog.plugins.threatintel.functions.minemeld;

import org.graylog.plugins.pipelineprocessor.EvaluationContext;
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionArgs;
import org.graylog.plugins.pipelineprocessor.ast.functions.FunctionDescriptor;
import org.graylog.plugins.pipelineprocessor.ast.functions.ParameterDescriptor;
import org.graylog.plugins.threatintel.functions.misc.LookupTableFunction;
import org.graylog.plugins.threatintel.functions.GenericLookupResult;
import org.graylog2.lookup.LookupTableService;
import org.graylog2.plugin.lookup.LookupResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;

public class MineMeldIpLookupFunction extends LookupTableFunction<GenericLookupResult> {

private static final Logger LOG = LoggerFactory.getLogger(MineMeldIpLookupFunction.class);

public static final String NAME = "minemeld_lookup_ip";
private static final String VALUE = "ip_address";
private static final String LOOKUP_TABLE_NAME = "minemeld-ip";

private final ParameterDescriptor<String, String> valueParam = ParameterDescriptor.string(VALUE).description("The IPv4 or IPv6 address to look up. Example: 198.51.100.1 or 2001:0db8:85a3:0000:0000:8a2e:0370:7334").build();

private final LookupTableService.Function lookupFunction;

@Inject
public MineMeldmIpLookupFunction(final LookupTableService lookupTableService) {
this.lookupFunction = lookupTableService.newBuilder().lookupTable(LOOKUP_TABLE_NAME).build();
}

@Override
public GenericLookupResult evaluate(FunctionArgs args, EvaluationContext context) {
String ip = valueParam.required(args, context);
if (ip == null) {
LOG.error("NULL parameter passed to abuse.ch Ransomware IP lookup.");
return null;
}

LOG.debug("Running abuse.ch Ransomware lookup for IP [{}].", ip);

final LookupResult lookupResult = this.lookupFunction.lookup(ip.trim());
if (lookupResult != null && !lookupResult.isEmpty() && lookupResult.singleValue() != null) {
if (lookupResult.singleValue() instanceof Boolean) {
return (Boolean)lookupResult.singleValue() ? GenericLookupResult.TRUE : GenericLookupResult.FALSE;
}
if (lookupResult.singleValue() instanceof String) {
return Boolean.valueOf((String) lookupResult.singleValue()) ? GenericLookupResult.TRUE : GenericLookupResult.FALSE;
}
}

return GenericLookupResult.FALSE;
}

@Override
public FunctionDescriptor<GenericLookupResult> descriptor() {
return FunctionDescriptor.<GenericLookupResult>builder()
.name(NAME)
.description("Match a IPv4 or IPv6 address against the abuse.ch Ransomware IP Blocklist. (RW_IPBL)")
.params(valueParam)
.returnType(GenericLookupResult.class)
.build();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,17 @@
"default_multi_value": "",
"default_multi_value_type": "NULL"
},
{
"title": "MineMeld Domains",
"description": "This is the lookup table for the MineMeld Domain list, listing infrastructure by domain names which are used for ransomware. For more information see https://ransomwaretracker.abuse.ch. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.",
"name": "minemeld-domains",
"cache_name": "threat-intel-uncached-adapters",
"data_adapter_name": "minemeld-domains",
"default_single_value": "",
"default_single_value_type": "NULL",
"default_multi_value": "",
"default_multi_value_type": "NULL"
},
{
"title": "Whois",
"description": "This is the lookup table for the WHOIS database, listing registered users of Internet resources like IPs, Netblocks or Domain Names. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.",
Expand All @@ -63,6 +74,17 @@
"default_multi_value": "",
"default_multi_value_type": "NULL"
},
{
"title": "MineMeld IP List",
"description": "This is the lookup table for the MineMeld IP List, listing infrastructure by IP which is used for nefarious reasons based on your threat feed configuration within MineMeld. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.",
"name": "minemeld-ip",
"cache_name": "threat-intel-uncached-adapters",
"data_adapter_name": "minemeld-ip",
"default_single_value": "",
"default_single_value_type": "NULL",
"default_multi_value": "",
"default_multi_value_type": "NULL"
},
{
"title": "Spamhaus DROP",
"description": "This is the lookup table for Spamhaus' DROP (Don't Route Or Peer) list, containing netblocks which are \"hijacked\" or leased by professional spam or cyber-crime operations. For more information see https://www.spamhaus.org/drop. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.",
Expand Down Expand Up @@ -189,6 +211,17 @@
"registry": "ARIN"
}
},
{
"title": "MineMeld Domains",
"description": "This is the lookup table for the MineMeld Domain list, listing infrastructure by domain names which are used for ransomware. For more information see https://ransomwaretracker.abuse.ch. This lookup table is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.",
"name": "minemeld-domains",
"config": {
"type": "minemeld",
"blocklist_type": "DOMAINS",
"refresh_interval": 150,
"refresh_interval_unit": "SECONDS"
}
},
{
"title": "abuse.ch ransomware Domains",
"description": "This is the data adapter for the abuse.ch ransomware Domain Tracker, listing infrastructure by domain names which are used for ransomware. For more information see https://ransomwaretracker.abuse.ch. This adapter is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.",
Expand Down Expand Up @@ -217,7 +250,19 @@
"blocklist_type": "IPS",
"refresh_interval": 150,
"refresh_interval_unit": "SECONDS"
}
},
{
"title": "MineMeld IP List",
"description": "This is the data adapter for the MineMeld IP List, listing infrastructure by IP which is used for nefarious reasons based on your threat feed configuration within MineMeld. This adapter is used internally by Graylog's Threat Intel Plugin. Do not delete it manually.",
"name": "minemeld-ip",
"config": {
"type": "minemeld",
"blocklist_type": "IPS",
"refresh_interval": 150,
"refresh_interval_unit": "SECONDS"
},


}
]
}
Loading

0 comments on commit 8019b62

Please sign in to comment.