Skip to content

Commit

Permalink
v1.6.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@qjerome
qjerome committed Aug 21, 2019
1 parent 3c42e7e commit bc80854
Show file tree
Hide file tree
Showing 21 changed files with 1,511 additions and 549 deletions.
71 changes: 44 additions & 27 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,14 +104,16 @@ which matched the event.

## WHIDS
1. Download and extract the latest WHIDS release https://github.com/0xrawsec/whids/releases
2. Run `install.bat` as **administrator**
3. Verify that files have been created at the **installation directory**
4. With a text editor **opened as administrator** (to prevent changing rights of WHIDs installation directory) open `config.json` and modify it as you wish
5. Skip this if running with a connection to a manager. If there is nothing in the **rules directory** the tool will be useless, so make sure there are some **gene** rules in there. You can get some compiled rules [here](https://raw.githubusercontent.com/0xrawsec/gene-rules/master/compiled.gen)
6. Start the HIDS with `Start.bat` script located in **installation directory**
7. If you configured a **manager** do not forget to run it
2. If you want WHIDS to run along with Sysmon (**strongly recommended**), install it first. An optimal **configuration file** is shipped with the release so that you can take the most out of WHIDS. At installation the **Sysmon service** will be made *dependant* of **WHIDS service** so that we are sure the IDS runs before **Sysmon** starts generating some events.
3. Run `manage.bat` as **administrator**
4. Launch installation by selecting the appropriate option
5. Verify that files have been created at the **installation directory**
6. With a text editor **opened as administrator** (to prevent changing the rights of the WHIDS installation directory) open `config.json` and modify it as you wish. This can also be done from `manage.bat`
7. Skip this if running with a connection to a manager, because rules will be updated automatically. If there is nothing in the **rules directory** the tool will be useless, so make sure there are some **gene** rules in there. Some rules are packaged with WHIDS and you will be prompted if you want to install those or not. If you want the last up to date rules, you can get those [here](https://raw.githubusercontent.com/0xrawsec/gene-rules/master/compiled.gen) (take the **compiled** ones)
8. Start the **services** from appropriate option in `manage.bat` or just reboot (**preferred option** otherwise some enrichment fields will be incomplete leading to false alerts)
9. If you configured a **manager** do not forget to run it in order to receive alerts and dumps

**NB:** whenever you go to the installation directory with **File Explorer** and if you are **Administrator** the explorer will ask you if you want to change the permission of the directory. **DO NOT CLICK YES**, otherwise it will break the folder permissions put in place at installation time. Always access installation directory from **applications started as Administrator**.
**NB:** whenever you go to the installation directory with **Explorer.exe** and if you are **Administrator** the explorer will ask you if you want to change the permission of the directory. **DO NOT CLICK YES**, otherwise it will break the folder permissions put in place at installation time. Always access installation directory from **applications started as Administrator** (i.e. text editor).

# Configuration

Expand All @@ -130,7 +132,7 @@ WHIDS configuration file example
"containers-db": "C:\\Program Files\\Whids\\Database\\Containers",
// Forwarder related configuration
"forwarder": {
"client": {
"manager-client": {
// Hostname or IP address of remote manager
"host": "",
// Port used by remote manager
Expand All @@ -146,8 +148,13 @@ WHIDS configuration file example
// Maximum upload side for dump forwarding
"max-upload-size": 104857600
},
// Path where to store the alerts
"logs-dir": "C:\\Program Files\\Whids\\Logs\\Alerts",
// Alert logging settings
"logging": {
// Path where to store the alerts
"dir": "C:\\Program Files\\Whids\\Logs\\Alerts",
// Rotation interval
"rotation-interval": "24h"
},
// If local=true the forwarder won't communicate with manager
"local": true
},
Expand All @@ -157,26 +164,33 @@ WHIDS configuration file example
"channels": [
"all"
],
// Dump mode: file, memory or all (can be empty)
// file: dumps anything identified as a file in the event
// memory: dumps (guilty) process memory in Windows minidump format
"dump-mode": "file",
// Dumps when criticality of the events is above or equal to treshold
"dump-treshold": 8,
// Where to store dumps
"dump-dir": "C:\\Program Files\\Whids\\Dumps",
// Whether or not to enable dump compression
"dump-compression": true,
// Dump related settings
"dump": {
// Dump mode: file, memory or all (can be empty)
// file: dumps anything identified as a file in the event
// memory: dumps (guilty) process memory in Windows minidump format
// registry: dumps registry in case alert concerns a registry
"mode": "file|registry",
// Dumps when criticality of the events is above or equal to treshold
"treshold": 8,
// Where to store dumps
"dir": "C:\\Program Files\\Whids\\Dumps",
// Whether or not to enable dump compression
"compression": true
},
// Log events with criticality above or equal to treshold
"criticality-treshold": 5,
// Sleep time in seconds between two rules updates (negative number means no update)
"update-interval": 60,
// Whether on not hooks should be enables
// Whether on not hooks should be enabled
"en-hooks": true,
// Whether or not DNS-Client logging should be enabled
"en-dns": true,
// Logfile used to store WHIDS stderr
"logfile": "C:\\Program Files\\Whids\\Logs\\whids.log"
"logfile": "C:\\Program Files\\Whids\\Logs\\whids.log",
// Log all the events passing through the engine (usefull for event dumping or debugging)
"log-all": false,
// Tells WHIDS that it is running on an endpoint. If false any kind
// of dump is disabled to avoid dump things if installed on a WEC
"endpoint": true
}
```

Expand All @@ -191,8 +205,7 @@ Manager configuration example
// Port used by the manager
"port": 1519,
// Logfile (automatically rotated) where to store alerts received
// the logs are GZIP compressed
"logfile": "alerts.gz",
"logfile": "alerts.log",
// Server authentication key (see server-key in WHIDS config)
"key": "someserverkey",
// List of authorized client keys (see key in WHIDS config)
Expand Down Expand Up @@ -222,7 +235,7 @@ Manager configuration example

# Documentation

To know how to write rules for the engine please visit: https://rawsec.lu/doc/gene/1.4/
To know how to write rules for the engine please visit: https://rawsec.lu/doc/gene/1.6/

# Known Issues

Expand All @@ -237,6 +250,10 @@ In order to get the most of WHIDS you need to activate specific features

# Changelog

## v1.6

TBD

## v1.5
* Bunch of code rewritten to make things more consistent:
* WHIDS is no longer command line based, most of the options are configured via a configuration file
Expand Down
88 changes: 53 additions & 35 deletions collector/forwarder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ type LoggingConfig struct {
// from configuration structure.
func (l *LoggingConfig) ParseRotationInterval() (d time.Duration, err error) {
d, err = time.ParseDuration(l.RotationInterval)
if d < MinRotationInterval {
/*if d < MinRotationInterval {
d = MinRotationInterval
}
}*/
return
}

Expand Down Expand Up @@ -168,13 +168,13 @@ func (f *Forwarder) Save() error {
if f.logfile == nil {
// This will reopen the first available alerts.gz.X file if several
//lf := filepath.Join(f.fwdConfig.LogConf.Dir, "alerts.gz")
lf := filepath.Join(f.fwdConfig.Logging.Dir, "alerts")
lf := filepath.Join(f.fwdConfig.Logging.Dir, "alerts.log")
ri, err := f.fwdConfig.Logging.ParseRotationInterval()
if err != nil {
return err
}
log.Infof("Rotating logfile every %s", ri)
if f.logfile, err = logfile.OpenTimeRotateLogFile(lf, DefaultLogPerm, ri, time.Second*5); err != nil {
if f.logfile, err = logfile.OpenTimeRotateLogFile(lf, DefaultLogPerm, ri); err != nil {
return err
}
}
Expand Down Expand Up @@ -233,6 +233,20 @@ func (f *Forwarder) DiskSpaceQueue() int64 {
return dp
}

// Here we rely on the fact that the layout of the directory is known
// and should be alert.log, alert.log.1, alert.log.2.gz, alert.log.3.gz ...
// if we append in reverse order, older files appears first in the list
func (f *Forwarder) listLogfiles() (files []string) {
files = make([]string, 0)
for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
for _, fi := range wi.Files {
fp := filepath.Join(f.fwdConfig.Logging.Dir, fi.Name())
files = append([]string{fp}, files...)
}
}
return
}

// ProcessQueue processes the events queued
// Todo: needs update with client
func (f *Forwarder) ProcessQueue() {
Expand All @@ -258,43 +272,47 @@ func (f *Forwarder) ProcessQueue() {

// Reset logfile for latter Save function use
f.logfile = nil
for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
for _, fi := range wi.Files {
// fullpath
fp := filepath.Join(f.fwdConfig.Logging.Dir, fi.Name())
log.Debug("Processing queued file: %s", fp)
fd, err := os.Open(fp)
//for wi := range fswalker.Walk(f.fwdConfig.Logging.Dir) {
//for _, fi := range wi.Files {
for _, fp := range f.listLogfiles() {
// fullpath
//fp := filepath.Join(f.fwdConfig.Logging.Dir, fi.Name())
//log.Debug("Processing queued file: %s", fp)
log.Infof("Processing queued file: %s", fp)
fd, err := os.Open(fp)
if err != nil {
log.Errorf("Failed to open queued file (%s): %s", fp, err)
continue
}
switch {
case strings.HasSuffix(fp, ".gz"):
// the file is gzip so we have to pass a gzip reader to prepCollectReq
gzr, err := gzip.NewReader(fd)
if err != nil {
log.Errorf("Failed to open queued file (%s): %s", fp, err)
continue
}

if strings.HasSuffix(fp, ".gz") {
// the file is gzip so we have to pass a gzip reader to prepCollectReq
gzr, err := gzip.NewReader(fd)
if err != nil {
log.Errorf("Failed to create gzip reader for queued file (%s): %s", fp, err)
// close file
fd.Close()
continue
}
err = f.Client.PostLogs(gzr)
// we can close the reader since we don't need those anymore
gzr.Close()
log.Errorf("Failed to create gzip reader for queued file (%s): %s", fp, err)
// close file
fd.Close()
}

// We do not remove the logs if we failed to send
if err != nil {
log.Errorf("%s", err)
continue
}
err = f.Client.PostLogs(gzr)
// we can close the reader since we don't need those anymore
gzr.Close()
fd.Close()
case strings.HasSuffix(fp, ".log.1"), strings.HasSuffix(fp, ".log"):
err = f.Client.PostLogs(fd)
}

// everything went fine, then we can delete the queued file
if err = os.Remove(fp); err != nil {
log.Errorf("Failed to delete queued file (%s): %s", fp, err)
}
// We do not remove the logs if we failed to send
if err != nil {
log.Errorf("%s", err)
continue
}

// everything went fine, then we can delete the queued file
if err = os.Remove(fp); err != nil {
log.Errorf("Failed to delete queued file (%s): %s", fp, err)
}
//}
}
}

Expand Down
13 changes: 12 additions & 1 deletion collector/manager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ type Manager struct {
tls TLSConfig
srv *http.Server
stop chan bool
done bool
// Gene related members
geneEng *engine.Engine
rules string // to cache the rules concatenated
Expand All @@ -197,7 +198,9 @@ func NewManager(c *ManagerConfig) (*Manager, error) {
}

m := Manager{Host: c.Host, Port: fmt.Sprintf("%d", c.Port)}
m.logfile, err = logfile.OpenTimeRotateLogFile(c.Logfile, DefaultLogPerm, time.Hour, time.Second*5)
if m.logfile, err = logfile.OpenTimeRotateLogFile(c.Logfile, DefaultLogPerm, time.Hour); err != nil {
return &m, err
}
m.key = c.Key
m.authorized = datastructs.NewInitSyncedSet(datastructs.ToInterfaceSlice(c.Authorized)...)
m.stop = make(chan bool)
Expand Down Expand Up @@ -293,9 +296,17 @@ func (m *Manager) Wait() {
<-m.stop
}

func (m *Manager) IsDone() bool {
return m.done
}

// Shutdown the Manager
func (m *Manager) Shutdown() error {
defer func() { go func() { m.stop <- true }() }()
if m.done {
return nil
}
m.done = true
if m.srv != nil {
m.srv.Shutdown(nil)
}
Expand Down
5 changes: 2 additions & 3 deletions collector/test/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ func TestClientGetRules(t *testing.T) {
}
r.AddAuthKey(key)
r.Run()
defer r.Shutdown()

cconf.Key = key
c, err := collector.NewManagerClient(&cconf)
Expand All @@ -45,12 +44,12 @@ func TestClientGetRules(t *testing.T) {
if err != nil {
t.Errorf("%s", err)
}

if sha256 != data.Sha256([]byte(rules)) {
t.Errorf("Rules integrity cannot be verified")
}

t.Log(rules)
r.Shutdown()

}

func TestClientPostDump(t *testing.T) {
Expand Down
Loading

0 comments on commit bc80854

Please sign in to comment.