Fixed #85: Add API endpoint to manage IOCs spread on endpoints for detection

Author: qjerome

.github/coverage/coverage.txt

@@ -39,7 +39,7 @@
 * Built by an Incident Responder for all Incident Responders to make their job easier  
 * Low footprint (no process injection)
 * Can co-exist with **any antivirus** product (advised to run it along with **MS Defender**)
-* Designed for high thoughput. It can easily enrich and analyse 4M events a day per endpoint without performance impact. Good luck to achieve that with a SIEM.
+* Designed for high throughput. It can easily enrich and analyze 4M events a day per endpoint without performance impact. Good luck to achieve that with a SIEM.
 * Easily integrable with other tools (Splunk, ELK, MISP ...)
 * Integrated with [ATT&CK framework](https://attack.mitre.org/)
 
@@ -88,12 +88,12 @@ This section covers the installation of the agent on the endpoint.
 2. Run `manage.bat` as **administrator**
 3. Launch installation by selecting the appropriate option
 4. Verify that files have been created at the **installation directory**
-5. Edit configuration file by selecting the appropriate option in `manage.bat` or using your prefered text editor
+5. Edit configuration file by selecting the appropriate option in `manage.bat` or using your preferred text editor
 6. Skip this if running with a connection to a manager, because rules will be updated automatically. If there is nothing in the **rules directory** the tool will be useless, so make sure there are some **gene** rules in there. Some rules are packaged with WHIDS and you will be prompted to choose if you want to install those or not. If you want the last up to date rules, you can get those [here](https://raw.githubusercontent.com/0xrawsec/gene-rules/master/compiled.gen) (take the **compiled** ones)
 7. Start the **services** from appropriate option in `manage.bat` or just reboot (**preferred option** otherwise some enrichment fields will be incomplete leading to false alerts)
 8. If you configured a **manager** do not forget to run it in order to receive alerts and dumps
 
-**NB:** At installation time the **Sysmon service** will be made *dependant* of **WHIDS service** so that we are sure the EDR runs before **Sysmon** starts generating some events.
+**NB:** At installation time the **Sysmon service** will be made *dependent* of **WHIDS service** so that we are sure the EDR runs before **Sysmon** starts generating some events.
 
 ## EDR Manager
 
@@ -105,7 +105,7 @@ The EDR manager can be installed on several platforms, pre-built binaries are pr
 
 # Configuration Examples
 
-Please visit [doc/configuration.md](https://github.com/0xrawsec/whids/blob/master/doc/configuration.md)
+Please visit [doc/configuration.md](doc/configuration.md)
 
 # Further Documentation
 
@@ -141,7 +141,7 @@ Please visit [doc/configuration.md](https://github.com/0xrawsec/whids/blob/maste
         * Put the content of the clipboard data inside the event to allow creating rule on the content of the clipboard
     - Integrate ProcessTampering events
         * Enrich event with a diffing score between .text section on disk and in memory
-- Implemented certificate pinning on client to enhance security of the communiaction channel between endpoints and management server
+- Implemented certificate pinning on client to enhance security of the communication channel between endpoints and management server
 - Log filtering capabilities, allowing one to collect contextual events. Log filtering is achieved by creating Gene filtering rules (c.f. [Gene Documentation](https://github.com/0xrawsec/gene)).
 - Configuration files in TOML format for better readability
 - Better protection of the installation directory

README.md

@@ -26,15 +26,40 @@ var (
 	}
 )
 
-func doRequest(method, url string) (r AdminAPIResponse) {
-	cl := http.Client{Transport: cconf.Transport()}
+func prepare(method, URL string, data []byte, params map[string]string) *http.Request {
 	key := testAdminUser.Key
-	uri := fmt.Sprintf("https://%s:%d%s", mconf.AdminAPI.Host, mconf.AdminAPI.Port, url)
-	req, err := http.NewRequest(method, uri, new(bytes.Buffer))
+	buf := new(bytes.Buffer)
+
+	// preparing request body
+	if data != nil {
+		if len(data) > 0 {
+			buf.Write(data)
+		}
+	}
+
+	// preparing parameters to be passed to the query
+	if params != nil {
+		v := url.Values{}
+		for param, value := range params {
+			v.Set(param, value)
+		}
+
+		if len(params) > 0 {
+			URL = fmt.Sprintf("%s?%s", URL, v.Encode())
+		}
+	}
+
+	uri := fmt.Sprintf("https://%s:%d%s", mconf.AdminAPI.Host, mconf.AdminAPI.Port, URL)
+	req, err := http.NewRequest(method, uri, buf)
 	if err != nil {
 		panic(err)
 	}
 	req.Header.Add(AuthKeyHeader, key)
+	return req
+}
+
+func do(req *http.Request) (r AdminAPIResponse) {
+	cl := http.Client{Transport: cconf.Transport()}
 	resp, err := cl.Do(req)
 	if err != nil {
 		panic(err)
@@ -54,11 +79,11 @@ func doRequest(method, url string) (r AdminAPIResponse) {
 }
 
 func get(url string) (r AdminAPIResponse) {
-	return doRequest("GET", url)
+	return do(prepare("GET", url, nil, nil))
 }
 
 func put(url string) (r AdminAPIResponse) {
-	return doRequest("PUT", url)
+	return do(prepare("PUT", url, nil, nil))
 }
 
 func post(url string, data []byte) (r AdminAPIResponse) {
@@ -82,8 +107,10 @@ func post(url string, data []byte) (r AdminAPIResponse) {
 	if err != nil {
 		panic(err)
 	}
-	if err := json.Unmarshal(b, &r); err != nil {
-		panic(err)
+	if len(b) > 0 {
+		if err := json.Unmarshal(b, &r); err != nil {
+			panic(err)
+		}
 	}
 	return
 }
@@ -323,7 +350,7 @@ func TestAdminAPIGetEndpointReport(t *testing.T) {
 	failOnAdminAPIError(t, r)
 	t.Logf("received: %s", prettyJSON(r))
 
-	r = doRequest("DELETE", AdmAPIEndpointsPath+"/"+euuid+"/report")
+	r = do(prepare("DELETE", AdmAPIEndpointsPath+"/"+euuid+"/report", nil, nil))
 	failOnAdminAPIError(t, r)
 	t.Logf("received: %s", prettyJSON(r))
 

api/adminapi_test.go

@@ -305,12 +305,12 @@ func (m *ManagerClient) GetRulesSha256() (string, error) {
 	return "", nil
 }
 
-// GetContainer retrieves a given container from the manager
-func (m *ManagerClient) GetContainer(name string) ([]string, error) {
+// GetIoCs get IoCs from manager
+func (m *ManagerClient) GetIoCs() ([]string, error) {
 	ctn := make([]string, 0)
 
 	if auth, _ := m.IsServerAuthenticated(); auth {
-		req, err := m.Prepare("GET", strings.Replace(EptAPIContainerPath, "{name}", name, 1), nil)
+		req, err := m.Prepare("GET", EptAPIIoCsPath, nil)
 		if err != nil {
 			return ctn, fmt.Errorf("GetContainer failed to prepare request: %s", err)
 		}
@@ -334,40 +334,11 @@ func (m *ManagerClient) GetContainer(name string) ([]string, error) {
 	return ctn, nil
 }
 
-// GetContainersList retrieves the names of the containers available in the manager
-func (m *ManagerClient) GetContainersList() ([]string, error) {
-	ctn := make([]string, 0)
-
-	if auth, _ := m.IsServerAuthenticated(); auth {
-		req, err := m.Prepare("GET", EptAPIContainerListPath, nil)
-		if err != nil {
-			return ctn, fmt.Errorf("GetContainersList failed to prepare request: %s", err)
-		}
-
-		resp, err := m.HTTPClient.Do(req)
-		if err != nil {
-			return ctn, fmt.Errorf("GetContainersList failed to issue HTTP request: %s", err)
-		}
-
-		if resp != nil {
-			defer resp.Body.Close()
-			if resp.StatusCode != 200 {
-				return ctn, fmt.Errorf("failed to retrieve containers list, unexpected HTTP status code %d", resp.StatusCode)
-			}
-			dec := json.NewDecoder(resp.Body)
-			if err = dec.Decode(&ctn); err != nil {
-				return ctn, fmt.Errorf("GetContainersList failed to decode container list")
-			}
-		}
-	}
-	return ctn, nil
-}
-
-// GetContainerSha256 retrieves a given container from the manager
-func (m *ManagerClient) GetContainerSha256(name string) (string, error) {
+// GetIoCsSha256 retrieves a sha256 from the IoCs available in the manager
+func (m *ManagerClient) GetIoCsSha256() (string, error) {
 
 	if auth, _ := m.IsServerAuthenticated(); auth {
-		req, err := m.Prepare("GET", strings.Replace(EptAPIContainerSha256Path, "{name}", name, 1), nil)
+		req, err := m.Prepare("GET", EptAPIIoCsSha256Path, nil)
 		if err != nil {
 			return "", fmt.Errorf("GetContainerSha256 failed to prepare request: %s", err)
 		}

api/api_client.go

@@ -1,20 +1,22 @@
 package api
 
 import (
+	"fmt"
+	"math/rand"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/0xrawsec/golang-utils/crypto/data"
 	"github.com/0xrawsec/golang-utils/crypto/file"
-	"github.com/0xrawsec/golang-utils/datastructs"
 	"github.com/0xrawsec/golang-utils/fsutil/fswalker"
+	"github.com/0xrawsec/whids/ioc"
 	"github.com/0xrawsec/whids/utils"
 )
 
 var (
-cconf = ClientConfig{
+	cconf = ClientConfig{
 		Proto:             "https",
 		Host:              "localhost",
 		Port:              8000,
@@ -100,47 +102,65 @@ func TestClientContainer(t *testing.T) {
 
 	key := KeyGen(DefaultKeySize)
 
-	r, err := NewManager(&mconf)
+	m, err := NewManager(&mconf)
 	if err != nil {
 		panic(err)
 	}
-	r.AddEndpoint(cconf.UUID, key)
-	r.Run()
-	defer r.Shutdown()
+	m.AddEndpoint(cconf.UUID, key)
+	m.Run()
+	defer m.Shutdown()
 
 	cconf.Key = key
 	c, err := NewManagerClient(&cconf)
 	if err != nil {
 		panic(err)
 	}
 
-	containers, err := c.GetContainersList()
-	if err != nil {
-		t.Error(err)
-		t.FailNow()
-	}
+	niocs := 1000
+	iocs := make([]ioc.IoC, 0, niocs)
+	del := 0
+	for i := 0; i < niocs; i++ {
+		key := "test"
+		if rand.Int()%3 == 0 {
+			key = "to_delete"
+			del++
+		}
 
-	verif := datastructs.NewInitSyncedSet(datastructs.ToInterfaceSlice(containers)...)
-	if !verif.Contains("blacklist") || !verif.Contains("whitelist") {
-		t.Error("Missing containers")
-		t.FailNow()
+		iocs = append(iocs, ioc.IoC{
+			Value: fmt.Sprintf("%d.random.com", i),
+			Key:   key})
 	}
+	post(AdmAPIIocsPath, JSON(iocs))
 
-	for _, cont := range containers {
-		bl, err := c.GetContainer(cont)
-		if err != nil {
-			t.Error(err)
+	if iocs, err := c.GetIoCs(); err != nil {
+		t.Error(err)
+	} else {
+		if len(iocs) != niocs {
+			t.Error("Unexpected IOC length")
+		}
+		if rsha256, _ := c.GetIoCsSha256(); rsha256 != utils.Sha256StringArray(m.iocs.StringSlice()) {
+			t.Error("IOC container hash is not correct")
 		}
+	}
 
-		if sha256, err := c.GetContainerSha256(cont); sha256 != utils.Sha256StringArray(bl) || err != nil {
-			if err != nil {
-				t.Error(err)
-			} else {
-				t.Errorf("Failed to verify container integrity")
-			}
+	// deleting iocs from admin API
+	r := prepare("DELETE",
+		AdmAPIIocsPath,
+		nil,
+		map[string]string{"key": "to_delete"})
+	do(r)
+
+	if iocs, err := c.GetIoCs(); err != nil {
+		t.Error(err)
+	} else {
+		if len(iocs) != niocs-del {
+			t.Errorf("Unexpected IOC length expected %d, got %d", niocs-del, len(iocs))
+		}
+		if rsha256, _ := c.GetIoCsSha256(); rsha256 != utils.Sha256StringArray(m.iocs.StringSlice()) {
+			t.Error("IOC container hash is not correct")
 		}
-		t.Logf("%v", bl)
 	}
+
 }
 
 func TestClientExecuteCommand(t *testing.T) {

api/api_client_test.go

@@ -46,10 +46,9 @@ var (
 			Root:        "./data/logs",
 			LogBasename: "alerts",
 		},
-		Database:      "./data/database",
-		RulesDir:      "./data",
-		DumpDir:       "./data/uploads/",
-		ContainersDir: "./data/containers",
+		Database: "./data/database",
+		RulesDir: "./data",
+		DumpDir:  "./data/uploads/",
 		TLS: TLSConfig{
 			Cert: "./data/cert.pem",
 			Key:  "./data/key.pem",