80 lines
1.6 KiB
Go
80 lines
1.6 KiB
Go
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"io/ioutil"
|
||
|
|
"log"
|
||
|
|
"net/http"
|
||
|
|
"regexp"
|
||
|
|
|
||
|
|
"github.com/miekg/dns"
|
||
|
|
)
|
||
|
|
|
||
|
|
func loadBlacklist(config []configBlacklist) map[string]bool {
|
||
|
|
list := make([]string, 0)
|
||
|
|
for _, element := range config {
|
||
|
|
raw, err := requestBacklist(element)
|
||
|
|
|
||
|
|
if err != nil {
|
||
|
|
log.Printf("Failed to load blacklist %s reason: %s", element.URL, err.Error())
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
|
||
|
|
domains := parseRawBlacklist(element, *raw)
|
||
|
|
list = append(list, domains...)
|
||
|
|
}
|
||
|
|
|
||
|
|
// list = removeDuplicates(list)
|
||
|
|
// sort.Strings(list)
|
||
|
|
|
||
|
|
domainMap := make(map[string]bool)
|
||
|
|
for _, e := range list {
|
||
|
|
domainMap[e] = true
|
||
|
|
}
|
||
|
|
|
||
|
|
return domainMap
|
||
|
|
}
|
||
|
|
|
||
|
|
func removeDuplicates(elements []string) []string {
|
||
|
|
encountered := map[string]bool{}
|
||
|
|
result := []string{}
|
||
|
|
|
||
|
|
for v := range elements {
|
||
|
|
if !encountered[elements[v]] {
|
||
|
|
encountered[elements[v]] = true
|
||
|
|
|
||
|
|
result = append(result, elements[v])
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return result
|
||
|
|
}
|
||
|
|
|
||
|
|
func requestBacklist(blacklist configBlacklist) (*string, error) {
|
||
|
|
// Request list
|
||
|
|
resp, err := http.Get(blacklist.URL)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
defer resp.Body.Close()
|
||
|
|
body, err := ioutil.ReadAll(resp.Body)
|
||
|
|
bodyString := string(body)
|
||
|
|
|
||
|
|
log.Printf("Downloaded blacklist %s", blacklist.URL)
|
||
|
|
|
||
|
|
return &bodyString, err
|
||
|
|
}
|
||
|
|
|
||
|
|
func parseRawBlacklist(blacklist configBlacklist, raw string) []string {
|
||
|
|
finalList := make([]string, 0)
|
||
|
|
reg := regexp.MustCompile(`(?mi)^\s*(#*)\s*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+([a-zA-Z0-9\.\- ]+)$`)
|
||
|
|
matches := reg.FindAllStringSubmatch(raw, -1)
|
||
|
|
for _, match := range matches {
|
||
|
|
if match[1] != "#" {
|
||
|
|
finalList = append(finalList, dns.Fqdn(match[3]))
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return finalList
|
||
|
|
}
|