improved blacklist parsing

blacklist.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"io/ioutil"
 	"log"
 	"net"
@@ -26,12 +27,10 @@ func loadBlacklist(config []configBlacklist) map[string]bool {
 		}
 
 		domains := parseRawBlacklist(element, *raw)
+		log.Printf("Added %d blocked domains", len(domains))
 		list = append(list, domains...)
 	}
 
-	// list = removeDuplicates(list)
-	// sort.Strings(list)
-
 	domainMap := make(map[string]bool)
 	for _, e := range list {
 		domainMap[e] = true
@@ -56,22 +55,46 @@ func removeDuplicates(elements []string) []string {
 }
 
 func requestBacklist(blacklist configBlacklist) (*string, error) {
+	if blacklist.URL != "" {
+		return getBlacklistFromURL(blacklist.URL)
+	}
+
+	return nil, errors.New("No blacklist provided")
+}
+
+func getBlacklistFromURL(url string) (*string, error) {
 	// Request list
-	resp, err := http.Get(blacklist.URL)
+	resp, err := http.Get(url)
 	if err != nil {
 		return nil, err
 	}
 
 	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		log.Printf("Got %d status code. Continueing anyway.", resp.StatusCode)
+	}
+
 	body, err := ioutil.ReadAll(resp.Body)
 	bodyString := string(body)
 
-	log.Printf("Downloaded blacklist %s", blacklist.URL)
+	log.Printf("Downloaded blacklist %s", url)
 
 	return &bodyString, err
 }
 
 func parseRawBlacklist(blacklist configBlacklist, raw string) []string {
+	switch blacklist.Format {
+	case "host":
+		return parseHostFormat(raw)
+	default:
+		log.Printf("Failed to parse blacklist. Format not supported: %s", blacklist.Format)
+		log.Println("Supported types are: host")
+		return make([]string, 0)
+	}
+}
+
+func parseHostFormat(raw string) []string {
 	finalList := make([]string, 0)
 	reg := regexp.MustCompile(`(?mi)^\s*(#*)\s*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+([a-zA-Z0-9\.\- ]+)$`)
 	matches := reg.FindAllStringSubmatch(raw, -1)