From 0d1902c7fcf0d451943a80fd8a3eeb108e036b10 Mon Sep 17 00:00:00 2001 From: Steven Lee Date: Fri, 14 Jun 2024 14:36:00 +0100 Subject: [PATCH] RF: block lists to check and test resolvers - Move block list logic to checks - Create IP resolver interfaces - Inject resolvers - Add code cov file for code that just calls stdlib --- checks/block_lists.go | 128 +++++++++++++++++++++++++++++++++ checks/block_lists_test.go | 24 +++++++ checks/checks.go | 3 + checks/clients/ip/ip.go | 54 ++++++++++++++ codecov.yml | 2 + handlers/block_lists.go | 134 ++--------------------------------- handlers/block_lists_test.go | 24 +------ handlers/legacy_rank_test.go | 2 +- handlers/rank_test.go | 2 +- server/server.go | 2 +- 10 files changed, 220 insertions(+), 155 deletions(-) create mode 100644 checks/block_lists.go create mode 100644 checks/block_lists_test.go create mode 100644 checks/clients/ip/ip.go create mode 100644 codecov.yml diff --git a/checks/block_lists.go b/checks/block_lists.go new file mode 100644 index 0000000..865f23b --- /dev/null +++ b/checks/block_lists.go @@ -0,0 +1,128 @@ +package checks + +import ( + "context" + "net" + "slices" + "sort" + "sync" + "time" + + "github.com/xray-web/web-check-api/checks/clients/ip" +) + +type dnsServer struct { + Name string + IP string +} + +var DNS_SERVERS = []dnsServer{ + {Name: "AdGuard", IP: "176.103.130.130"}, + {Name: "AdGuard Family", IP: "176.103.130.132"}, + {Name: "CleanBrowsing Adult", IP: "185.228.168.10"}, + {Name: "CleanBrowsing Family", IP: "185.228.168.168"}, + {Name: "CleanBrowsing Security", IP: "185.228.168.9"}, + {Name: "CloudFlare", IP: "1.1.1.1"}, + {Name: "CloudFlare Family", IP: "1.1.1.3"}, + {Name: "Comodo Secure", IP: "8.26.56.26"}, + {Name: "Google DNS", IP: "8.8.8.8"}, + {Name: "Neustar Family", IP: "156.154.70.3"}, + {Name: "Neustar Protection", IP: "156.154.70.2"}, + {Name: "Norton Family", IP: "199.85.126.20"}, + {Name: "OpenDNS", IP: "208.67.222.222"}, + {Name: "OpenDNS Family", IP: "208.67.222.123"}, + {Name: "Quad9", IP: "9.9.9.9"}, + {Name: "Yandex Family", IP: "77.88.8.7"}, + {Name: "Yandex Safe", IP: "77.88.8.88"}, +} + +var knownBlockIPs = []string{ + "146.112.61.106", + "185.228.168.10", + "8.26.56.26", + "9.9.9.9", + "208.69.38.170", + "208.69.39.170", + "208.67.222.222", + "208.67.222.123", + "199.85.126.10", + "199.85.126.20", + "156.154.70.22", + "77.88.8.7", + "77.88.8.8", + "::1", + "2a02:6b8::feed:0ff", + "2a02:6b8::feed:bad", + "2a02:6b8::feed:a11", + "2620:119:35::35", + "2620:119:53::53", + "2606:4700:4700::1111", + "2606:4700:4700::1001", + "2001:4860:4860::8888", + "2a0d:2a00:1::", + "2a0d:2a00:2::", +} + +type Blocklist struct { + Server string `json:"server"` + ServerIP string `json:"serverIp"` + IsBlocked bool `json:"isBlocked"` +} + +type BlockList struct { + lookup ip.DNSLookup +} + +func NewBlockList(lookup ip.DNSLookup) *BlockList { + return &BlockList{lookup: lookup} +} + +func (b *BlockList) domainBlocked(ctx context.Context, domain, serverIP string) bool { + ips, err := b.lookup.DNSLookupIP(ctx, "ip4", domain, serverIP) + if err != nil { + // if there's an error, consider it not blocked + // TODO: return more detailed errors for each server + return false + } + + return slices.ContainsFunc(ips, func(ip net.IP) bool { + return slices.Contains(knownBlockIPs, ip.String()) + }) +} + +func (b *BlockList) BlockedServers(ctx context.Context, domain string) []Blocklist { + var lock sync.Mutex + var wg sync.WaitGroup + limit := make(chan struct{}, 5) + + var results []Blocklist + + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + for _, server := range DNS_SERVERS { + wg.Add(1) + go func(server dnsServer) { + limit <- struct{}{} + defer func() { + <-limit + wg.Done() + }() + + isBlocked := b.domainBlocked(ctx, domain, server.IP) + lock.Lock() + defer lock.Unlock() + results = append(results, Blocklist{ + Server: server.Name, + ServerIP: server.IP, + IsBlocked: isBlocked, + }) + }(server) + } + wg.Wait() + + sort.Slice(results, func(i, j int) bool { + return results[i].Server < results[j].Server + }) + return results +} diff --git a/checks/block_lists_test.go b/checks/block_lists_test.go new file mode 100644 index 0000000..293a1ff --- /dev/null +++ b/checks/block_lists_test.go @@ -0,0 +1,24 @@ +package checks + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/xray-web/web-check-api/checks/clients/ip" +) + +func TestBlockList(t *testing.T) { + t.Parallel() + + t.Run("blocked IP", func(t *testing.T) { + t.Parallel() + + dnsLookup := ip.DNSLookupFunc(func(ctx context.Context, network, host, dns string) ([]net.IP, error) { + return []net.IP{net.ParseIP("146.112.61.106")}, nil + }) + list := NewBlockList(dnsLookup).BlockedServers(context.Background(), "example.com") + assert.Contains(t, list, Blocklist{Server: "AdGuard", ServerIP: "176.103.130.130", IsBlocked: true}) + }) +} diff --git a/checks/checks.go b/checks/checks.go index 6c92238..8d8af27 100644 --- a/checks/checks.go +++ b/checks/checks.go @@ -4,10 +4,12 @@ import ( "net/http" "time" + "github.com/xray-web/web-check-api/checks/clients/ip" "github.com/xray-web/web-check-api/checks/store/legacyrank" ) type Checks struct { + BlockList *BlockList Carbon *Carbon IpAddress *Ip LegacyRank *LegacyRank @@ -21,6 +23,7 @@ func NewChecks() *Checks { Timeout: 5 * time.Second, } return &Checks{ + BlockList: NewBlockList(&ip.NetDNSLookup{}), Carbon: NewCarbon(client), IpAddress: NewIp(NewNetIp()), LegacyRank: NewLegacyRank(legacyrank.NewInMemoryStore()), diff --git a/checks/clients/ip/ip.go b/checks/clients/ip/ip.go new file mode 100644 index 0000000..9591140 --- /dev/null +++ b/checks/clients/ip/ip.go @@ -0,0 +1,54 @@ +package ip + +import ( + "context" + "fmt" + "net" + "time" +) + +type Lookup interface { + LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) +} + +type LookupFunc func(ctx context.Context, network string, host string) ([]net.IP, error) + +func (fn LookupFunc) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) { + return fn(ctx, network, host) +} + +// NetLookup is a client for looking up IP addresses using a net.Resolver. +type NetLookup struct{} + +func (l *NetLookup) LookupIP(ctx context.Context, network string, host string) ([]net.IP, error) { + netResolver := &net.Resolver{ + PreferGo: true, + } + return netResolver.LookupIP(ctx, network, host) +} + +type DNSLookup interface { + DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) +} + +type DNSLookupFunc func(ctx context.Context, network, host, dns string) ([]net.IP, error) + +func (fn DNSLookupFunc) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) { + return fn(ctx, network, host, dns) +} + +// DNSLookup is a client for looking up IP addresses with a custom DNS server. +type NetDNSLookup struct{} + +func (l *NetDNSLookup) DNSLookupIP(ctx context.Context, network, host, dns string) ([]net.IP, error) { + netResolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{ + Timeout: 3 * time.Second, + } + return d.DialContext(ctx, network, fmt.Sprintf("%s:%d", dns, 53)) + }, + } + return netResolver.LookupIP(ctx, network, host) +} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..3b10c6e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +ignore: + - checks/clients/ip/ip.go # this contains go std lib code wrapped for interfaces, not worth testing diff --git a/handlers/block_lists.go b/handlers/block_lists.go index 3a8062e..25080d8 100644 --- a/handlers/block_lists.go +++ b/handlers/block_lists.go @@ -1,143 +1,19 @@ package handlers import ( - "context" - "encoding/json" - "net" "net/http" - "slices" - "sort" - "sync" - "time" -) - -type dnsServer struct { - Name string - IP string -} - -var DNS_SERVERS = []dnsServer{ - {Name: "AdGuard", IP: "176.103.130.130"}, - {Name: "AdGuard Family", IP: "176.103.130.132"}, - {Name: "CleanBrowsing Adult", IP: "185.228.168.10"}, - {Name: "CleanBrowsing Family", IP: "185.228.168.168"}, - {Name: "CleanBrowsing Security", IP: "185.228.168.9"}, - {Name: "CloudFlare", IP: "1.1.1.1"}, - {Name: "CloudFlare Family", IP: "1.1.1.3"}, - {Name: "Comodo Secure", IP: "8.26.56.26"}, - {Name: "Google DNS", IP: "8.8.8.8"}, - {Name: "Neustar Family", IP: "156.154.70.3"}, - {Name: "Neustar Protection", IP: "156.154.70.2"}, - {Name: "Norton Family", IP: "199.85.126.20"}, - {Name: "OpenDNS", IP: "208.67.222.222"}, - {Name: "OpenDNS Family", IP: "208.67.222.123"}, - {Name: "Quad9", IP: "9.9.9.9"}, - {Name: "Yandex Family", IP: "77.88.8.7"}, - {Name: "Yandex Safe", IP: "77.88.8.88"}, -} - -var knownBlockIPs = []string{ - "146.112.61.106", - "185.228.168.10", - "8.26.56.26", - "9.9.9.9", - "208.69.38.170", - "208.69.39.170", - "208.67.222.222", - "208.67.222.123", - "199.85.126.10", - "199.85.126.20", - "156.154.70.22", - "77.88.8.7", - "77.88.8.8", - "::1", - "2a02:6b8::feed:0ff", - "2a02:6b8::feed:bad", - "2a02:6b8::feed:a11", - "2620:119:35::35", - "2620:119:53::53", - "2606:4700:4700::1111", - "2606:4700:4700::1001", - "2001:4860:4860::8888", - "2a0d:2a00:1::", - "2a0d:2a00:2::", -} - -type Blocklist struct { - Server string `json:"server"` - ServerIP string `json:"serverIp"` - IsBlocked bool `json:"isBlocked"` -} - -func isDomainBlocked(domain, serverIP string) bool { - resolver := &net.Resolver{ - PreferGo: true, - Dial: func(ctx context.Context, network, address string) (net.Conn, error) { - d := net.Dialer{ - Timeout: time.Second * 3, - } - return d.DialContext(ctx, network, serverIP+":53") - }, - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - ips, err := resolver.LookupIP(ctx, "ip4", domain) - if err != nil { - // if there's an error, consider it not blocked - return false - } - - return slices.ContainsFunc(ips, func(ip net.IP) bool { - return slices.Contains(knownBlockIPs, ip.String()) - }) -} - -func checkDomainAgainstDNSServers(domain string) []Blocklist { - var lock sync.Mutex - var wg sync.WaitGroup - limit := make(chan struct{}, 5) - - var results []Blocklist - - for _, server := range DNS_SERVERS { - wg.Add(1) - go func(server dnsServer) { - limit <- struct{}{} - defer func() { - <-limit - wg.Done() - }() - - isBlocked := isDomainBlocked(domain, server.IP) - lock.Lock() - defer lock.Unlock() - results = append(results, Blocklist{ - Server: server.Name, - ServerIP: server.IP, - IsBlocked: isBlocked, - }) - }(server) - } - wg.Wait() - - sort.Slice(results, func(i, j int) bool { - return results[i].Server > results[j].Server - }) - return results -} + "github.com/xray-web/web-check-api/checks" +) -func HandleBlockLists() http.Handler { - type Response struct { - BlockLists []Blocklist `json:"blocklists"` - } +func HandleBlockLists(b *checks.BlockList) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { rawURL, err := extractURL(r) if err != nil { JSONError(w, ErrMissingURLParameter, http.StatusBadRequest) return } - json.NewEncoder(w).Encode(Response{BlockLists: checkDomainAgainstDNSServers(rawURL.Hostname())}) + list := b.BlockedServers(r.Context(), rawURL.Hostname()) + JSON(w, list, http.StatusOK) }) } diff --git a/handlers/block_lists_test.go b/handlers/block_lists_test.go index 252f4ba..164d817 100644 --- a/handlers/block_lists_test.go +++ b/handlers/block_lists_test.go @@ -16,31 +16,9 @@ func TestHandleBlockLists(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/blocklists", nil) rec := httptest.NewRecorder() - HandleBlockLists().ServeHTTP(rec, req) + HandleBlockLists(nil).ServeHTTP(rec, req) assert.Equal(t, http.StatusBadRequest, rec.Code) assert.JSONEq(t, `{"error": "missing URL parameter"}`, rec.Body.String()) }) - - t.Run("blocked domain", func(t *testing.T) { - t.Parallel() - req := httptest.NewRequest(http.MethodGet, "/blocklists?url=http://blocked.example.com", nil) - rec := httptest.NewRecorder() - - HandleBlockLists().ServeHTTP(rec, req) - - assert.Equal(t, http.StatusOK, rec.Code) - - }) - - t.Run("unblocked domain", func(t *testing.T) { - t.Parallel() - req := httptest.NewRequest(http.MethodGet, "/blocklists?url=http://unblocked.example.com", nil) - rec := httptest.NewRecorder() - - HandleBlockLists().ServeHTTP(rec, req) - - assert.Equal(t, http.StatusOK, rec.Code) - - }) } diff --git a/handlers/legacy_rank_test.go b/handlers/legacy_rank_test.go index 62d9d40..ede5956 100644 --- a/handlers/legacy_rank_test.go +++ b/handlers/legacy_rank_test.go @@ -16,7 +16,7 @@ func TestHandleLegacyRank(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/legacy-rank", nil) rec := httptest.NewRecorder() - HandleBlockLists().ServeHTTP(rec, req) + HandleLegacyRank(nil).ServeHTTP(rec, req) assert.Equal(t, http.StatusBadRequest, rec.Code) assert.JSONEq(t, `{"error": "missing URL parameter"}`, rec.Body.String()) diff --git a/handlers/rank_test.go b/handlers/rank_test.go index 9bfc75e..e9bd1be 100644 --- a/handlers/rank_test.go +++ b/handlers/rank_test.go @@ -19,7 +19,7 @@ func TestHandleGetRank(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/rank", nil) rec := httptest.NewRecorder() - HandleBlockLists().ServeHTTP(rec, req) + HandleGetRank(nil).ServeHTTP(rec, req) assert.Equal(t, http.StatusBadRequest, rec.Code) assert.JSONEq(t, `{"error": "missing URL parameter"}`, rec.Body.String()) diff --git a/server/server.go b/server/server.go index 1b55c4b..b8f00ee 100644 --- a/server/server.go +++ b/server/server.go @@ -29,7 +29,7 @@ func (s *Server) routes() { s.mux.Handle("GET /health", HealthCheck()) - s.mux.Handle("GET /api/block-lists", handlers.HandleBlockLists()) + s.mux.Handle("GET /api/block-lists", handlers.HandleBlockLists(s.checks.BlockList)) s.mux.Handle("GET /api/carbon", handlers.HandleCarbon(s.checks.Carbon)) s.mux.Handle("GET /api/cookies", handlers.HandleCookies()) s.mux.Handle("GET /api/dns-server", handlers.HandleDNSServer())