Files
go-crawler/parser_test.go
Darius Rapalis f00624b0b6 init
2025-04-14 20:29:02 +03:00

143 lines
3.1 KiB
Go

package main
import (
url2 "net/url"
"reflect"
"sort"
"testing"
)
func TestNormalizeURL(t *testing.T) {
tests := []struct {
name string
inputURL string
expected string
}{
{
name: "remove scheme",
inputURL: "https://blog.boot.dev/path",
expected: "blog.boot.dev/path",
},
{
name: "lowercase domain",
inputURL: "https://blog.bOOt.dev/path",
expected: "blog.boot.dev/path",
},
{
name: "remove empty query",
inputURL: "https://blog.bOOt.dev/path?",
expected: "blog.boot.dev/path",
},
{
name: "remove railing slash",
inputURL: "https://blog.bOOt.dev/path/",
expected: "blog.boot.dev/path",
},
}
for i, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
actual, err := normalizeURL(tc.inputURL)
if err != nil {
t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
return
}
if actual != tc.expected {
t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
}
})
}
}
func TestGetURLsFromHTML(t *testing.T) {
tests := []struct {
name string
inputURL string
inputBody string
expected []string
}{
{
name: "absolute and relative URLs",
inputURL: "https://blog.boot.dev",
inputBody: `
<html>
<body>
<a href="/path/one">
<span>Boot.dev</span>
</a>
<a href="https://other.com/path/one">
<span>Boot.dev</span>
</a>
</body>
</html>
`,
expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"},
},
{
name: "absolute and relative URLs",
inputURL: "https://blog.boot.dev",
inputBody: `
<html>
<body>
<strong><a href="/path/one">
<span>Boot.dev</span>
</a></strong>
<a href="https://other.com/path/one">
<span>Boot.dev</span>
</a>
</body>
</html>
`,
expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"},
},
{
name: "absolute and relative URLs",
inputURL: "https://blog.boot.dev",
inputBody: `
<html>
<body>
<a href="https://blog.boot.dev/path/one/">
<span>Boot.dev</span>
</a>
<a href="https://other.com/path/one">
<span>Boot.dev</span>
</a>
</body>
</html>
`,
expected: []string{"https://blog.boot.dev/path/one/", "https://other.com/path/one"},
},
}
for i, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
parsed, err := url2.Parse(tc.inputURL)
if err != nil {
t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
return
}
actual, err := getURLsFromHTML(tc.inputBody, parsed)
if err != nil {
t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
return
}
if len(actual) != len(tc.expected) {
t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
return
}
aCopy := make([]string, len(actual))
copy(aCopy, actual)
bCopy := make([]string, len(tc.expected))
copy(bCopy, tc.expected)
sort.Strings(aCopy)
sort.Strings(bCopy)
if !reflect.DeepEqual(aCopy, bCopy) {
t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
}
})
}
}