From f00624b0b6096afa7d4a92dac1f4ffe15f676e3f Mon Sep 17 00:00:00 2001 From: Darius Rapalis Date: Mon, 14 Apr 2025 20:29:02 +0300 Subject: [PATCH] init --- .gitignore | 2 + go.mod | 5 ++ go.sum | 2 + main.go | 191 +++++++++++++++++++++++++++++++++++++++++++++++++ parser.go | 67 +++++++++++++++++ parser_test.go | 142 ++++++++++++++++++++++++++++++++++++ 6 files changed, 409 insertions(+) create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 parser.go create mode 100644 parser_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3f050c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +crawler +.idea \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cbeedf0 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/rdarius/go-web-crawler + +go 1.23.6 + +require golang.org/x/net v0.39.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..05eec08 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= diff --git a/main.go b/main.go new file mode 100644 index 0000000..4873af3 --- /dev/null +++ b/main.go @@ -0,0 +1,191 @@ +package main + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "sync" +) + +type config struct { + pages map[string]int + baseURL *url.URL + mu *sync.Mutex + concurrencyControl chan struct{} + wg *sync.WaitGroup + maxPages int +} + +func (cfg *config) addPageVisit(normalizedURL string) (isFirst bool) { + cfg.mu.Lock() + defer cfg.mu.Unlock() + _, exists := cfg.pages[normalizedURL] + if exists { + cfg.pages[normalizedURL]++ + return false + } + cfg.pages[normalizedURL] = 1 + return true +} + +func configure(rawBaseURL string, maxConcurrency int, maxPages int) (*config, error) { + baseURL, err := url.Parse(rawBaseURL) + if err != nil { + return nil, fmt.Errorf("couldn't parse base URL: %v", err) + } + + return &config{ + pages: make(map[string]int), + baseURL: baseURL, + mu: &sync.Mutex{}, + concurrencyControl: make(chan struct{}, maxConcurrency), + wg: &sync.WaitGroup{}, + maxPages: maxPages, + }, nil +} + +func (cfg *config) crawlPage(rawCurrentURL string) { + cfg.concurrencyControl <- struct{}{} + defer func() { + <-cfg.concurrencyControl + cfg.wg.Done() + }() + + if cfg.maxPages <= len(cfg.pages) { + return + } + + currentURL, err := url.Parse(rawCurrentURL) + if err != nil { + fmt.Printf("Error parsing URL: %v\n", err) + return + } + + if currentURL.Hostname() != cfg.baseURL.Hostname() { + return + } + + normalized, err := normalizeURL(rawCurrentURL) + if err != nil { + fmt.Println(err) + return + } + isFirst := cfg.addPageVisit(normalized) + if !isFirst { + return + } + fmt.Printf("Crawling page: %s\n", rawCurrentURL) + html, err := getHTML(rawCurrentURL) + if err != nil { + fmt.Println(err) + return + } + urls, err := getURLsFromHTML(html, cfg.baseURL) + if err != nil { + fmt.Println(err) + return + } + for _, urlItem := range urls { + cfg.wg.Add(1) + go cfg.crawlPage(urlItem) + } + +} + +func getHTML(rawURL string) (string, error) { + res, err := http.Get(rawURL) + if err != nil { + return "", err + } + if res.StatusCode >= 400 { + return "", errors.New(res.Status) + } + contentType := res.Header.Get("Content-Type") + if !strings.Contains(contentType, "text/html") { + return "", fmt.Errorf("got non-HTML response: %s", contentType) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +type Page struct { + Name string + Count int +} +type ByCountAndName []Page + +func (a ByCountAndName) Len() int { return len(a) } +func (a ByCountAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByCountAndName) Less(i, j int) bool { + // Sort by Count (descending) and then by Name (ascending) + if a[i].Count == a[j].Count { + return a[i].Name < a[j].Name + } + return a[i].Count > a[j].Count +} +func printReport(pages map[string]int, baseURL string) { + + var pagesList []Page + for name, count := range pages { + pagesList = append(pagesList, Page{Name: name, Count: count}) + } + sort.Sort(ByCountAndName(pagesList)) + + fmt.Printf("=============================\n REPORT for %s\n=============================", baseURL) + for _, page := range pagesList { + fmt.Printf("Found %d internal links to %s\n", page.Count, page.Name) + } + +} + +func main() { + if len(os.Args) < 2 { + fmt.Println("no website provided") + os.Exit(1) + } + if len(os.Args) > 4 { + fmt.Println("too many arguments provided") + os.Exit(1) + } + + baseURL, err := url.Parse(os.Args[1]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + concurrent, err := strconv.Atoi(os.Args[2]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + maxPages, err := strconv.Atoi(os.Args[3]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + cfg, err := configure(baseURL.String(), concurrent, maxPages) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + fmt.Printf("starting crawl of: %s\n", baseURL) + + cfg.wg.Add(1) + go cfg.crawlPage(baseURL.String()) + cfg.wg.Wait() + + printReport(cfg.pages, baseURL.String()) + +} diff --git a/parser.go b/parser.go new file mode 100644 index 0000000..a9eca62 --- /dev/null +++ b/parser.go @@ -0,0 +1,67 @@ +package main + +import ( + "golang.org/x/net/html" + url2 "net/url" + "strings" +) + +func normalizeURL(url string) (string, error) { + + parsedDomain, err := url2.Parse(url) + if err != nil { + return "", err + } + + newDomain := strings.ToLower(parsedDomain.Host) + if len(parsedDomain.Path) > 0 { + newDomain += strings.TrimRight(parsedDomain.Path, "/") + } + + return newDomain, nil +} + +func extractUrls(nodes *html.Node) []string { + urls := make([]string, 0) + + if nodes.Type == html.ElementNode && nodes.Data == "a" { + for _, a := range nodes.Attr { + if a.Key == "href" { + urls = append(urls, a.Val) + } + } + } + + if nodes.FirstChild != nil { + urls = append(urls, extractUrls(nodes.FirstChild)...) + } + + if nodes.NextSibling != nil { + urls = append(urls, extractUrls(nodes.NextSibling)...) + } + + return urls +} + +func getURLsFromHTML(htmlBody string, rawBaseURL *url2.URL) ([]string, error) { + + reader := strings.NewReader(htmlBody) + nodes, err := html.Parse(reader) + if err != nil { + return nil, err + } + + parsedUrls := extractUrls(nodes) + + // append rawBaseUrl if it does not exist + for index, url := range parsedUrls { + if !strings.HasPrefix(url, "http") { + parsedUrls[index], err = url2.JoinPath(rawBaseURL.String(), url) + if err != nil { + return nil, err + } + } + } + + return parsedUrls, nil +} diff --git a/parser_test.go b/parser_test.go new file mode 100644 index 0000000..5794ebe --- /dev/null +++ b/parser_test.go @@ -0,0 +1,142 @@ +package main + +import ( + url2 "net/url" + "reflect" + "sort" + "testing" +) + +func TestNormalizeURL(t *testing.T) { + tests := []struct { + name string + inputURL string + expected string + }{ + { + name: "remove scheme", + inputURL: "https://blog.boot.dev/path", + expected: "blog.boot.dev/path", + }, + { + name: "lowercase domain", + inputURL: "https://blog.bOOt.dev/path", + expected: "blog.boot.dev/path", + }, + { + name: "remove empty query", + inputURL: "https://blog.bOOt.dev/path?", + expected: "blog.boot.dev/path", + }, + { + name: "remove railing slash", + inputURL: "https://blog.bOOt.dev/path/", + expected: "blog.boot.dev/path", + }, + } + + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual, err := normalizeURL(tc.inputURL) + if err != nil { + t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err) + return + } + if actual != tc.expected { + t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual) + } + }) + } +} + +func TestGetURLsFromHTML(t *testing.T) { + tests := []struct { + name string + inputURL string + inputBody string + expected []string + }{ + { + name: "absolute and relative URLs", + inputURL: "https://blog.boot.dev", + inputBody: ` + + + + Boot.dev + + + Boot.dev + + + +`, + expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"}, + }, + { + name: "absolute and relative URLs", + inputURL: "https://blog.boot.dev", + inputBody: ` + + + + Boot.dev + + + Boot.dev + + + +`, + expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"}, + }, + { + name: "absolute and relative URLs", + inputURL: "https://blog.boot.dev", + inputBody: ` + + + + Boot.dev + + + Boot.dev + + + +`, + expected: []string{"https://blog.boot.dev/path/one/", "https://other.com/path/one"}, + }, + } + + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + parsed, err := url2.Parse(tc.inputURL) + if err != nil { + t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err) + return + } + actual, err := getURLsFromHTML(tc.inputBody, parsed) + if err != nil { + t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err) + return + } + + if len(actual) != len(tc.expected) { + t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual) + return + } + aCopy := make([]string, len(actual)) + copy(aCopy, actual) + bCopy := make([]string, len(tc.expected)) + copy(bCopy, tc.expected) + + sort.Strings(aCopy) + sort.Strings(bCopy) + + if !reflect.DeepEqual(aCopy, bCopy) { + t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual) + } + }) + } +}