init

.gitignore

@@ -0,0 +1,2 @@
+crawler
+.idea

go.mod

@@ -0,0 +1,5 @@
+module github.com/rdarius/go-web-crawler
+
+go 1.23.6
+
+require golang.org/x/net v0.39.0 // indirect

go.sum

@@ -0,0 +1,2 @@
+golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
+golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=

main.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type config struct {
+	pages              map[string]int
+	baseURL            *url.URL
+	mu                 *sync.Mutex
+	concurrencyControl chan struct{}
+	wg                 *sync.WaitGroup
+	maxPages           int
+}
+
+func (cfg *config) addPageVisit(normalizedURL string) (isFirst bool) {
+	cfg.mu.Lock()
+	defer cfg.mu.Unlock()
+	_, exists := cfg.pages[normalizedURL]
+	if exists {
+		cfg.pages[normalizedURL]++
+		return false
+	}
+	cfg.pages[normalizedURL] = 1
+	return true
+}
+
+func configure(rawBaseURL string, maxConcurrency int, maxPages int) (*config, error) {
+	baseURL, err := url.Parse(rawBaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't parse base URL: %v", err)
+	}
+
+	return &config{
+		pages:              make(map[string]int),
+		baseURL:            baseURL,
+		mu:                 &sync.Mutex{},
+		concurrencyControl: make(chan struct{}, maxConcurrency),
+		wg:                 &sync.WaitGroup{},
+		maxPages:           maxPages,
+	}, nil
+}
+
+func (cfg *config) crawlPage(rawCurrentURL string) {
+	cfg.concurrencyControl <- struct{}{}
+	defer func() {
+		<-cfg.concurrencyControl
+		cfg.wg.Done()
+	}()
+
+	if cfg.maxPages <= len(cfg.pages) {
+		return
+	}
+
+	currentURL, err := url.Parse(rawCurrentURL)
+	if err != nil {
+		fmt.Printf("Error parsing URL: %v\n", err)
+		return
+	}
+
+	if currentURL.Hostname() != cfg.baseURL.Hostname() {
+		return
+	}
+
+	normalized, err := normalizeURL(rawCurrentURL)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	isFirst := cfg.addPageVisit(normalized)
+	if !isFirst {
+		return
+	}
+	fmt.Printf("Crawling page: %s\n", rawCurrentURL)
+	html, err := getHTML(rawCurrentURL)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	urls, err := getURLsFromHTML(html, cfg.baseURL)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	for _, urlItem := range urls {
+		cfg.wg.Add(1)
+		go cfg.crawlPage(urlItem)
+	}
+
+}
+
+func getHTML(rawURL string) (string, error) {
+	res, err := http.Get(rawURL)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode >= 400 {
+		return "", errors.New(res.Status)
+	}
+	contentType := res.Header.Get("Content-Type")
+	if !strings.Contains(contentType, "text/html") {
+		return "", fmt.Errorf("got non-HTML response: %s", contentType)
+	}
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+type Page struct {
+	Name  string
+	Count int
+}
+type ByCountAndName []Page
+
+func (a ByCountAndName) Len() int      { return len(a) }
+func (a ByCountAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByCountAndName) Less(i, j int) bool {
+	// Sort by Count (descending) and then by Name (ascending)
+	if a[i].Count == a[j].Count {
+		return a[i].Name < a[j].Name
+	}
+	return a[i].Count > a[j].Count
+}
+func printReport(pages map[string]int, baseURL string) {
+
+	var pagesList []Page
+	for name, count := range pages {
+		pagesList = append(pagesList, Page{Name: name, Count: count})
+	}
+	sort.Sort(ByCountAndName(pagesList))
+
+	fmt.Printf("=============================\n  REPORT for %s\n=============================", baseURL)
+	for _, page := range pagesList {
+		fmt.Printf("Found %d internal links to %s\n", page.Count, page.Name)
+	}
+
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("no website provided")
+		os.Exit(1)
+	}
+	if len(os.Args) > 4 {
+		fmt.Println("too many arguments provided")
+		os.Exit(1)
+	}
+
+	baseURL, err := url.Parse(os.Args[1])
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	concurrent, err := strconv.Atoi(os.Args[2])
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	maxPages, err := strconv.Atoi(os.Args[3])
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	cfg, err := configure(baseURL.String(), concurrent, maxPages)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("starting crawl of: %s\n", baseURL)
+
+	cfg.wg.Add(1)
+	go cfg.crawlPage(baseURL.String())
+	cfg.wg.Wait()
+
+	printReport(cfg.pages, baseURL.String())
+
+}

parser.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"golang.org/x/net/html"
+	url2 "net/url"
+	"strings"
+)
+
+func normalizeURL(url string) (string, error) {
+
+	parsedDomain, err := url2.Parse(url)
+	if err != nil {
+		return "", err
+	}
+
+	newDomain := strings.ToLower(parsedDomain.Host)
+	if len(parsedDomain.Path) > 0 {
+		newDomain += strings.TrimRight(parsedDomain.Path, "/")
+	}
+
+	return newDomain, nil
+}
+
+func extractUrls(nodes *html.Node) []string {
+	urls := make([]string, 0)
+
+	if nodes.Type == html.ElementNode && nodes.Data == "a" {
+		for _, a := range nodes.Attr {
+			if a.Key == "href" {
+				urls = append(urls, a.Val)
+			}
+		}
+	}
+
+	if nodes.FirstChild != nil {
+		urls = append(urls, extractUrls(nodes.FirstChild)...)
+	}
+
+	if nodes.NextSibling != nil {
+		urls = append(urls, extractUrls(nodes.NextSibling)...)
+	}
+
+	return urls
+}
+
+func getURLsFromHTML(htmlBody string, rawBaseURL *url2.URL) ([]string, error) {
+
+	reader := strings.NewReader(htmlBody)
+	nodes, err := html.Parse(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	parsedUrls := extractUrls(nodes)
+
+	// append rawBaseUrl if it does not exist
+	for index, url := range parsedUrls {
+		if !strings.HasPrefix(url, "http") {
+			parsedUrls[index], err = url2.JoinPath(rawBaseURL.String(), url)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return parsedUrls, nil
+}

parser_test.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	url2 "net/url"
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func TestNormalizeURL(t *testing.T) {
+	tests := []struct {
+		name     string
+		inputURL string
+		expected string
+	}{
+		{
+			name:     "remove scheme",
+			inputURL: "https://blog.boot.dev/path",
+			expected: "blog.boot.dev/path",
+		},
+		{
+			name:     "lowercase domain",
+			inputURL: "https://blog.bOOt.dev/path",
+			expected: "blog.boot.dev/path",
+		},
+		{
+			name:     "remove empty query",
+			inputURL: "https://blog.bOOt.dev/path?",
+			expected: "blog.boot.dev/path",
+		},
+		{
+			name:     "remove railing slash",
+			inputURL: "https://blog.bOOt.dev/path/",
+			expected: "blog.boot.dev/path",
+		},
+	}
+
+	for i, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			actual, err := normalizeURL(tc.inputURL)
+			if err != nil {
+				t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
+				return
+			}
+			if actual != tc.expected {
+				t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
+			}
+		})
+	}
+}
+
+func TestGetURLsFromHTML(t *testing.T) {
+	tests := []struct {
+		name      string
+		inputURL  string
+		inputBody string
+		expected  []string
+	}{
+		{
+			name:     "absolute and relative URLs",
+			inputURL: "https://blog.boot.dev",
+			inputBody: `
+<html>
+	<body>
+		<a href="/path/one">
+			<span>Boot.dev</span>
+		</a>
+		<a href="https://other.com/path/one">
+			<span>Boot.dev</span>
+		</a>
+	</body>
+</html>
+`,
+			expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"},
+		},
+		{
+			name:     "absolute and relative URLs",
+			inputURL: "https://blog.boot.dev",
+			inputBody: `
+<html>
+	<body>
+		<strong><a href="/path/one">
+			<span>Boot.dev</span>
+		</a></strong>
+		<a href="https://other.com/path/one">
+			<span>Boot.dev</span>
+		</a>
+	</body>
+</html>
+`,
+			expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"},
+		},
+		{
+			name:     "absolute and relative URLs",
+			inputURL: "https://blog.boot.dev",
+			inputBody: `
+<html>
+	<body>
+		<a href="https://blog.boot.dev/path/one/">
+			<span>Boot.dev</span>
+		</a>
+		<a href="https://other.com/path/one">
+			<span>Boot.dev</span>
+		</a>
+	</body>
+</html>
+`,
+			expected: []string{"https://blog.boot.dev/path/one/", "https://other.com/path/one"},
+		},
+	}
+
+	for i, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			parsed, err := url2.Parse(tc.inputURL)
+			if err != nil {
+				t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
+				return
+			}
+			actual, err := getURLsFromHTML(tc.inputBody, parsed)
+			if err != nil {
+				t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
+				return
+			}
+
+			if len(actual) != len(tc.expected) {
+				t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
+				return
+			}
+			aCopy := make([]string, len(actual))
+			copy(aCopy, actual)
+			bCopy := make([]string, len(tc.expected))
+			copy(bCopy, tc.expected)
+
+			sort.Strings(aCopy)
+			sort.Strings(bCopy)
+
+			if !reflect.DeepEqual(aCopy, bCopy) {
+				t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
+			}
+		})
+	}
+}