go-crawler/main.go

package main

import (
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"sync"
)

type config struct {
	pages              map[string]int
	baseURL            *url.URL
	mu                 *sync.Mutex
	concurrencyControl chan struct{}
	wg                 *sync.WaitGroup
	maxPages           int
}

func (cfg *config) addPageVisit(normalizedURL string) (isFirst bool) {
	cfg.mu.Lock()
	defer cfg.mu.Unlock()
	_, exists := cfg.pages[normalizedURL]
	if exists {
		cfg.pages[normalizedURL]++
		return false
	}
	cfg.pages[normalizedURL] = 1
	return true
}

func configure(rawBaseURL string, maxConcurrency int, maxPages int) (*config, error) {
	baseURL, err := url.Parse(rawBaseURL)
	if err != nil {
		return nil, fmt.Errorf("couldn't parse base URL: %v", err)
	}

	return &config{
		pages:              make(map[string]int),
		baseURL:            baseURL,
		mu:                 &sync.Mutex{},
		concurrencyControl: make(chan struct{}, maxConcurrency),
		wg:                 &sync.WaitGroup{},
		maxPages:           maxPages,
	}, nil
}

func (cfg *config) crawlPage(rawCurrentURL string) {
	cfg.concurrencyControl <- struct{}{}
	defer func() {
		<-cfg.concurrencyControl
		cfg.wg.Done()
	}()

	if cfg.maxPages <= len(cfg.pages) {
		return
	}

	currentURL, err := url.Parse(rawCurrentURL)
	if err != nil {
		fmt.Printf("Error parsing URL: %v\n", err)
		return
	}

	if currentURL.Hostname() != cfg.baseURL.Hostname() {
		return
	}

	normalized, err := normalizeURL(rawCurrentURL)
	if err != nil {
		fmt.Println(err)
		return
	}
	isFirst := cfg.addPageVisit(normalized)
	if !isFirst {
		return
	}
	fmt.Printf("Crawling page: %s\n", rawCurrentURL)
	html, err := getHTML(rawCurrentURL)
	if err != nil {
		fmt.Println(err)
		return
	}
	urls, err := getURLsFromHTML(html, cfg.baseURL)
	if err != nil {
		fmt.Println(err)
		return
	}
	for _, urlItem := range urls {
		cfg.wg.Add(1)
		go cfg.crawlPage(urlItem)
	}

}

func getHTML(rawURL string) (string, error) {
	res, err := http.Get(rawURL)
	if err != nil {
		return "", err
	}
	if res.StatusCode >= 400 {
		return "", errors.New(res.Status)
	}
	contentType := res.Header.Get("Content-Type")
	if !strings.Contains(contentType, "text/html") {
		return "", fmt.Errorf("got non-HTML response: %s", contentType)
	}
	body, err := io.ReadAll(res.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}

type Page struct {
	Name  string
	Count int
}
type ByCountAndName []Page

func (a ByCountAndName) Len() int      { return len(a) }
func (a ByCountAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByCountAndName) Less(i, j int) bool {
	// Sort by Count (descending) and then by Name (ascending)
	if a[i].Count == a[j].Count {
		return a[i].Name < a[j].Name
	}
	return a[i].Count > a[j].Count
}
func printReport(pages map[string]int, baseURL string) {

	var pagesList []Page
	for name, count := range pages {
		pagesList = append(pagesList, Page{Name: name, Count: count})
	}
	sort.Sort(ByCountAndName(pagesList))

	fmt.Printf("=============================\n  REPORT for %s\n=============================", baseURL)
	for _, page := range pagesList {
		fmt.Printf("Found %d internal links to %s\n", page.Count, page.Name)
	}

}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("no website provided")
		os.Exit(1)
	}
	if len(os.Args) > 4 {
		fmt.Println("too many arguments provided")
		os.Exit(1)
	}

	baseURL, err := url.Parse(os.Args[1])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	concurrent, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	maxPages, err := strconv.Atoi(os.Args[3])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	cfg, err := configure(baseURL.String(), concurrent, maxPages)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	fmt.Printf("starting crawl of: %s\n", baseURL)

	cfg.wg.Add(1)
	go cfg.crawlPage(baseURL.String())
	cfg.wg.Wait()

	printReport(cfg.pages, baseURL.String())

}