init

main.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+type config struct {
+	pages              map[string]int
+	baseURL            *url.URL
+	mu                 *sync.Mutex
+	concurrencyControl chan struct{}
+	wg                 *sync.WaitGroup
+	maxPages           int
+}
+
+func (cfg *config) addPageVisit(normalizedURL string) (isFirst bool) {
+	cfg.mu.Lock()
+	defer cfg.mu.Unlock()
+	_, exists := cfg.pages[normalizedURL]
+	if exists {
+		cfg.pages[normalizedURL]++
+		return false
+	}
+	cfg.pages[normalizedURL] = 1
+	return true
+}
+
+func configure(rawBaseURL string, maxConcurrency int, maxPages int) (*config, error) {
+	baseURL, err := url.Parse(rawBaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't parse base URL: %v", err)
+	}
+
+	return &config{
+		pages:              make(map[string]int),
+		baseURL:            baseURL,
+		mu:                 &sync.Mutex{},
+		concurrencyControl: make(chan struct{}, maxConcurrency),
+		wg:                 &sync.WaitGroup{},
+		maxPages:           maxPages,
+	}, nil
+}
+
+func (cfg *config) crawlPage(rawCurrentURL string) {
+	cfg.concurrencyControl <- struct{}{}
+	defer func() {
+		<-cfg.concurrencyControl
+		cfg.wg.Done()
+	}()
+
+	if cfg.maxPages <= len(cfg.pages) {
+		return
+	}
+
+	currentURL, err := url.Parse(rawCurrentURL)
+	if err != nil {
+		fmt.Printf("Error parsing URL: %v\n", err)
+		return
+	}
+
+	if currentURL.Hostname() != cfg.baseURL.Hostname() {
+		return
+	}
+
+	normalized, err := normalizeURL(rawCurrentURL)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	isFirst := cfg.addPageVisit(normalized)
+	if !isFirst {
+		return
+	}
+	fmt.Printf("Crawling page: %s\n", rawCurrentURL)
+	html, err := getHTML(rawCurrentURL)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	urls, err := getURLsFromHTML(html, cfg.baseURL)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	for _, urlItem := range urls {
+		cfg.wg.Add(1)
+		go cfg.crawlPage(urlItem)
+	}
+
+}
+
+func getHTML(rawURL string) (string, error) {
+	res, err := http.Get(rawURL)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode >= 400 {
+		return "", errors.New(res.Status)
+	}
+	contentType := res.Header.Get("Content-Type")
+	if !strings.Contains(contentType, "text/html") {
+		return "", fmt.Errorf("got non-HTML response: %s", contentType)
+	}
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+type Page struct {
+	Name  string
+	Count int
+}
+type ByCountAndName []Page
+
+func (a ByCountAndName) Len() int      { return len(a) }
+func (a ByCountAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByCountAndName) Less(i, j int) bool {
+	// Sort by Count (descending) and then by Name (ascending)
+	if a[i].Count == a[j].Count {
+		return a[i].Name < a[j].Name
+	}
+	return a[i].Count > a[j].Count
+}
+func printReport(pages map[string]int, baseURL string) {
+
+	var pagesList []Page
+	for name, count := range pages {
+		pagesList = append(pagesList, Page{Name: name, Count: count})
+	}
+	sort.Sort(ByCountAndName(pagesList))
+
+	fmt.Printf("=============================\n  REPORT for %s\n=============================", baseURL)
+	for _, page := range pagesList {
+		fmt.Printf("Found %d internal links to %s\n", page.Count, page.Name)
+	}
+
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("no website provided")
+		os.Exit(1)
+	}
+	if len(os.Args) > 4 {
+		fmt.Println("too many arguments provided")
+		os.Exit(1)
+	}
+
+	baseURL, err := url.Parse(os.Args[1])
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	concurrent, err := strconv.Atoi(os.Args[2])
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	maxPages, err := strconv.Atoi(os.Args[3])
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	cfg, err := configure(baseURL.String(), concurrent, maxPages)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("starting crawl of: %s\n", baseURL)
+
+	cfg.wg.Add(1)
+	go cfg.crawlPage(baseURL.String())
+	cfg.wg.Wait()
+
+	printReport(cfg.pages, baseURL.String())
+
+}