package main import ( "errors" "fmt" "io" "net/http" "net/url" "os" "sort" "strconv" "strings" "sync" ) type config struct { pages map[string]int baseURL *url.URL mu *sync.Mutex concurrencyControl chan struct{} wg *sync.WaitGroup maxPages int } func (cfg *config) addPageVisit(normalizedURL string) (isFirst bool) { cfg.mu.Lock() defer cfg.mu.Unlock() _, exists := cfg.pages[normalizedURL] if exists { cfg.pages[normalizedURL]++ return false } cfg.pages[normalizedURL] = 1 return true } func configure(rawBaseURL string, maxConcurrency int, maxPages int) (*config, error) { baseURL, err := url.Parse(rawBaseURL) if err != nil { return nil, fmt.Errorf("couldn't parse base URL: %v", err) } return &config{ pages: make(map[string]int), baseURL: baseURL, mu: &sync.Mutex{}, concurrencyControl: make(chan struct{}, maxConcurrency), wg: &sync.WaitGroup{}, maxPages: maxPages, }, nil } func (cfg *config) crawlPage(rawCurrentURL string) { cfg.concurrencyControl <- struct{}{} defer func() { <-cfg.concurrencyControl cfg.wg.Done() }() if cfg.maxPages <= len(cfg.pages) { return } currentURL, err := url.Parse(rawCurrentURL) if err != nil { fmt.Printf("Error parsing URL: %v\n", err) return } if currentURL.Hostname() != cfg.baseURL.Hostname() { return } normalized, err := normalizeURL(rawCurrentURL) if err != nil { fmt.Println(err) return } isFirst := cfg.addPageVisit(normalized) if !isFirst { return } fmt.Printf("Crawling page: %s\n", rawCurrentURL) html, err := getHTML(rawCurrentURL) if err != nil { fmt.Println(err) return } urls, err := getURLsFromHTML(html, cfg.baseURL) if err != nil { fmt.Println(err) return } for _, urlItem := range urls { cfg.wg.Add(1) go cfg.crawlPage(urlItem) } } func getHTML(rawURL string) (string, error) { res, err := http.Get(rawURL) if err != nil { return "", err } if res.StatusCode >= 400 { return "", errors.New(res.Status) } contentType := res.Header.Get("Content-Type") if !strings.Contains(contentType, "text/html") { return "", fmt.Errorf("got non-HTML response: %s", contentType) } body, err := io.ReadAll(res.Body) if err != nil { return "", err } return string(body), nil } type Page struct { Name string Count int } type ByCountAndName []Page func (a ByCountAndName) Len() int { return len(a) } func (a ByCountAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByCountAndName) Less(i, j int) bool { // Sort by Count (descending) and then by Name (ascending) if a[i].Count == a[j].Count { return a[i].Name < a[j].Name } return a[i].Count > a[j].Count } func printReport(pages map[string]int, baseURL string) { var pagesList []Page for name, count := range pages { pagesList = append(pagesList, Page{Name: name, Count: count}) } sort.Sort(ByCountAndName(pagesList)) fmt.Printf("=============================\n REPORT for %s\n=============================", baseURL) for _, page := range pagesList { fmt.Printf("Found %d internal links to %s\n", page.Count, page.Name) } } func main() { if len(os.Args) < 2 { fmt.Println("no website provided") os.Exit(1) } if len(os.Args) > 4 { fmt.Println("too many arguments provided") os.Exit(1) } baseURL, err := url.Parse(os.Args[1]) if err != nil { fmt.Println(err) os.Exit(1) } concurrent, err := strconv.Atoi(os.Args[2]) if err != nil { fmt.Println(err) os.Exit(1) } maxPages, err := strconv.Atoi(os.Args[3]) if err != nil { fmt.Println(err) os.Exit(1) } cfg, err := configure(baseURL.String(), concurrent, maxPages) if err != nil { fmt.Println(err) os.Exit(1) } fmt.Printf("starting crawl of: %s\n", baseURL) cfg.wg.Add(1) go cfg.crawlPage(baseURL.String()) cfg.wg.Wait() printReport(cfg.pages, baseURL.String()) }