package main

import (
	"golang.org/x/net/html"
	url2 "net/url"
	"strings"
)

func normalizeURL(url string) (string, error) {

	parsedDomain, err := url2.Parse(url)
	if err != nil {
		return "", err
	}

	newDomain := strings.ToLower(parsedDomain.Host)
	if len(parsedDomain.Path) > 0 {
		newDomain += strings.TrimRight(parsedDomain.Path, "/")
	}

	return newDomain, nil
}

func extractUrls(nodes *html.Node) []string {
	urls := make([]string, 0)

	if nodes.Type == html.ElementNode && nodes.Data == "a" {
		for _, a := range nodes.Attr {
			if a.Key == "href" {
				urls = append(urls, a.Val)
			}
		}
	}

	if nodes.FirstChild != nil {
		urls = append(urls, extractUrls(nodes.FirstChild)...)
	}

	if nodes.NextSibling != nil {
		urls = append(urls, extractUrls(nodes.NextSibling)...)
	}

	return urls
}

func getURLsFromHTML(htmlBody string, rawBaseURL *url2.URL) ([]string, error) {

	reader := strings.NewReader(htmlBody)
	nodes, err := html.Parse(reader)
	if err != nil {
		return nil, err
	}

	parsedUrls := extractUrls(nodes)

	// append rawBaseUrl if it does not exist
	for index, url := range parsedUrls {
		if !strings.HasPrefix(url, "http") {
			parsedUrls[index], err = url2.JoinPath(rawBaseURL.String(), url)
			if err != nil {
				return nil, err
			}
		}
	}

	return parsedUrls, nil
}