68 lines
1.3 KiB
Go
68 lines
1.3 KiB
Go
package main
|
|
|
|
import (
|
|
"golang.org/x/net/html"
|
|
url2 "net/url"
|
|
"strings"
|
|
)
|
|
|
|
func normalizeURL(url string) (string, error) {
|
|
|
|
parsedDomain, err := url2.Parse(url)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
newDomain := strings.ToLower(parsedDomain.Host)
|
|
if len(parsedDomain.Path) > 0 {
|
|
newDomain += strings.TrimRight(parsedDomain.Path, "/")
|
|
}
|
|
|
|
return newDomain, nil
|
|
}
|
|
|
|
func extractUrls(nodes *html.Node) []string {
|
|
urls := make([]string, 0)
|
|
|
|
if nodes.Type == html.ElementNode && nodes.Data == "a" {
|
|
for _, a := range nodes.Attr {
|
|
if a.Key == "href" {
|
|
urls = append(urls, a.Val)
|
|
}
|
|
}
|
|
}
|
|
|
|
if nodes.FirstChild != nil {
|
|
urls = append(urls, extractUrls(nodes.FirstChild)...)
|
|
}
|
|
|
|
if nodes.NextSibling != nil {
|
|
urls = append(urls, extractUrls(nodes.NextSibling)...)
|
|
}
|
|
|
|
return urls
|
|
}
|
|
|
|
func getURLsFromHTML(htmlBody string, rawBaseURL *url2.URL) ([]string, error) {
|
|
|
|
reader := strings.NewReader(htmlBody)
|
|
nodes, err := html.Parse(reader)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
parsedUrls := extractUrls(nodes)
|
|
|
|
// append rawBaseUrl if it does not exist
|
|
for index, url := range parsedUrls {
|
|
if !strings.HasPrefix(url, "http") {
|
|
parsedUrls[index], err = url2.JoinPath(rawBaseURL.String(), url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
}
|
|
|
|
return parsedUrls, nil
|
|
}
|