init

parser.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"golang.org/x/net/html"
+	url2 "net/url"
+	"strings"
+)
+
+func normalizeURL(url string) (string, error) {
+
+	parsedDomain, err := url2.Parse(url)
+	if err != nil {
+		return "", err
+	}
+
+	newDomain := strings.ToLower(parsedDomain.Host)
+	if len(parsedDomain.Path) > 0 {
+		newDomain += strings.TrimRight(parsedDomain.Path, "/")
+	}
+
+	return newDomain, nil
+}
+
+func extractUrls(nodes *html.Node) []string {
+	urls := make([]string, 0)
+
+	if nodes.Type == html.ElementNode && nodes.Data == "a" {
+		for _, a := range nodes.Attr {
+			if a.Key == "href" {
+				urls = append(urls, a.Val)
+			}
+		}
+	}
+
+	if nodes.FirstChild != nil {
+		urls = append(urls, extractUrls(nodes.FirstChild)...)
+	}
+
+	if nodes.NextSibling != nil {
+		urls = append(urls, extractUrls(nodes.NextSibling)...)
+	}
+
+	return urls
+}
+
+func getURLsFromHTML(htmlBody string, rawBaseURL *url2.URL) ([]string, error) {
+
+	reader := strings.NewReader(htmlBody)
+	nodes, err := html.Parse(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	parsedUrls := extractUrls(nodes)
+
+	// append rawBaseUrl if it does not exist
+	for index, url := range parsedUrls {
+		if !strings.HasPrefix(url, "http") {
+			parsedUrls[index], err = url2.JoinPath(rawBaseURL.String(), url)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return parsedUrls, nil
+}