init
This commit is contained in:
Darius Rapalis
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
crawler
|
||||
.idea
|
||||
5
go.mod
Normal file
5
go.mod
Normal file
@@ -0,0 +1,5 @@
|
||||
module github.com/rdarius/go-web-crawler
|
||||
|
||||
go 1.23.6
|
||||
|
||||
require golang.org/x/net v0.39.0 // indirect
|
||||
2
go.sum
Normal file
2
go.sum
Normal file
@@ -0,0 +1,2 @@
|
||||
golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
|
||||
golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
|
||||
191
main.go
Normal file
191
main.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
pages map[string]int
|
||||
baseURL *url.URL
|
||||
mu *sync.Mutex
|
||||
concurrencyControl chan struct{}
|
||||
wg *sync.WaitGroup
|
||||
maxPages int
|
||||
}
|
||||
|
||||
func (cfg *config) addPageVisit(normalizedURL string) (isFirst bool) {
|
||||
cfg.mu.Lock()
|
||||
defer cfg.mu.Unlock()
|
||||
_, exists := cfg.pages[normalizedURL]
|
||||
if exists {
|
||||
cfg.pages[normalizedURL]++
|
||||
return false
|
||||
}
|
||||
cfg.pages[normalizedURL] = 1
|
||||
return true
|
||||
}
|
||||
|
||||
func configure(rawBaseURL string, maxConcurrency int, maxPages int) (*config, error) {
|
||||
baseURL, err := url.Parse(rawBaseURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't parse base URL: %v", err)
|
||||
}
|
||||
|
||||
return &config{
|
||||
pages: make(map[string]int),
|
||||
baseURL: baseURL,
|
||||
mu: &sync.Mutex{},
|
||||
concurrencyControl: make(chan struct{}, maxConcurrency),
|
||||
wg: &sync.WaitGroup{},
|
||||
maxPages: maxPages,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (cfg *config) crawlPage(rawCurrentURL string) {
|
||||
cfg.concurrencyControl <- struct{}{}
|
||||
defer func() {
|
||||
<-cfg.concurrencyControl
|
||||
cfg.wg.Done()
|
||||
}()
|
||||
|
||||
if cfg.maxPages <= len(cfg.pages) {
|
||||
return
|
||||
}
|
||||
|
||||
currentURL, err := url.Parse(rawCurrentURL)
|
||||
if err != nil {
|
||||
fmt.Printf("Error parsing URL: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
if currentURL.Hostname() != cfg.baseURL.Hostname() {
|
||||
return
|
||||
}
|
||||
|
||||
normalized, err := normalizeURL(rawCurrentURL)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
isFirst := cfg.addPageVisit(normalized)
|
||||
if !isFirst {
|
||||
return
|
||||
}
|
||||
fmt.Printf("Crawling page: %s\n", rawCurrentURL)
|
||||
html, err := getHTML(rawCurrentURL)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
urls, err := getURLsFromHTML(html, cfg.baseURL)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
for _, urlItem := range urls {
|
||||
cfg.wg.Add(1)
|
||||
go cfg.crawlPage(urlItem)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getHTML(rawURL string) (string, error) {
|
||||
res, err := http.Get(rawURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if res.StatusCode >= 400 {
|
||||
return "", errors.New(res.Status)
|
||||
}
|
||||
contentType := res.Header.Get("Content-Type")
|
||||
if !strings.Contains(contentType, "text/html") {
|
||||
return "", fmt.Errorf("got non-HTML response: %s", contentType)
|
||||
}
|
||||
body, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
type Page struct {
|
||||
Name string
|
||||
Count int
|
||||
}
|
||||
type ByCountAndName []Page
|
||||
|
||||
func (a ByCountAndName) Len() int { return len(a) }
|
||||
func (a ByCountAndName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
||||
func (a ByCountAndName) Less(i, j int) bool {
|
||||
// Sort by Count (descending) and then by Name (ascending)
|
||||
if a[i].Count == a[j].Count {
|
||||
return a[i].Name < a[j].Name
|
||||
}
|
||||
return a[i].Count > a[j].Count
|
||||
}
|
||||
func printReport(pages map[string]int, baseURL string) {
|
||||
|
||||
var pagesList []Page
|
||||
for name, count := range pages {
|
||||
pagesList = append(pagesList, Page{Name: name, Count: count})
|
||||
}
|
||||
sort.Sort(ByCountAndName(pagesList))
|
||||
|
||||
fmt.Printf("=============================\n REPORT for %s\n=============================", baseURL)
|
||||
for _, page := range pagesList {
|
||||
fmt.Printf("Found %d internal links to %s\n", page.Count, page.Name)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Println("no website provided")
|
||||
os.Exit(1)
|
||||
}
|
||||
if len(os.Args) > 4 {
|
||||
fmt.Println("too many arguments provided")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
baseURL, err := url.Parse(os.Args[1])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
concurrent, err := strconv.Atoi(os.Args[2])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
maxPages, err := strconv.Atoi(os.Args[3])
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cfg, err := configure(baseURL.String(), concurrent, maxPages)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
fmt.Printf("starting crawl of: %s\n", baseURL)
|
||||
|
||||
cfg.wg.Add(1)
|
||||
go cfg.crawlPage(baseURL.String())
|
||||
cfg.wg.Wait()
|
||||
|
||||
printReport(cfg.pages, baseURL.String())
|
||||
|
||||
}
|
||||
67
parser.go
Normal file
67
parser.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"golang.org/x/net/html"
|
||||
url2 "net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func normalizeURL(url string) (string, error) {
|
||||
|
||||
parsedDomain, err := url2.Parse(url)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
newDomain := strings.ToLower(parsedDomain.Host)
|
||||
if len(parsedDomain.Path) > 0 {
|
||||
newDomain += strings.TrimRight(parsedDomain.Path, "/")
|
||||
}
|
||||
|
||||
return newDomain, nil
|
||||
}
|
||||
|
||||
func extractUrls(nodes *html.Node) []string {
|
||||
urls := make([]string, 0)
|
||||
|
||||
if nodes.Type == html.ElementNode && nodes.Data == "a" {
|
||||
for _, a := range nodes.Attr {
|
||||
if a.Key == "href" {
|
||||
urls = append(urls, a.Val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if nodes.FirstChild != nil {
|
||||
urls = append(urls, extractUrls(nodes.FirstChild)...)
|
||||
}
|
||||
|
||||
if nodes.NextSibling != nil {
|
||||
urls = append(urls, extractUrls(nodes.NextSibling)...)
|
||||
}
|
||||
|
||||
return urls
|
||||
}
|
||||
|
||||
func getURLsFromHTML(htmlBody string, rawBaseURL *url2.URL) ([]string, error) {
|
||||
|
||||
reader := strings.NewReader(htmlBody)
|
||||
nodes, err := html.Parse(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parsedUrls := extractUrls(nodes)
|
||||
|
||||
// append rawBaseUrl if it does not exist
|
||||
for index, url := range parsedUrls {
|
||||
if !strings.HasPrefix(url, "http") {
|
||||
parsedUrls[index], err = url2.JoinPath(rawBaseURL.String(), url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return parsedUrls, nil
|
||||
}
|
||||
142
parser_test.go
Normal file
142
parser_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
url2 "net/url"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputURL string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "remove scheme",
|
||||
inputURL: "https://blog.boot.dev/path",
|
||||
expected: "blog.boot.dev/path",
|
||||
},
|
||||
{
|
||||
name: "lowercase domain",
|
||||
inputURL: "https://blog.bOOt.dev/path",
|
||||
expected: "blog.boot.dev/path",
|
||||
},
|
||||
{
|
||||
name: "remove empty query",
|
||||
inputURL: "https://blog.bOOt.dev/path?",
|
||||
expected: "blog.boot.dev/path",
|
||||
},
|
||||
{
|
||||
name: "remove railing slash",
|
||||
inputURL: "https://blog.bOOt.dev/path/",
|
||||
expected: "blog.boot.dev/path",
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
actual, err := normalizeURL(tc.inputURL)
|
||||
if err != nil {
|
||||
t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
|
||||
return
|
||||
}
|
||||
if actual != tc.expected {
|
||||
t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetURLsFromHTML(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputURL string
|
||||
inputBody string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "absolute and relative URLs",
|
||||
inputURL: "https://blog.boot.dev",
|
||||
inputBody: `
|
||||
<html>
|
||||
<body>
|
||||
<a href="/path/one">
|
||||
<span>Boot.dev</span>
|
||||
</a>
|
||||
<a href="https://other.com/path/one">
|
||||
<span>Boot.dev</span>
|
||||
</a>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"},
|
||||
},
|
||||
{
|
||||
name: "absolute and relative URLs",
|
||||
inputURL: "https://blog.boot.dev",
|
||||
inputBody: `
|
||||
<html>
|
||||
<body>
|
||||
<strong><a href="/path/one">
|
||||
<span>Boot.dev</span>
|
||||
</a></strong>
|
||||
<a href="https://other.com/path/one">
|
||||
<span>Boot.dev</span>
|
||||
</a>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
expected: []string{"https://blog.boot.dev/path/one", "https://other.com/path/one"},
|
||||
},
|
||||
{
|
||||
name: "absolute and relative URLs",
|
||||
inputURL: "https://blog.boot.dev",
|
||||
inputBody: `
|
||||
<html>
|
||||
<body>
|
||||
<a href="https://blog.boot.dev/path/one/">
|
||||
<span>Boot.dev</span>
|
||||
</a>
|
||||
<a href="https://other.com/path/one">
|
||||
<span>Boot.dev</span>
|
||||
</a>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
expected: []string{"https://blog.boot.dev/path/one/", "https://other.com/path/one"},
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
parsed, err := url2.Parse(tc.inputURL)
|
||||
if err != nil {
|
||||
t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
|
||||
return
|
||||
}
|
||||
actual, err := getURLsFromHTML(tc.inputBody, parsed)
|
||||
if err != nil {
|
||||
t.Errorf("Test %v - '%s' FAIL: unexpected error: %v", i, tc.name, err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(actual) != len(tc.expected) {
|
||||
t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
|
||||
return
|
||||
}
|
||||
aCopy := make([]string, len(actual))
|
||||
copy(aCopy, actual)
|
||||
bCopy := make([]string, len(tc.expected))
|
||||
copy(bCopy, tc.expected)
|
||||
|
||||
sort.Strings(aCopy)
|
||||
sort.Strings(bCopy)
|
||||
|
||||
if !reflect.DeepEqual(aCopy, bCopy) {
|
||||
t.Errorf("Test %v - %s FAIL: expected URL: %v, actual: %v", i, tc.name, tc.expected, actual)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user