this is all wrong and barely working... no parallel running since... it messed things up

.gitignore

@@ -0,0 +1 @@
+node_modules/

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/boot-dev-typescript-web-scraper.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/dictionaries/project.xml

@@ -0,0 +1,7 @@
+<component name="ProjectDictionaryState">
+  <dictionary name="project">
+    <words>
+      <w>rdarius</w>
+    </words>
+  </dictionary>
+</component>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/boot-dev-typescript-web-scraper.iml" filepath="$PROJECT_DIR$/.idea/boot-dev-typescript-web-scraper.iml" />
+    </modules>
+  </component>
+</project>

.idea/php.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="MessDetectorOptionsConfiguration">
+    <option name="transferred" value="true" />
+  </component>
+  <component name="PHPCSFixerOptionsConfiguration">
+    <option name="transferred" value="true" />
+  </component>
+  <component name="PHPCodeSnifferOptionsConfiguration">
+    <option name="highlightLevel" value="WARNING" />
+    <option name="transferred" value="true" />
+  </component>
+  <component name="PhpStanOptionsConfiguration">
+    <option name="transferred" value="true" />
+  </component>
+  <component name="PsalmOptionsConfiguration">
+    <option name="transferred" value="true" />
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

package.json

@@ -0,0 +1,32 @@
+{
+  "name": "boot-dev-typescript-web-scraper",
+  "version": "1.0.0",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "start": "tsx ./src/index.ts",
+    "test": "vitest run"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/rdarius/boot-dev-typescript-web-scraper.git"
+  },
+  "author": "",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/rdarius/boot-dev-typescript-web-scraper/issues"
+  },
+  "homepage": "https://github.com/rdarius/boot-dev-typescript-web-scraper#readme",
+  "description": "",
+  "devDependencies": {
+    "@types/jsdom": "^21.1.7",
+    "@types/node": "^24.3.3",
+    "tsx": "^4.20.5",
+    "typescript": "^5.9.2",
+    "vitest": "^3.2.4"
+  },
+  "dependencies": {
+    "jsdom": "^27.0.0",
+    "p-limit": "^7.1.1"
+  }
+}

src/crawl.test.ts

@@ -0,0 +1,152 @@
+import { expect, test } from 'vitest'
+import {
+    extractPageData,
+    getFirstParagraphFromHTML,
+    getH1FromHTML,
+    getImagesFromHTML,
+    getURLsFromHTML,
+    normalizeURL
+} from "./crawl";
+
+test('adds 1 + 2 to equal 3', () => {
+    expect(normalizeURL('https://rdarius.com/')).toBe('rdarius.com');
+    expect(normalizeURL('http://rdarius.com/')).toBe('rdarius.com');
+    expect(normalizeURL('https://rdarius.com/some/web/')).toBe('rdarius.com/some/web');
+    expect(normalizeURL('http://rdarius.com/some/web/')).toBe('rdarius.com/some/web');
+    expect(normalizeURL('https://rdarius.com/?id=1')).toBe('rdarius.com/?id=1');
+    expect(normalizeURL('https://rdarius.com/some-page?id=1')).toBe('rdarius.com/some-page?id=1');
+    expect(normalizeURL('https://rdarius.com/some-page/?id=1')).toBe('rdarius.com/some-page?id=1');
+    expect(normalizeURL('https://rdarius.com/some/web/#some-comment')).toBe('rdarius.com/some/web');
+    expect(normalizeURL('https://rdarius.com/some-page?id=1#some-comment')).toBe('rdarius.com/some-page?id=1');
+})
+
+test("getH1FromHTML basic", () => {
+    const inputBody = `<html><body><h1>Test Title</h1></body></html>`;
+    const actual = getH1FromHTML(inputBody);
+    const expected = "Test Title";
+    expect(actual).toEqual(expected);
+});
+
+test("getFirstParagraphFromHTML main priority", () => {
+    const inputBody = `
+    <html><body>
+      <p>Outside paragraph.</p>
+      <main>
+        <p>Main paragraph.</p>
+      </main>
+    </body></html>
+  `;
+    const actual = getFirstParagraphFromHTML(inputBody);
+    const expected = "Main paragraph.";
+    expect(actual).toEqual(expected);
+});
+
+test("getURLsFromHTML absolute", () => {
+    const inputURL = "https://blog.boot.dev";
+    const inputBody = `<html><body><a href="https://blog.boot.dev"><span>Boot.dev</span></a></body></html>`;
+
+    const actual = getURLsFromHTML(inputBody, inputURL);
+    const expected = ["https://blog.boot.dev"];
+
+    expect(actual).toEqual(expected);
+});
+
+test("getURLsFromHTML relative", () => {
+    const inputURL = "https://blog.boot.dev";
+    const inputBody = `<html><body><a href="/posts"><span>Boot.dev</span></a></body></html>`;
+
+    const actual = getURLsFromHTML(inputBody, inputURL);
+    const expected = ["https://blog.boot.dev/posts"];
+
+    expect(actual).toEqual(expected);
+});
+
+test("getImagesFromHTML absolute", () => {
+    const inputURL = "https://blog.boot.dev";
+    const inputBody = `<html><body><img src="https://blog.boot.dev/logo.png" alt="Logo"></body></html>`;
+
+    const actual = getImagesFromHTML(inputBody, inputURL);
+    const expected = ["https://blog.boot.dev/logo.png"];
+
+    expect(actual).toEqual(expected);
+});
+
+test("getImagesFromHTML relative", () => {
+    const inputURL = "https://blog.boot.dev";
+    const inputBody = `<html><body><img src="/logo.png" alt="Logo"></body></html>`;
+
+    const actual = getImagesFromHTML(inputBody, inputURL);
+    const expected = ["https://blog.boot.dev/logo.png"];
+
+    expect(actual).toEqual(expected);
+});
+
+test("extractPageData basic", () => {
+    const inputURL = "https://blog.boot.dev";
+    const inputBody = `
+    <html><body>
+      <h1>Test Title</h1>
+      <p>This is the first paragraph.</p>
+      <a href="/link1">Link 1</a>
+      <img src="/image1.jpg" alt="Image 1">
+    </body></html>
+  `;
+
+    const actual = extractPageData(inputBody, inputURL);
+    const expected = {
+        url: "https://blog.boot.dev",
+        h1: "Test Title",
+        first_paragraph: "This is the first paragraph.",
+        outgoing_links: ["https://blog.boot.dev/link1"],
+        image_urls: ["https://blog.boot.dev/image1.jpg"],
+    };
+
+    expect(actual).toEqual(expected);
+});
+
+test("extractPageData with multiple links and images", () => {
+    const inputURL = "https://example.com";
+    const inputBody = `
+    <html><body>
+      <h1>Another Title</h1>
+      <main><p>Main content paragraph.</p></main>
+      <a href="https://example.com/about">About</a>
+      <a href="/contact">Contact</a>
+      <img src="https://example.com/img/logo.png">
+      <img src="/banner.png">
+    </body></html>
+  `;
+
+    const actual = extractPageData(inputBody, inputURL);
+    const expected = {
+        url: "https://example.com",
+        h1: "Another Title",
+        first_paragraph: "Main content paragraph.",
+        outgoing_links: [
+            "https://example.com/about",
+            "https://example.com/contact",
+        ],
+        image_urls: [
+            "https://example.com/img/logo.png",
+            "https://example.com/banner.png",
+        ],
+    };
+
+    expect(actual).toEqual(expected);
+});
+
+test("extractPageData empty elements", () => {
+    const inputURL = "https://empty.org";
+    const inputBody = `<html><body><div>No useful content</div></body></html>`;
+
+    const actual = extractPageData(inputBody, inputURL);
+    const expected = {
+        url: "https://empty.org",
+        h1: "",
+        first_paragraph: "",
+        outgoing_links: [],
+        image_urls: [],
+    };
+
+    expect(actual).toEqual(expected);
+});

src/crawl.ts

@@ -0,0 +1,246 @@
+import {JSDOM} from 'jsdom'
+import pLimit from "p-limit";
+
+export interface ExtractedPageData {
+    url: string;
+    h1: string;
+    first_paragraph: string;
+    outgoing_links: string[];
+    image_urls: string[];
+}
+
+function removeTrailingSlash(url: string): string {
+    return url.endsWith("/") && url.indexOf("/", 9) === url.length - 1
+        ? url.slice(0, -1)
+        : url;
+}
+
+export function normalizeURL(url: string): string {
+    const parsed = new URL(url);
+    let final = parsed.hostname;
+    let path = parsed.pathname.split("/").filter((x) => x);
+    if (path.length > 0) {
+        final += "/" + path.join("/");
+    }
+    if (parsed.search) {
+        final += path.length > 0 ? parsed.search : "/" + parsed.search;
+    }
+    return final;
+}
+
+export function getH1FromHTML(html: string): string {
+    const dom = new JSDOM(html);
+    const h1 = dom.window.document.body.querySelector('h1');
+    if (h1) {
+        return h1.innerHTML;
+    }
+    return '';
+}
+
+export function getFirstParagraphFromHTML(html: string): string {
+    const dom = new JSDOM(html);
+    const p = dom.window.document.body.querySelector('main p');
+    if (p) {
+        return p.innerHTML;
+    } else {
+        const secondaryP = dom.window.document.body.querySelector('p');
+        if (secondaryP) {
+            return secondaryP.innerHTML;
+        }
+    }
+    return '';
+}
+
+export function getURLsFromHTML(html: string, baseURL: string): string[] {
+    const urls: string[] = [];
+    const dom = new JSDOM(html);
+    const a = dom.window.document.querySelectorAll("a");
+
+    for (const link of a) {
+        const href = link.getAttribute("href");
+        if (!href) continue;
+        try {
+            const urlObj = new URL(href, baseURL);
+            urls.push(removeTrailingSlash(urlObj.toString()));
+        } catch {
+            console.error(`Invalid URL found: ${href}`);
+        }
+    }
+    return urls;
+}
+
+export function getImagesFromHTML(html: string, baseURL: string): string[] {
+    const urls = [];
+    const dom = new JSDOM(html);
+    const a = dom.window.document.body.querySelectorAll('img');
+
+    for (const link of a) {
+        const src = link.getAttribute("src");
+        if (!src) continue;
+        try {
+            const urlObj = new URL(src, baseURL);
+            urls.push(removeTrailingSlash(urlObj.toString()));
+        } catch {
+            console.error(`Invalid Image URL found: ${src}`);
+        }
+    }
+
+    return urls;
+}
+
+export function extractPageData(html: string, pageURL: string): ExtractedPageData {
+    return {
+        url: pageURL,
+        h1: getH1FromHTML(html),
+        first_paragraph: getFirstParagraphFromHTML(html),
+        outgoing_links: getURLsFromHTML(html, pageURL),
+        image_urls: getImagesFromHTML(html, pageURL),
+    };
+}
+
+export class ConcurrentCrawler {
+    private readonly baseURL: string;
+    private readonly pages: Record<string, ExtractedPageData>;
+    private readonly limit: <T>(fn: () => Promise<T>) => Promise<T>;
+    private readonly maxPages: number;
+    private shouldStop: boolean;
+    private readonly allTasks: Set<Promise<void>>;
+    private abortController: AbortController;
+    private readonly toCrawl: string[];
+
+    constructor(
+        baseURL: string,
+        maxConcurrency: number = 5,
+        maxPages: number = 50
+    ) {
+        this.baseURL = baseURL;
+        this.pages = {};
+        this.limit = pLimit(maxConcurrency);
+        this.maxPages = maxPages;
+        this.shouldStop = false;
+        this.allTasks = new Set();
+        this.abortController = new AbortController();
+        this.toCrawl = [];
+    }
+
+    private addPageVisit(
+        normalizedURL: string,
+        data: ExtractedPageData
+    ): boolean {
+        if (this.shouldStop) return false;
+
+        if (this.pages[normalizedURL]) {
+            // already crawled
+            return false;
+        }
+
+        this.pages[normalizedURL] = data;
+
+        if (Object.keys(this.pages).length >= this.maxPages) {
+            console.log("⚠️ Reached maximum number of pages to crawl.");
+            this.shouldStop = true;
+            this.abortController.abort();
+            return false;
+        }
+
+        return true;
+    }
+
+    private async getHTML(currentURL: string): Promise<string | undefined> {
+        return this.limit(async () => {
+            if (this.shouldStop) return;
+            try {
+                const res = await fetch(currentURL, {
+                    headers: { "User-Agent": "BootCrawler/1.0" },
+                    signal: this.abortController.signal,
+                });
+                if (res.status >= 400) {
+                    console.error(`Error: HTTP ${res.status} for ${currentURL}`);
+                    return;
+                }
+                const contentType = res.headers.get("content-type");
+                if (!contentType || !contentType.includes("text/html")) {
+                    console.error(
+                        `Error: Non-HTML content-type (${contentType}) for ${currentURL}`
+                    );
+                    return;
+                }
+                return await res.text();
+            } catch (err: any) {
+                if (err.name === "AbortError") {
+                    console.warn(`Fetch aborted for ${currentURL}`);
+                } else {
+                    console.error(`Error fetching ${currentURL}:`, err);
+                }
+                return;
+            }
+        });
+    }
+
+    private async crawlPage(): Promise<void> {
+        let currentURL = '';
+        if (this.toCrawl.length > 0) {
+            const currentURLTemp = this.toCrawl.shift();
+            if (!currentURLTemp) return;
+            currentURL = currentURLTemp;
+        }
+
+        if (this.shouldStop) return;
+
+        const base = new URL(this.baseURL);
+        let current: URL;
+        try {
+            current = new URL(currentURL);
+        } catch {
+            return;
+        }
+
+        if (current.hostname !== base.hostname) return;
+
+        const normalized = normalizeURL(current.href);
+        const html = await this.getHTML(current.href);
+        if (!html) return;
+
+        const data = extractPageData(html, current.href);
+        if (!this.addPageVisit(normalized, data)) return;
+
+        console.log(`🌐 Crawled: ${current.href}`);
+
+        for (const url of data.outgoing_links) {
+            if (this.shouldStop) break;
+            //this.crawlPage(url)
+            this.schedule(url);
+        }
+    }
+
+    private schedule(url: string) {
+        // const tracked = promise.finally(() => this.allTasks.delete(tracked));
+        // this.allTasks.add(tracked);
+        this.toCrawl.push(url);
+    }
+
+    public async crawl(): Promise<Record<string, ExtractedPageData>> {
+        this.schedule(this.baseURL);
+
+        do {
+            await this.crawlPage()
+        } while (this.toCrawl.length > 0);
+
+        // while (this.allTasks.size > 0) {
+        //     console.log("Active tasks:", this.allTasks.size);
+        //     await Promise.race(this.allTasks);
+        // }
+        // console.log(this.pages); // ✅ will now run
+
+        return this.pages;
+    }
+}
+
+export async function crawlSiteAsync(
+    baseURL: string,
+    maxConcurrency: number,
+    maxPages: number
+): Promise<Record<string, ExtractedPageData>> {
+    const crawler = new ConcurrentCrawler(baseURL, maxConcurrency, maxPages);
+    return await crawler.crawl();
+}

src/index.ts

@@ -0,0 +1,44 @@
+import {crawlSiteAsync, ExtractedPageData} from "./crawl";
+import {writeCSVReport} from "./report";
+
+
+async function main() {
+    const args = process.argv.slice(2);
+
+    if (args.length < 1) {
+        console.error("Usage: npm run start <URL> <maxConcurrency> <maxPages>");
+        process.exit(1);
+    }
+    if (args.length > 3) {
+        console.error("Error: Too many arguments.");
+        process.exit(1);
+    }
+
+    const baseURL = args[0];
+    const maxConcurrency = args[1] ? parseInt(args[1]) : 3;
+    const maxPages = args[2] ? parseInt(args[2]) : 50;
+
+    console.log(`🚀 Starting crawl at ${baseURL} (concurrency=${maxConcurrency}, maxPages=${maxPages})`);
+
+    let pageData: Record<string, ExtractedPageData> = {};
+    writeCSVReport(pageData, "report.csv");
+
+    try {
+        pageData = await crawlSiteAsync(baseURL, maxConcurrency, maxPages);
+        // console.log(pageData);
+    } catch (err) {
+        console.error("❌ Crawler failed:", err);
+    } finally {
+        try {
+            if (Object.keys(pageData).length > 0) {
+                writeCSVReport(pageData, "report.csv");
+            } else {
+                console.warn("⚠️ No pages crawled, skipping report generation.");
+            }
+        } catch (err) {
+            console.error("❌ Failed to write report:", err);
+        }
+    }
+}
+
+main().finally();

src/report.ts

@@ -0,0 +1,42 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { ExtractedPageData } from "./crawl";
+
+function csvEscape(field: string): string {
+    const str = field ?? "";
+    const needsQuoting = /[",\n]/.test(str);
+    const escaped = str.replace(/"/g, '""');
+    return needsQuoting ? `"${escaped}"` : escaped;
+}
+
+export function writeCSVReport(
+    pageData: Record<string, ExtractedPageData>,
+    filename = "report.csv"
+): void {
+    const filepath = path.resolve(process.cwd(), filename);
+
+    const headers = [
+        "page_url",
+        "h1",
+        "first_paragraph",
+        "outgoing_link_urls",
+        "image_urls",
+    ];
+
+    const rows: string[] = [headers.join(",")];
+
+    for (const page of Object.values(pageData)) {
+        const row = [
+            csvEscape(page.url),
+            csvEscape(page.h1),
+            csvEscape(page.first_paragraph),
+            csvEscape(page.outgoing_links.join(";")),
+            csvEscape(page.image_urls.join(";")),
+        ].join(",");
+        rows.push(row);
+    }
+
+    fs.writeFileSync(filepath, rows.join("\n"), "utf-8");
+
+    console.log(`✅ Report written to ${filepath}`);
+}

tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "esnext",
+    "strict": true,
+    "moduleResolution": "Node"
+  },
+  "include": ["./src/**/*.ts"],
+  "exclude": ["node_modules"]
+}