diff --git a/internal/database/createFeed.sql.go b/internal/database/createFeed.sql.go index 6d40167..18e9f6f 100644 --- a/internal/database/createFeed.sql.go +++ b/internal/database/createFeed.sql.go @@ -22,7 +22,7 @@ VALUES ( $5, $6 ) -RETURNING id, created_at, updated_at, name, url, user_id +RETURNING id, created_at, updated_at, name, url, user_id, last_fetched_at ` type CreateFeedParams struct { @@ -51,6 +51,7 @@ func (q *Queries) CreateFeed(ctx context.Context, arg CreateFeedParams) (Feed, e &i.Name, &i.Url, &i.UserID, + &i.LastFetchedAt, ) return i, err } diff --git a/internal/database/getFeedByUrl.sql.go b/internal/database/getFeedByUrl.sql.go index 94e69fd..fe6c88a 100644 --- a/internal/database/getFeedByUrl.sql.go +++ b/internal/database/getFeedByUrl.sql.go @@ -10,7 +10,7 @@ import ( ) const getFeedByUrl = `-- name: GetFeedByUrl :one -SELECT id, created_at, updated_at, name, url, user_id FROM feeds WHERE url = $1 +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds WHERE url = $1 ` func (q *Queries) GetFeedByUrl(ctx context.Context, url string) (Feed, error) { @@ -23,6 +23,7 @@ func (q *Queries) GetFeedByUrl(ctx context.Context, url string) (Feed, error) { &i.Name, &i.Url, &i.UserID, + &i.LastFetchedAt, ) return i, err } diff --git a/internal/database/getNextFeedToFetch.sql.go b/internal/database/getNextFeedToFetch.sql.go new file mode 100644 index 0000000..49a0255 --- /dev/null +++ b/internal/database/getNextFeedToFetch.sql.go @@ -0,0 +1,29 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.28.0 +// source: getNextFeedToFetch.sql + +package database + +import ( + "context" +) + +const getNextFeedToFetch = `-- name: GetNextFeedToFetch :one +SELECT id, created_at, updated_at, name, url, user_id, last_fetched_at FROM feeds ORDER BY last_fetched_at NULLS FIRST LIMIT 1 +` + +func (q *Queries) GetNextFeedToFetch(ctx context.Context) (Feed, error) { + row := q.db.QueryRowContext(ctx, getNextFeedToFetch) + var i Feed + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Name, + &i.Url, + &i.UserID, + &i.LastFetchedAt, + ) + return i, err +} diff --git a/internal/database/markFeeedFetched.sql.go b/internal/database/markFeeedFetched.sql.go new file mode 100644 index 0000000..ed7343e --- /dev/null +++ b/internal/database/markFeeedFetched.sql.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.28.0 +// source: markFeeedFetched.sql + +package database + +import ( + "context" + + "github.com/google/uuid" +) + +const markFeedFetched = `-- name: MarkFeedFetched :one +UPDATE feeds SET last_fetched_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = $1 +RETURNING id, created_at, updated_at, name, url, user_id, last_fetched_at +` + +func (q *Queries) MarkFeedFetched(ctx context.Context, id uuid.UUID) (Feed, error) { + row := q.db.QueryRowContext(ctx, markFeedFetched, id) + var i Feed + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Name, + &i.Url, + &i.UserID, + &i.LastFetchedAt, + ) + return i, err +} diff --git a/internal/database/models.go b/internal/database/models.go index 40c75a3..68f3984 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -5,18 +5,20 @@ package database import ( + "database/sql" "time" "github.com/google/uuid" ) type Feed struct { - ID uuid.UUID - CreatedAt time.Time - UpdatedAt time.Time - Name string - Url string - UserID uuid.UUID + ID uuid.UUID + CreatedAt time.Time + UpdatedAt time.Time + Name string + Url string + UserID uuid.UUID + LastFetchedAt sql.NullTime } type FeedFollow struct { diff --git a/internal/handlers/fetchFeedHandler.go b/internal/handlers/fetchFeedHandler.go index 39a2d25..1136bc4 100644 --- a/internal/handlers/fetchFeedHandler.go +++ b/internal/handlers/fetchFeedHandler.go @@ -1,30 +1,28 @@ package handlers import ( - "context" - "fmt" "github.com/rdarius/boot-dev-blog-aggregator/internal/config" - "github.com/rdarius/boot-dev-blog-aggregator/internal/rss" - "os" + "log" + "time" ) func FetchFeedHandler(s *config.State, cmd config.Command) error { - var feedUrl string if len(cmd.Args) < 1 { - feedUrl = "https://www.wagslane.dev/index.xml" - } else { - feedUrl = cmd.Args[0] + log.Fatal("usage: boot-dev-blog-aggregator agg time_between_reqs") } - ctx := context.Background() - - feed, err := rss.FetchFeed(ctx, feedUrl) + t, err := time.ParseDuration(cmd.Args[0]) if err != nil { - fmt.Println("Failed to fetch feed") - os.Exit(1) + return err } - fmt.Printf("%v\n", feed) + ticker := time.NewTicker(t) + for ; ; <-ticker.C { + err := ScrapeFeedsHandler(s, cmd) + if err != nil { + return err + } + } return nil } diff --git a/internal/handlers/scrapeFeedsHandler.go b/internal/handlers/scrapeFeedsHandler.go new file mode 100644 index 0000000..85e0c1c --- /dev/null +++ b/internal/handlers/scrapeFeedsHandler.go @@ -0,0 +1,36 @@ +package handlers + +import ( + "context" + "fmt" + "github.com/rdarius/boot-dev-blog-aggregator/internal/config" + "github.com/rdarius/boot-dev-blog-aggregator/internal/rss" +) + +func ScrapeFeedsHandler(s *config.State, cmd config.Command) error { + + ctx := context.Background() + + nextFeed, err := s.DB.GetNextFeedToFetch(ctx) + if err != nil { + return err + } + + _, err = s.DB.MarkFeedFetched(ctx, nextFeed.ID) + if err != nil { + return err + } + + feedData, err := rss.FetchFeed(ctx, nextFeed.Url) + if err != nil { + return err + } + + fmt.Println("Feed Title: " + feedData.Channel.Title) + + for _, i := range feedData.Channel.Item { + fmt.Println("Item Title: " + i.Title) + } + + return nil +} diff --git a/sql/queries/getNextFeedToFetch.sql b/sql/queries/getNextFeedToFetch.sql new file mode 100644 index 0000000..0df9716 --- /dev/null +++ b/sql/queries/getNextFeedToFetch.sql @@ -0,0 +1,2 @@ +-- name: GetNextFeedToFetch :one +SELECT * FROM feeds ORDER BY last_fetched_at NULLS FIRST LIMIT 1; \ No newline at end of file diff --git a/sql/queries/markFeeedFetched.sql b/sql/queries/markFeeedFetched.sql new file mode 100644 index 0000000..67a8326 --- /dev/null +++ b/sql/queries/markFeeedFetched.sql @@ -0,0 +1,3 @@ +-- name: MarkFeedFetched :one +UPDATE feeds SET last_fetched_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE id = $1 +RETURNING *; \ No newline at end of file diff --git a/sql/schema/0004_last_fetched_field_in_feed.sql b/sql/schema/0004_last_fetched_field_in_feed.sql new file mode 100644 index 0000000..cb722bb --- /dev/null +++ b/sql/schema/0004_last_fetched_field_in_feed.sql @@ -0,0 +1,5 @@ +-- +goose Up +ALTER TABLE feeds ADD COLUMN last_fetched_at timestamp DEFAULT null; + +-- +goose Down +ALTER TABLE feeds DROP COLUMN last_fetched_at; \ No newline at end of file