summaryrefslogtreecommitdiffstats
path: root/feed/feed.go
diff options
context:
space:
mode:
authorsiddharth <s@ricketyspace.net>2022-04-17 02:41:16 -0400
committersiddharth <s@ricketyspace.net>2022-04-17 02:41:16 -0400
commita0ebdaf4d3bc0702a8534f1a46afa9a178099583 (patch)
treec8d48e3c575e66ccf9ad0f37672d7b2acd3f9dc1 /feed/feed.go
parent43d7efcd76ae55e905d2aa158956101c19672aa6 (diff)
feed: update Process
Add capability to filter entries based on title.
Diffstat (limited to 'feed/feed.go')
-rw-r--r--feed/feed.go33
1 files changed, 23 insertions, 10 deletions
diff --git a/feed/feed.go b/feed/feed.go
index 25c5750..a3b125c 100644
--- a/feed/feed.go
+++ b/feed/feed.go
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: ISC
-// Copyright © 2021 siddharth <s@ricketyspace.net>
+// Copyright © 2022 siddharth <s@ricketyspace.net>
package feed
@@ -18,13 +18,14 @@ import (
)
type Feed struct {
- Id string `json:"id"`
- Source string `json:"source"`
- Schema string `json:"schema"`
- Last int `json:"last"`
- YDLPath string
- DumpDir string
- Entries []schema.Entry
+ Id string
+ Source string
+ Schema string
+ Last int
+ TitleContains string `json:"title-contains"`
+ YDLPath string
+ DumpDir string
+ Entries []schema.Entry
}
func (feed *Feed) Validate(baseDumpDir string) error {
@@ -124,10 +125,21 @@ func (feed *Feed) Process(pState *state.ProcessState) {
// Number entries being processed.
errors := 0
processing := 0
+ traversed := 0
// Channel for receiving entry results.
erChan := make(chan state.EntryResult)
- for i, entry := range feed.Entries {
+ for _, entry := range feed.Entries {
e := entry
+
+ // Ignore entry if its title does not matches
+ // feed's 'title-contains' string.
+ if len(feed.TitleContains) > 0 &&
+ !e.TitleContains(feed.TitleContains) {
+ fmt.Printf("[%s][%s]: Skipping '%s'\n",
+ feed.Id, e.Id, e.Title)
+ continue
+ }
+
// Process entry only if it was not downloaded before.
if !pState.DB.Exists(feed.Id, e.Id) {
go feed.processEntry(e, erChan)
@@ -136,9 +148,10 @@ func (feed *Feed) Process(pState *state.ProcessState) {
fmt.Printf("[%s][%s]: Already downloaded '%s' before\n",
feed.Id, e.Id, e.Title)
}
+ traversed += 1
// Process only `feed.Last` entries.
- if i >= feed.Last-1 {
+ if traversed >= feed.Last-1 {
break
}
}