nak req --paginate

README.md

@@ -173,6 +173,13 @@ listening at [wss://relay.damus.io wss://nos.lol wss://relay.nsecbunker.com]:
 {"kind":1,"id":"f030fccd90c783858dfcee204af94826cf0f1c85d6fc85a0087e9e5172419393","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1719677535,"tags":[["-"],["e","f59911b561c37c90b01e9e5c2557307380835c83399756f4d62d8167227e420a","wss://relay.whatever.com","root","a9e0f110f636f3191644110c19a33448daf09d7cda9708a769e91b7e91340208"],["p","a9e0f110f636f3191644110c19a33448daf09d7cda9708a769e91b7e91340208","wss://p-relay.com"]],"content":"I know the future","sig":"8b36a74e29df8bc12bed66896820da6940d4d9409721b3ed2e910c838833a178cb45fd5bb1c6eb6adc66ab2808bfac9f6644a2c55a6570bb2ad90f221c9c7551"}
 ```
 
+### download the latest 50000 notes from a relay, regardless of their natural query limits, by paginating requests
+```shell
+~> nak req -k 1 --limit 50000 --paginate --paginate-interval 2s nos.lol > events.jsonl
+~> wc -l events.jsonl
+50000 events.jsonl
+```
+
 ## contributing to this repository
 
 Use NIP-34 to send your patches to `naddr1qqpkucttqy28wumn8ghj7un9d3shjtnwdaehgu3wvfnsz9nhwden5te0wfjkccte9ehx7um5wghxyctwvsq3gamnwvaz7tmjv4kxz7fwv3sk6atn9e5k7q3q80cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwsxpqqqpmej2wctpn`.

paginate.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"context"
+	"math"
+	"slices"
+	"time"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+func paginateWithPoolAndParams(pool *nostr.SimplePool, interval time.Duration, globalLimit uint64) func(ctx context.Context, urls []string, filters nostr.Filters) chan nostr.IncomingEvent {
+	return func(ctx context.Context, urls []string, filters nostr.Filters) chan nostr.IncomingEvent {
+		// filters will always be just one
+		filter := filters[0]
+
+		nextUntil := nostr.Now()
+		if filter.Until != nil {
+			nextUntil = *filter.Until
+		}
+
+		if globalLimit == 0 {
+			globalLimit = uint64(filter.Limit)
+			if globalLimit == 0 && !filter.LimitZero {
+				globalLimit = math.MaxUint64
+			}
+		}
+		var globalCount uint64 = 0
+		globalCh := make(chan nostr.IncomingEvent)
+
+		repeatedCache := make([]string, 0, 300)
+		nextRepeatedCache := make([]string, 0, 300)
+
+		go func() {
+			defer close(globalCh)
+
+			for {
+				filter.Until = &nextUntil
+				time.Sleep(interval)
+
+				keepGoing := false
+				for evt := range pool.SubManyEose(ctx, urls, nostr.Filters{filter}) {
+					if slices.Contains(repeatedCache, evt.ID) {
+						continue
+					}
+
+					keepGoing = true // if we get one that isn't repeated, then keep trying to get more
+					nextRepeatedCache = append(nextRepeatedCache, evt.ID)
+
+					globalCh <- evt
+
+					globalCount++
+					if globalCount >= globalLimit {
+						return
+					}
+
+					if evt.CreatedAt < *filter.Until {
+						nextUntil = evt.CreatedAt
+					}
+				}
+
+				if !keepGoing {
+					return
+				}
+
+				repeatedCache = nextRepeatedCache
+				nextRepeatedCache = nextRepeatedCache[:0]
+			}
+		}()
+
+		return globalCh
+	}
+}

req.go

@@ -96,6 +96,20 @@ example:
 			Usage:       "keep the subscription open, printing all events as they are returned",
 			DefaultText: "false, will close on EOSE",
 		},
+		&cli.BoolFlag{
+			Name:        "paginate",
+			Usage:       "make multiple REQs to the relay decreasing the value of 'until' until 'limit' or 'since' conditions are met",
+			DefaultText: "false",
+		},
+		&cli.DurationFlag{
+			Name:  "paginate-interval",
+			Usage: "time between queries when using --paginate",
+		},
+		&cli.UintFlag{
+			Name:        "paginate-global-limit",
+			Usage:       "global limit at which --paginate should stop",
+			DefaultText: "uses the value given by --limit/-l or infinite",
+		},
 		&cli.BoolFlag{
 			Name:  "bare",
 			Usage: "when printing the filter, print just the filter, not enveloped in a [\"REQ\", ...] array",
@@ -247,7 +261,9 @@ example:
 
 			if len(relayUrls) > 0 {
 				fn := pool.SubManyEose
-				if c.Bool("stream") {
+				if c.Bool("paginate") {
+					fn = paginateWithPoolAndParams(pool, c.Duration("paginate-interval"), c.Uint("paginate-global-limit"))
+				} else if c.Bool("stream") {
 					fn = pool.SubMany
 				}