diff --git a/README.md b/README.md index 245531b..8884642 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,13 @@ listening at [wss://relay.damus.io wss://nos.lol wss://relay.nsecbunker.com]: {"kind":1,"id":"f030fccd90c783858dfcee204af94826cf0f1c85d6fc85a0087e9e5172419393","pubkey":"79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798","created_at":1719677535,"tags":[["-"],["e","f59911b561c37c90b01e9e5c2557307380835c83399756f4d62d8167227e420a","wss://relay.whatever.com","root","a9e0f110f636f3191644110c19a33448daf09d7cda9708a769e91b7e91340208"],["p","a9e0f110f636f3191644110c19a33448daf09d7cda9708a769e91b7e91340208","wss://p-relay.com"]],"content":"I know the future","sig":"8b36a74e29df8bc12bed66896820da6940d4d9409721b3ed2e910c838833a178cb45fd5bb1c6eb6adc66ab2808bfac9f6644a2c55a6570bb2ad90f221c9c7551"} ``` +### download the latest 50000 notes from a relay, regardless of their natural query limits, by paginating requests +```shell +~> nak req -k 1 --limit 50000 --paginate --paginate-interval 2s nos.lol > events.jsonl +~> wc -l events.jsonl +50000 events.jsonl +``` + ## contributing to this repository Use NIP-34 to send your patches to `naddr1qqpkucttqy28wumn8ghj7un9d3shjtnwdaehgu3wvfnsz9nhwden5te0wfjkccte9ehx7um5wghxyctwvsq3gamnwvaz7tmjv4kxz7fwv3sk6atn9e5k7q3q80cvv07tjdrrgpa0j7j7tmnyl2yr6yr7l8j4s3evf6u64th6gkwsxpqqqpmej2wctpn`. diff --git a/paginate.go b/paginate.go new file mode 100644 index 0000000..f8babd8 --- /dev/null +++ b/paginate.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "math" + "slices" + "time" + + "github.com/nbd-wtf/go-nostr" +) + +func paginateWithPoolAndParams(pool *nostr.SimplePool, interval time.Duration, globalLimit uint64) func(ctx context.Context, urls []string, filters nostr.Filters) chan nostr.IncomingEvent { + return func(ctx context.Context, urls []string, filters nostr.Filters) chan nostr.IncomingEvent { + // filters will always be just one + filter := filters[0] + + nextUntil := nostr.Now() + if filter.Until != nil { + nextUntil = *filter.Until + } + + if globalLimit == 0 { + globalLimit = uint64(filter.Limit) + if globalLimit == 0 && !filter.LimitZero { + globalLimit = math.MaxUint64 + } + } + var globalCount uint64 = 0 + globalCh := make(chan nostr.IncomingEvent) + + repeatedCache := make([]string, 0, 300) + nextRepeatedCache := make([]string, 0, 300) + + go func() { + defer close(globalCh) + + for { + filter.Until = &nextUntil + time.Sleep(interval) + + keepGoing := false + for evt := range pool.SubManyEose(ctx, urls, nostr.Filters{filter}) { + if slices.Contains(repeatedCache, evt.ID) { + continue + } + + keepGoing = true // if we get one that isn't repeated, then keep trying to get more + nextRepeatedCache = append(nextRepeatedCache, evt.ID) + + globalCh <- evt + + globalCount++ + if globalCount >= globalLimit { + return + } + + if evt.CreatedAt < *filter.Until { + nextUntil = evt.CreatedAt + } + } + + if !keepGoing { + return + } + + repeatedCache = nextRepeatedCache + nextRepeatedCache = nextRepeatedCache[:0] + } + }() + + return globalCh + } +} diff --git a/req.go b/req.go index 10713a8..627ec05 100644 --- a/req.go +++ b/req.go @@ -96,6 +96,20 @@ example: Usage: "keep the subscription open, printing all events as they are returned", DefaultText: "false, will close on EOSE", }, + &cli.BoolFlag{ + Name: "paginate", + Usage: "make multiple REQs to the relay decreasing the value of 'until' until 'limit' or 'since' conditions are met", + DefaultText: "false", + }, + &cli.DurationFlag{ + Name: "paginate-interval", + Usage: "time between queries when using --paginate", + }, + &cli.UintFlag{ + Name: "paginate-global-limit", + Usage: "global limit at which --paginate should stop", + DefaultText: "uses the value given by --limit/-l or infinite", + }, &cli.BoolFlag{ Name: "bare", Usage: "when printing the filter, print just the filter, not enveloped in a [\"REQ\", ...] array", @@ -247,7 +261,9 @@ example: if len(relayUrls) > 0 { fn := pool.SubManyEose - if c.Bool("stream") { + if c.Bool("paginate") { + fn = paginateWithPoolAndParams(pool, c.Duration("paginate-interval"), c.Uint("paginate-global-limit")) + } else if c.Bool("stream") { fn = pool.SubMany }