mirror of
https://github.com/scsibug/nostr-rs-relay.git
synced 2024-11-09 21:29:06 -05:00
fix(NIP-01): allow limits on a per-filter basis
The original implementation of subscription limit applied to the entire query, instead of the specific filter. Now, each filter gets its own query limit. When a limit is applied, the most recent N events will be returned, otherwise the default is to return the earliest events (in order), for all matching events.
This commit is contained in:
parent
e894a86566
commit
1c14adc766
257
src/db.rs
257
src/db.rs
|
@ -7,6 +7,7 @@ use crate::hexrange::hex_range;
|
||||||
use crate::hexrange::HexSearch;
|
use crate::hexrange::HexSearch;
|
||||||
use crate::nip05;
|
use crate::nip05;
|
||||||
use crate::schema::{upgrade_db, STARTUP_SQL};
|
use crate::schema::{upgrade_db, STARTUP_SQL};
|
||||||
|
use crate::subscription::ReqFilter;
|
||||||
use crate::subscription::Subscription;
|
use crate::subscription::Subscription;
|
||||||
use crate::utils::is_hex;
|
use crate::utils::is_hex;
|
||||||
use governor::clock::Clock;
|
use governor::clock::Clock;
|
||||||
|
@ -377,152 +378,156 @@ fn repeat_vars(count: usize) -> String {
|
||||||
s
|
s
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a dynamic SQL query string and params from a subscription.
|
/// Create a dynamic SQL subquery and params from a subscription filter.
|
||||||
fn query_from_sub(sub: &Subscription) -> (String, Vec<Box<dyn ToSql>>) {
|
fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>) {
|
||||||
// build a dynamic SQL query. all user-input is either an integer
|
// build a dynamic SQL query. all user-input is either an integer
|
||||||
// (sqli-safe), or a string that is filtered to only contain
|
// (sqli-safe), or a string that is filtered to only contain
|
||||||
// hexadecimal characters. Strings that require escaping (tag
|
// hexadecimal characters. Strings that require escaping (tag
|
||||||
// names/values) use parameters.
|
// names/values) use parameters.
|
||||||
let mut limit: Option<u32> = None;
|
|
||||||
let mut query =
|
let mut query =
|
||||||
"SELECT DISTINCT(e.content) FROM event e LEFT JOIN tag t ON e.id=t.event_id ".to_owned();
|
"SELECT DISTINCT(e.content), e.created_at FROM event e LEFT JOIN tag t ON e.id=t.event_id "
|
||||||
// parameters
|
.to_owned();
|
||||||
|
// query parameters for SQLite
|
||||||
let mut params: Vec<Box<dyn ToSql>> = vec![];
|
let mut params: Vec<Box<dyn ToSql>> = vec![];
|
||||||
|
// individual filter components (single conditions such as an author or event ID)
|
||||||
// for every filter in the subscription, generate a where clause
|
let mut filter_components: Vec<String> = Vec::new();
|
||||||
let mut filter_clauses: Vec<String> = Vec::new();
|
// Query for "authors", allowing prefix matches
|
||||||
for f in sub.filters.iter() {
|
if let Some(authvec) = &f.authors {
|
||||||
// individual filter components
|
// take each author and convert to a hexsearch
|
||||||
let mut filter_components: Vec<String> = Vec::new();
|
let mut auth_searches: Vec<String> = vec![];
|
||||||
// Query for "authors", allowing prefix matches
|
for auth in authvec {
|
||||||
if let Some(authvec) = &f.authors {
|
match hex_range(auth) {
|
||||||
// take each author and convert to a hexsearch
|
Some(HexSearch::Exact(ex)) => {
|
||||||
let mut auth_searches: Vec<String> = vec![];
|
auth_searches.push("author=?".to_owned());
|
||||||
for auth in authvec {
|
params.push(Box::new(ex));
|
||||||
match hex_range(auth) {
|
}
|
||||||
Some(HexSearch::Exact(ex)) => {
|
Some(HexSearch::Range(lower, upper)) => {
|
||||||
auth_searches.push("author=?".to_owned());
|
auth_searches.push("(author>? AND author<?)".to_owned());
|
||||||
params.push(Box::new(ex));
|
params.push(Box::new(lower));
|
||||||
}
|
params.push(Box::new(upper));
|
||||||
Some(HexSearch::Range(lower, upper)) => {
|
}
|
||||||
auth_searches.push("(author>? AND author<?)".to_owned());
|
Some(HexSearch::LowerOnly(lower)) => {
|
||||||
params.push(Box::new(lower));
|
auth_searches.push("author>?".to_owned());
|
||||||
params.push(Box::new(upper));
|
params.push(Box::new(lower));
|
||||||
}
|
}
|
||||||
Some(HexSearch::LowerOnly(lower)) => {
|
None => {
|
||||||
auth_searches.push("author>?".to_owned());
|
info!("Could not parse hex range from author {:?}", auth);
|
||||||
params.push(Box::new(lower));
|
|
||||||
}
|
|
||||||
None => {
|
|
||||||
info!("Could not parse hex range from author {:?}", auth);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let authors_clause = format!("({})", auth_searches.join(" OR "));
|
|
||||||
filter_components.push(authors_clause);
|
|
||||||
}
|
}
|
||||||
if let Some(lim) = f.limit {
|
let authors_clause = format!("({})", auth_searches.join(" OR "));
|
||||||
limit = Some(lim)
|
filter_components.push(authors_clause);
|
||||||
}
|
}
|
||||||
// Query for Kind
|
// Query for Kind
|
||||||
if let Some(ks) = &f.kinds {
|
if let Some(ks) = &f.kinds {
|
||||||
// kind is number, no escaping needed
|
// kind is number, no escaping needed
|
||||||
let str_kinds: Vec<String> = ks.iter().map(|x| x.to_string()).collect();
|
let str_kinds: Vec<String> = ks.iter().map(|x| x.to_string()).collect();
|
||||||
let kind_clause = format!("kind IN ({})", str_kinds.join(", "));
|
let kind_clause = format!("kind IN ({})", str_kinds.join(", "));
|
||||||
filter_components.push(kind_clause);
|
filter_components.push(kind_clause);
|
||||||
}
|
}
|
||||||
// Query for event, allowing prefix matches
|
// Query for event, allowing prefix matches
|
||||||
if let Some(idvec) = &f.ids {
|
if let Some(idvec) = &f.ids {
|
||||||
// take each author and convert to a hexsearch
|
// take each author and convert to a hexsearch
|
||||||
let mut id_searches: Vec<String> = vec![];
|
let mut id_searches: Vec<String> = vec![];
|
||||||
for id in idvec {
|
for id in idvec {
|
||||||
match hex_range(id) {
|
match hex_range(id) {
|
||||||
Some(HexSearch::Exact(ex)) => {
|
Some(HexSearch::Exact(ex)) => {
|
||||||
id_searches.push("event_hash=?".to_owned());
|
id_searches.push("event_hash=?".to_owned());
|
||||||
params.push(Box::new(ex));
|
params.push(Box::new(ex));
|
||||||
}
|
}
|
||||||
Some(HexSearch::Range(lower, upper)) => {
|
Some(HexSearch::Range(lower, upper)) => {
|
||||||
id_searches.push("(event_hash>? AND event_hash<?)".to_owned());
|
id_searches.push("(event_hash>? AND event_hash<?)".to_owned());
|
||||||
params.push(Box::new(lower));
|
params.push(Box::new(lower));
|
||||||
params.push(Box::new(upper));
|
params.push(Box::new(upper));
|
||||||
}
|
}
|
||||||
Some(HexSearch::LowerOnly(lower)) => {
|
Some(HexSearch::LowerOnly(lower)) => {
|
||||||
id_searches.push("event_hash>?".to_owned());
|
id_searches.push("event_hash>?".to_owned());
|
||||||
params.push(Box::new(lower));
|
params.push(Box::new(lower));
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
info!("Could not parse hex range from id {:?}", id);
|
info!("Could not parse hex range from id {:?}", id);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let id_clause = format!("({})", id_searches.join(" OR "));
|
|
||||||
filter_components.push(id_clause);
|
|
||||||
}
|
}
|
||||||
// Query for tags
|
let id_clause = format!("({})", id_searches.join(" OR "));
|
||||||
if let Some(map) = &f.tags {
|
filter_components.push(id_clause);
|
||||||
for (key, val) in map.iter() {
|
}
|
||||||
let mut str_vals: Vec<Box<dyn ToSql>> = vec![];
|
// Query for tags
|
||||||
let mut blob_vals: Vec<Box<dyn ToSql>> = vec![];
|
if let Some(map) = &f.tags {
|
||||||
for v in val {
|
for (key, val) in map.iter() {
|
||||||
if is_hex(v) {
|
let mut str_vals: Vec<Box<dyn ToSql>> = vec![];
|
||||||
if let Ok(h) = hex::decode(&v) {
|
let mut blob_vals: Vec<Box<dyn ToSql>> = vec![];
|
||||||
blob_vals.push(Box::new(h));
|
for v in val {
|
||||||
}
|
if is_hex(v) {
|
||||||
} else {
|
if let Ok(h) = hex::decode(&v) {
|
||||||
str_vals.push(Box::new(v.to_owned()));
|
blob_vals.push(Box::new(h));
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
str_vals.push(Box::new(v.to_owned()));
|
||||||
}
|
}
|
||||||
// create clauses with "?" params for each tag value being searched
|
|
||||||
let str_clause = format!("value IN ({})", repeat_vars(str_vals.len()));
|
|
||||||
let blob_clause = format!("value_hex IN ({})", repeat_vars(blob_vals.len()));
|
|
||||||
let tag_clause = format!("(name=? AND ({} OR {}))", str_clause, blob_clause);
|
|
||||||
// add the tag name as the first parameter
|
|
||||||
params.push(Box::new(key.to_owned()));
|
|
||||||
// add all tag values that are plain strings as params
|
|
||||||
params.append(&mut str_vals);
|
|
||||||
// add all tag values that are blobs as params
|
|
||||||
params.append(&mut blob_vals);
|
|
||||||
filter_components.push(tag_clause);
|
|
||||||
}
|
}
|
||||||
}
|
// create clauses with "?" params for each tag value being searched
|
||||||
// Query for timestamp
|
let str_clause = format!("value IN ({})", repeat_vars(str_vals.len()));
|
||||||
if f.since.is_some() {
|
let blob_clause = format!("value_hex IN ({})", repeat_vars(blob_vals.len()));
|
||||||
let created_clause = format!("created_at > {}", f.since.unwrap());
|
let tag_clause = format!("(name=? AND ({} OR {}))", str_clause, blob_clause);
|
||||||
filter_components.push(created_clause);
|
// add the tag name as the first parameter
|
||||||
}
|
params.push(Box::new(key.to_owned()));
|
||||||
// Query for timestamp
|
// add all tag values that are plain strings as params
|
||||||
if f.until.is_some() {
|
params.append(&mut str_vals);
|
||||||
let until_clause = format!("created_at < {}", f.until.unwrap());
|
// add all tag values that are blobs as params
|
||||||
filter_components.push(until_clause);
|
params.append(&mut blob_vals);
|
||||||
}
|
filter_components.push(tag_clause);
|
||||||
|
|
||||||
// combine all clauses, and add to filter_clauses
|
|
||||||
if !filter_components.is_empty() {
|
|
||||||
let mut fc = "( ".to_owned();
|
|
||||||
fc.push_str(&filter_components.join(" AND "));
|
|
||||||
fc.push_str(" )");
|
|
||||||
filter_clauses.push(fc);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Query for timestamp
|
||||||
|
if f.since.is_some() {
|
||||||
|
let created_clause = format!("created_at > {}", f.since.unwrap());
|
||||||
|
filter_components.push(created_clause);
|
||||||
|
}
|
||||||
|
// Query for timestamp
|
||||||
|
if f.until.is_some() {
|
||||||
|
let until_clause = format!("created_at < {}", f.until.unwrap());
|
||||||
|
filter_components.push(until_clause);
|
||||||
|
}
|
||||||
// never display hidden events
|
// never display hidden events
|
||||||
query.push_str(" WHERE hidden!=TRUE ");
|
query.push_str(" WHERE hidden!=TRUE");
|
||||||
|
// build filter component conditions
|
||||||
|
if !filter_components.is_empty() {
|
||||||
|
query.push_str(" AND ");
|
||||||
|
query.push_str(&filter_components.join(" AND "));
|
||||||
|
}
|
||||||
|
// Apply per-filter limit to this subquery.
|
||||||
|
// The use of a LIMIT implies a DESC order, to capture only the most recent events.
|
||||||
|
if let Some(lim) = f.limit {
|
||||||
|
query.push_str(&format!(" ORDER BY e.created_at DESC LIMIT {}", lim))
|
||||||
|
} else {
|
||||||
|
query.push_str(" ORDER BY e.created_at ASC")
|
||||||
|
}
|
||||||
|
(query, params)
|
||||||
|
}
|
||||||
|
|
||||||
// combine all filters with OR clauses, if any exist
|
/// Create a dynamic SQL query string and params from a subscription.
|
||||||
if !filter_clauses.is_empty() {
|
fn query_from_sub(sub: &Subscription) -> (String, Vec<Box<dyn ToSql>>) {
|
||||||
query.push_str(" AND (");
|
// build a dynamic SQL query for an entire subscription, based on
|
||||||
query.push_str(&filter_clauses.join(" OR "));
|
// SQL subqueries for filters.
|
||||||
query.push_str(") ");
|
let mut subqueries: Vec<String> = Vec::new();
|
||||||
|
// subquery params
|
||||||
|
let mut params: Vec<Box<dyn ToSql>> = vec![];
|
||||||
|
// for every filter in the subscription, generate a subquery
|
||||||
|
for f in sub.filters.iter() {
|
||||||
|
let (f_subquery, mut f_params) = query_from_filter(&f);
|
||||||
|
subqueries.push(f_subquery);
|
||||||
|
params.append(&mut f_params);
|
||||||
}
|
}
|
||||||
// add order clause
|
// encapsulate subqueries into select statements
|
||||||
query.push_str(&format!(
|
let subqueries_selects: Vec<String> = subqueries
|
||||||
" ORDER BY created_at {}",
|
.iter()
|
||||||
limit.map_or("ASC", |_| "DESC")
|
.map(|s| {
|
||||||
));
|
return format!("SELECT content, created_at FROM ({})", s);
|
||||||
if let Some(lim) = limit {
|
})
|
||||||
query.push_str(&format!(" LIMIT {}", lim))
|
.collect();
|
||||||
}
|
let query: String = subqueries_selects.join(" UNION ");
|
||||||
debug!("query string: {}", query);
|
info!("final query string: {}", query);
|
||||||
(query, params)
|
(query, params)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -32,7 +32,7 @@ pub struct ReqFilter {
|
||||||
/// List of author public keys
|
/// List of author public keys
|
||||||
pub authors: Option<Vec<String>>,
|
pub authors: Option<Vec<String>>,
|
||||||
/// Limit number of results
|
/// Limit number of results
|
||||||
pub limit: Option<u32>,
|
pub limit: Option<u64>,
|
||||||
/// Set of tags
|
/// Set of tags
|
||||||
#[serde(skip)]
|
#[serde(skip)]
|
||||||
pub tags: Option<HashMap<String, HashSet<String>>>,
|
pub tags: Option<HashMap<String, HashSet<String>>>,
|
||||||
|
|
Loading…
Reference in New Issue
Block a user