nostr/nostr-rs-relay
1
0
Fork 0
mirror of https://github.com/scsibug/nostr-rs-relay.git synced 2024-11-22 00:59:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix(NIP-01): allow limits on a per-filter basis

Browse Source 
The original implementation of subscription limit applied to the
entire query, instead of the specific filter.  Now, each filter gets
its own query limit.  When a limit is applied, the most recent N
events will be returned, otherwise the default is to return the
earliest events (in order), for all matching events.
This commit is contained in:
Greg Heartsfield 2022-07-04 17:25:32 -05:00
parent e894a86566
commit 1c14adc766
2 changed files with 132 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

257
src/db.rs
View File

@ -7,6 +7,7 @@ use crate::hexrange::hex_range;
use crate::hexrange::HexSearch; use crate::hexrange::HexSearch;
use crate::nip05; use crate::nip05;
use crate::schema::{upgrade_db, STARTUP_SQL}; use crate::schema::{upgrade_db, STARTUP_SQL};
use crate::subscription::ReqFilter;
use crate::subscription::Subscription; use crate::subscription::Subscription;
use crate::utils::is_hex; use crate::utils::is_hex;
use governor::clock::Clock; use governor::clock::Clock;
@ -377,152 +378,156 @@ fn repeat_vars(count: usize) -> String {
s s
} }
/// Create a dynamic SQL query string and params from a subscription. /// Create a dynamic SQL subquery and params from a subscription filter.
fn query_from_sub(sub: &Subscription) -> (String, Vec<Box<dyn ToSql>>) { fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>) {
// build a dynamic SQL query. all user-input is either an integer // build a dynamic SQL query. all user-input is either an integer
// (sqli-safe), or a string that is filtered to only contain // (sqli-safe), or a string that is filtered to only contain
// hexadecimal characters. Strings that require escaping (tag // hexadecimal characters. Strings that require escaping (tag
// names/values) use parameters. // names/values) use parameters.
let mut limit: Option<u32> = None;
let mut query = let mut query =
"SELECT DISTINCT(e.content) FROM event e LEFT JOIN tag t ON e.id=t.event_id ".to_owned(); "SELECT DISTINCT(e.content), e.created_at FROM event e LEFT JOIN tag t ON e.id=t.event_id "
// parameters .to_owned();
// query parameters for SQLite
let mut params: Vec<Box<dyn ToSql>> = vec![]; let mut params: Vec<Box<dyn ToSql>> = vec![];
// individual filter components (single conditions such as an author or event ID)
// for every filter in the subscription, generate a where clause let mut filter_components: Vec<String> = Vec::new();
let mut filter_clauses: Vec<String> = Vec::new(); // Query for "authors", allowing prefix matches
for f in sub.filters.iter() { if let Some(authvec) = &f.authors {
// individual filter components // take each author and convert to a hexsearch
let mut filter_components: Vec<String> = Vec::new(); let mut auth_searches: Vec<String> = vec![];
// Query for "authors", allowing prefix matches for auth in authvec {
if let Some(authvec) = &f.authors { match hex_range(auth) {
// take each author and convert to a hexsearch Some(HexSearch::Exact(ex)) => {
let mut auth_searches: Vec<String> = vec![]; auth_searches.push("author=?".to_owned());
for auth in authvec { params.push(Box::new(ex));
match hex_range(auth) { }
Some(HexSearch::Exact(ex)) => { Some(HexSearch::Range(lower, upper)) => {
auth_searches.push("author=?".to_owned()); auth_searches.push("(author>? AND author<?)".to_owned());
params.push(Box::new(ex)); params.push(Box::new(lower));
} params.push(Box::new(upper));
Some(HexSearch::Range(lower, upper)) => { }
auth_searches.push("(author>? AND author<?)".to_owned()); Some(HexSearch::LowerOnly(lower)) => {
params.push(Box::new(lower)); auth_searches.push("author>?".to_owned());
params.push(Box::new(upper)); params.push(Box::new(lower));
} }
Some(HexSearch::LowerOnly(lower)) => { None => {
auth_searches.push("author>?".to_owned()); info!("Could not parse hex range from author {:?}", auth);
params.push(Box::new(lower));
}
None => {
info!("Could not parse hex range from author {:?}", auth);
}
} }
} }
let authors_clause = format!("({})", auth_searches.join(" OR "));
filter_components.push(authors_clause);
} }
if let Some(lim) = f.limit { let authors_clause = format!("({})", auth_searches.join(" OR "));
limit = Some(lim) filter_components.push(authors_clause);
} }
// Query for Kind // Query for Kind
if let Some(ks) = &f.kinds { if let Some(ks) = &f.kinds {
// kind is number, no escaping needed // kind is number, no escaping needed
let str_kinds: Vec<String> = ks.iter().map(|x| x.to_string()).collect(); let str_kinds: Vec<String> = ks.iter().map(|x| x.to_string()).collect();
let kind_clause = format!("kind IN ({})", str_kinds.join(", ")); let kind_clause = format!("kind IN ({})", str_kinds.join(", "));
filter_components.push(kind_clause); filter_components.push(kind_clause);
} }
// Query for event, allowing prefix matches // Query for event, allowing prefix matches
if let Some(idvec) = &f.ids { if let Some(idvec) = &f.ids {
// take each author and convert to a hexsearch // take each author and convert to a hexsearch
let mut id_searches: Vec<String> = vec![]; let mut id_searches: Vec<String> = vec![];
for id in idvec { for id in idvec {
match hex_range(id) { match hex_range(id) {
Some(HexSearch::Exact(ex)) => { Some(HexSearch::Exact(ex)) => {
id_searches.push("event_hash=?".to_owned()); id_searches.push("event_hash=?".to_owned());
params.push(Box::new(ex)); params.push(Box::new(ex));
} }
Some(HexSearch::Range(lower, upper)) => { Some(HexSearch::Range(lower, upper)) => {
id_searches.push("(event_hash>? AND event_hash<?)".to_owned()); id_searches.push("(event_hash>? AND event_hash<?)".to_owned());
params.push(Box::new(lower)); params.push(Box::new(lower));
params.push(Box::new(upper)); params.push(Box::new(upper));
} }
Some(HexSearch::LowerOnly(lower)) => { Some(HexSearch::LowerOnly(lower)) => {
id_searches.push("event_hash>?".to_owned()); id_searches.push("event_hash>?".to_owned());
params.push(Box::new(lower)); params.push(Box::new(lower));
} }
None => { None => {
info!("Could not parse hex range from id {:?}", id); info!("Could not parse hex range from id {:?}", id);
}
} }
} }
let id_clause = format!("({})", id_searches.join(" OR "));
filter_components.push(id_clause);
} }
// Query for tags let id_clause = format!("({})", id_searches.join(" OR "));
if let Some(map) = &f.tags { filter_components.push(id_clause);
for (key, val) in map.iter() { }
let mut str_vals: Vec<Box<dyn ToSql>> = vec![]; // Query for tags
let mut blob_vals: Vec<Box<dyn ToSql>> = vec![]; if let Some(map) = &f.tags {
for v in val { for (key, val) in map.iter() {
if is_hex(v) { let mut str_vals: Vec<Box<dyn ToSql>> = vec![];
if let Ok(h) = hex::decode(&v) { let mut blob_vals: Vec<Box<dyn ToSql>> = vec![];
blob_vals.push(Box::new(h)); for v in val {
} if is_hex(v) {
} else { if let Ok(h) = hex::decode(&v) {
str_vals.push(Box::new(v.to_owned())); blob_vals.push(Box::new(h));
} }
} else {
str_vals.push(Box::new(v.to_owned()));
} }
// create clauses with "?" params for each tag value being searched
let str_clause = format!("value IN ({})", repeat_vars(str_vals.len()));
let blob_clause = format!("value_hex IN ({})", repeat_vars(blob_vals.len()));
let tag_clause = format!("(name=? AND ({} OR {}))", str_clause, blob_clause);
// add the tag name as the first parameter
params.push(Box::new(key.to_owned()));
// add all tag values that are plain strings as params
params.append(&mut str_vals);
// add all tag values that are blobs as params
params.append(&mut blob_vals);
filter_components.push(tag_clause);
} }
} // create clauses with "?" params for each tag value being searched
// Query for timestamp let str_clause = format!("value IN ({})", repeat_vars(str_vals.len()));
if f.since.is_some() { let blob_clause = format!("value_hex IN ({})", repeat_vars(blob_vals.len()));
let created_clause = format!("created_at > {}", f.since.unwrap()); let tag_clause = format!("(name=? AND ({} OR {}))", str_clause, blob_clause);
filter_components.push(created_clause); // add the tag name as the first parameter
} params.push(Box::new(key.to_owned()));
// Query for timestamp // add all tag values that are plain strings as params
if f.until.is_some() { params.append(&mut str_vals);
let until_clause = format!("created_at < {}", f.until.unwrap()); // add all tag values that are blobs as params
filter_components.push(until_clause); params.append(&mut blob_vals);
} filter_components.push(tag_clause);
// combine all clauses, and add to filter_clauses
if !filter_components.is_empty() {
let mut fc = "( ".to_owned();
fc.push_str(&filter_components.join(" AND "));
fc.push_str(" )");
filter_clauses.push(fc);
} }
} }
// Query for timestamp
if f.since.is_some() {
let created_clause = format!("created_at > {}", f.since.unwrap());
filter_components.push(created_clause);
}
// Query for timestamp
if f.until.is_some() {
let until_clause = format!("created_at < {}", f.until.unwrap());
filter_components.push(until_clause);
}
// never display hidden events // never display hidden events
query.push_str(" WHERE hidden!=TRUE "); query.push_str(" WHERE hidden!=TRUE");
// build filter component conditions
if !filter_components.is_empty() {
query.push_str(" AND ");
query.push_str(&filter_components.join(" AND "));
}
// Apply per-filter limit to this subquery.
// The use of a LIMIT implies a DESC order, to capture only the most recent events.
if let Some(lim) = f.limit {
query.push_str(&format!(" ORDER BY e.created_at DESC LIMIT {}", lim))
} else {
query.push_str(" ORDER BY e.created_at ASC")
}
(query, params)
}
// combine all filters with OR clauses, if any exist /// Create a dynamic SQL query string and params from a subscription.
if !filter_clauses.is_empty() { fn query_from_sub(sub: &Subscription) -> (String, Vec<Box<dyn ToSql>>) {
query.push_str(" AND ("); // build a dynamic SQL query for an entire subscription, based on
query.push_str(&filter_clauses.join(" OR ")); // SQL subqueries for filters.
query.push_str(") "); let mut subqueries: Vec<String> = Vec::new();
// subquery params
let mut params: Vec<Box<dyn ToSql>> = vec![];
// for every filter in the subscription, generate a subquery
for f in sub.filters.iter() {
let (f_subquery, mut f_params) = query_from_filter(&f);
subqueries.push(f_subquery);
params.append(&mut f_params);
} }
// add order clause // encapsulate subqueries into select statements
query.push_str(&format!( let subqueries_selects: Vec<String> = subqueries
" ORDER BY created_at {}", .iter()
limit.map_or("ASC", |_| "DESC") .map(|s| {
)); return format!("SELECT content, created_at FROM ({})", s);
if let Some(lim) = limit { })
query.push_str(&format!(" LIMIT {}", lim)) .collect();
} let query: String = subqueries_selects.join(" UNION ");
debug!("query string: {}", query); info!("final query string: {}", query);
(query, params) (query, params)
} }

2
src/subscription.rs
View File

@ -32,7 +32,7 @@ pub struct ReqFilter {
/// List of author public keys /// List of author public keys
pub authors: Option<Vec<String>>, pub authors: Option<Vec<String>>,
/// Limit number of results /// Limit number of results
pub limit: Option<u32>, pub limit: Option<u64>,
/// Set of tags /// Set of tags
#[serde(skip)] #[serde(skip)]
pub tags: Option<HashMap<String, HashSet<String>>>, pub tags: Option<HashMap<String, HashSet<String>>>,