nostr/nostr-rs-relay
1
0
Fork 0
mirror of https://github.com/scsibug/nostr-rs-relay.git synced 2024-11-22 00:59:07 -05:00
Code Issues Packages Projects Releases Wiki Activity

improvement: run filters as separate queries, to reduce complexity on SQLite query planner

Browse Source
This commit is contained in:
Greg Heartsfield 2023-01-30 18:02:40 -06:00
parent 3979a94726
commit 2557c7f69c
2 changed files with 106 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/repo/sqlite.rs
View File

@ -332,19 +332,20 @@ impl NostrRepo for SqliteRepo {
} }
let start = Instant::now(); let start = Instant::now();
let mut row_count: usize = 0; let mut row_count: usize = 0;
// generate SQL query // cutoff for displaying slow queries
let (q, p, idxs) = query_from_sub(&sub); let slow_cutoff = Duration::from_millis(2000);
let mut filter_count = 0;
for filter in sub.filters.iter() {
filter_count += 1;
let (q, p, idx) = query_from_filter(&filter);
let sql_gen_elapsed = start.elapsed(); let sql_gen_elapsed = start.elapsed();
if sql_gen_elapsed > Duration::from_millis(10) { if sql_gen_elapsed > Duration::from_millis(10) {
debug!("SQL (slow) generated in {:?}", start.elapsed()); debug!("SQL (slow) generated in {:?}", start.elapsed());
} }
// cutoff for displaying slow queries
let slow_cutoff = Duration::from_millis(2000);
// any client that doesn't cause us to generate new rows in 5 // any client that doesn't cause us to generate new rows in 5
// seconds gets dropped. // seconds gets dropped.
let abort_cutoff = Duration::from_secs(5); let abort_cutoff = Duration::from_secs(5);
let start = Instant::now();
let mut slow_first_event; let mut slow_first_event;
let mut last_successful_send = Instant::now(); let mut last_successful_send = Instant::now();
if let Ok(mut conn) = self.read_pool.get() { if let Ok(mut conn) = self.read_pool.get() {
@ -360,8 +361,8 @@ impl NostrRepo for SqliteRepo {
slow_first_event = first_event_elapsed >= slow_cutoff; slow_first_event = first_event_elapsed >= slow_cutoff;
if first_result { if first_result {
debug!( debug!(
"first result in {:?} (cid: {}, sub: {:?}) [used indexes: {:?}]", "first result in {:?} (cid: {}, sub: {:?}, filter: {}) [used index: {:?}]",
first_event_elapsed, client_id, sub.id, idxs first_event_elapsed, client_id, sub.id, filter_count, idx
); );
first_result = false; first_result = false;
} }
@ -428,12 +429,10 @@ impl NostrRepo for SqliteRepo {
.ok(); .ok();
last_successful_send = Instant::now(); last_successful_send = Instant::now();
} }
query_tx } else {
.blocking_send(QueryResult { warn!("Could not get a database connection for querying");
sub_id: sub.get_id(), }
event: "EOSE".to_string(), }
})
.ok();
debug!( debug!(
"query completed in {:?} (cid: {}, sub: {:?}, db_time: {:?}, rows: {})", "query completed in {:?} (cid: {}, sub: {:?}, db_time: {:?}, rows: {})",
pre_spawn_start.elapsed(), pre_spawn_start.elapsed(),
@ -442,9 +441,12 @@ impl NostrRepo for SqliteRepo {
start.elapsed(), start.elapsed(),
row_count row_count
); );
} else { query_tx
warn!("Could not get a database connection for querying"); .blocking_send(QueryResult {
} sub_id: sub.get_id(),
event: "EOSE".to_string(),
})
.ok();
metrics metrics
.query_sub .query_sub
.observe(pre_spawn_start.elapsed().as_secs_f64()); .observe(pre_spawn_start.elapsed().as_secs_f64());
@ -668,7 +670,7 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
// if the filter is malformed, don't return anything. // if the filter is malformed, don't return anything.
if f.force_no_match { if f.force_no_match {
let empty_query = "SELECT e.content, e.created_at FROM event e WHERE 1=0".to_owned(); let empty_query = "SELECT e.content FROM event e WHERE 1=0".to_owned();
// query parameters for SQLite // query parameters for SQLite
let empty_params: Vec<Box<dyn ToSql>> = vec![]; let empty_params: Vec<Box<dyn ToSql>> = vec![];
return (empty_query, empty_params, None); return (empty_query, empty_params, None);
@ -677,7 +679,7 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
// check if the index needs to be overriden // check if the index needs to be overriden
let idx_name = override_index(f); let idx_name = override_index(f);
let idx_stmt = idx_name.as_ref().map_or_else(|| "".to_owned(), |i| format!("INDEXED BY {}",i)); let idx_stmt = idx_name.as_ref().map_or_else(|| "".to_owned(), |i| format!("INDEXED BY {}",i));
let mut query = format!("SELECT e.content, e.created_at FROM event e {}", idx_stmt); let mut query = format!("SELECT e.content FROM event e {}", idx_stmt);
// query parameters for SQLite // query parameters for SQLite
let mut params: Vec<Box<dyn ToSql>> = vec![]; let mut params: Vec<Box<dyn ToSql>> = vec![];
@ -815,7 +817,7 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
} }
/// Create a dynamic SQL query string and params from a subscription. /// Create a dynamic SQL query string and params from a subscription.
fn query_from_sub(sub: &Subscription) -> (String, Vec<Box<dyn ToSql>>, Vec<String>) { fn _query_from_sub(sub: &Subscription) -> (String, Vec<Box<dyn ToSql>>, Vec<String>) {
// build a dynamic SQL query for an entire subscription, based on // build a dynamic SQL query for an entire subscription, based on
// SQL subqueries for filters. // SQL subqueries for filters.
let mut subqueries: Vec<String> = Vec::new(); let mut subqueries: Vec<String> = Vec::new();

2
src/server.rs
View File

@ -719,7 +719,7 @@ async fn nostr_server(
match parsed { match parsed {
Ok(e) => { Ok(e) => {
let id_prefix:String = e.id.chars().take(8).collect(); let id_prefix:String = e.id.chars().take(8).collect();
debug!("successfully parsed/validated event: {:?} (cid: {})", id_prefix, cid); debug!("successfully parsed/validated event: {:?} (cid: {}, kind: {})", id_prefix, cid, e.kind);
// check if the event is too far in the future. // check if the event is too far in the future.
if e.is_valid_timestamp(settings.options.reject_future_seconds) { if e.is_valid_timestamp(settings.options.reject_future_seconds) {
// Write this to the database. // Write this to the database.