mirror of
https://github.com/scsibug/nostr-rs-relay.git
synced 2024-11-14 06:59:07 -05:00
refactor: format
cargo fmt
This commit is contained in:
parent
440217e1ee
commit
6df92f9580
|
@ -1,16 +1,16 @@
|
||||||
|
use nostr_rs_relay::config;
|
||||||
|
use nostr_rs_relay::error::{Error, Result};
|
||||||
|
use nostr_rs_relay::event::{single_char_tagname, Event};
|
||||||
|
use nostr_rs_relay::repo::sqlite::{build_pool, PooledConnection};
|
||||||
|
use nostr_rs_relay::repo::sqlite_migration::{curr_db_version, DB_VERSION};
|
||||||
|
use nostr_rs_relay::utils::is_lower_hex;
|
||||||
|
use rusqlite::params;
|
||||||
|
use rusqlite::{OpenFlags, Transaction};
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use nostr_rs_relay::utils::is_lower_hex;
|
|
||||||
use tracing::info;
|
|
||||||
use nostr_rs_relay::config;
|
|
||||||
use nostr_rs_relay::event::{Event,single_char_tagname};
|
|
||||||
use nostr_rs_relay::error::{Error, Result};
|
|
||||||
use nostr_rs_relay::repo::sqlite::{PooledConnection, build_pool};
|
|
||||||
use nostr_rs_relay::repo::sqlite_migration::{curr_db_version, DB_VERSION};
|
|
||||||
use rusqlite::{OpenFlags, Transaction};
|
|
||||||
use std::sync::mpsc;
|
use std::sync::mpsc;
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use rusqlite::params;
|
use tracing::info;
|
||||||
|
|
||||||
/// Bulk load JSONL data from STDIN to the database specified in config.toml (or ./nostr.db as a default).
|
/// Bulk load JSONL data from STDIN to the database specified in config.toml (or ./nostr.db as a default).
|
||||||
/// The database must already exist, this will not create a new one.
|
/// The database must already exist, this will not create a new one.
|
||||||
|
@ -26,7 +26,14 @@ pub fn main() -> Result<()> {
|
||||||
return Err(Error::DatabaseDirError);
|
return Err(Error::DatabaseDirError);
|
||||||
}
|
}
|
||||||
// Get a database pool
|
// Get a database pool
|
||||||
let pool = build_pool("bulk-loader", &settings, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, 1,4,false);
|
let pool = build_pool(
|
||||||
|
"bulk-loader",
|
||||||
|
&settings,
|
||||||
|
OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE,
|
||||||
|
1,
|
||||||
|
4,
|
||||||
|
false,
|
||||||
|
);
|
||||||
{
|
{
|
||||||
// check for database schema version
|
// check for database schema version
|
||||||
let mut conn: PooledConnection = pool.get()?;
|
let mut conn: PooledConnection = pool.get()?;
|
||||||
|
@ -86,19 +93,19 @@ pub fn main() -> Result<()> {
|
||||||
match write_event(&tx, e) {
|
match write_event(&tx, e) {
|
||||||
Ok(c) => {
|
Ok(c) => {
|
||||||
new_events += c;
|
new_events += c;
|
||||||
},
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
info!("error inserting event: {:?}", e);
|
info!("error inserting event: {:?}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
// signal that the sender will never produce more
|
// signal that the sender will never produce more
|
||||||
// events
|
// events
|
||||||
has_more_events = false;
|
has_more_events = false;
|
||||||
break;
|
break;
|
||||||
},
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
info!("sender is closed");
|
info!("sender is closed");
|
||||||
// sender is done
|
// sender is done
|
||||||
|
@ -108,7 +115,6 @@ pub fn main() -> Result<()> {
|
||||||
info!("committed {} events...", new_events);
|
info!("committed {} events...", new_events);
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
conn.execute_batch("pragma wal_checkpoint(truncate)")?;
|
conn.execute_batch("pragma wal_checkpoint(truncate)")?;
|
||||||
|
|
||||||
}
|
}
|
||||||
info!("processed {} events", events_read);
|
info!("processed {} events", events_read);
|
||||||
info!("stored {} new events", new_events);
|
info!("stored {} new events", new_events);
|
||||||
|
|
|
@ -7,14 +7,14 @@ pub struct CLIArgs {
|
||||||
short,
|
short,
|
||||||
long,
|
long,
|
||||||
help = "Use the <directory> as the location of the database",
|
help = "Use the <directory> as the location of the database",
|
||||||
required = false,
|
required = false
|
||||||
)]
|
)]
|
||||||
pub db: Option<String>,
|
pub db: Option<String>,
|
||||||
#[arg(
|
#[arg(
|
||||||
short,
|
short,
|
||||||
long,
|
long,
|
||||||
help = "Use the <file name> as the location of the config file",
|
help = "Use the <file name> as the location of the config file",
|
||||||
required = false,
|
required = false
|
||||||
)]
|
)]
|
||||||
pub config: Option<String>,
|
pub config: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
|
@ -70,7 +70,7 @@ pub struct Limits {
|
||||||
pub broadcast_buffer: usize, // events to buffer for subscribers (prevents slow readers from consuming memory)
|
pub broadcast_buffer: usize, // events to buffer for subscribers (prevents slow readers from consuming memory)
|
||||||
pub event_persist_buffer: usize, // events to buffer for database commits (block senders if database writes are too slow)
|
pub event_persist_buffer: usize, // events to buffer for database commits (block senders if database writes are too slow)
|
||||||
pub event_kind_blacklist: Option<Vec<u64>>,
|
pub event_kind_blacklist: Option<Vec<u64>>,
|
||||||
pub event_kind_allowlist: Option<Vec<u64>>
|
pub event_kind_allowlist: Option<Vec<u64>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
@ -178,12 +178,14 @@ impl Settings {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn new_from_default(
|
||||||
fn new_from_default(default: &Settings, config_file_name: &Option<String>) -> Result<Self, ConfigError> {
|
default: &Settings,
|
||||||
|
config_file_name: &Option<String>,
|
||||||
|
) -> Result<Self, ConfigError> {
|
||||||
let default_config_file_name = "config.toml".to_string();
|
let default_config_file_name = "config.toml".to_string();
|
||||||
let config: &String = match config_file_name {
|
let config: &String = match config_file_name {
|
||||||
Some(value) => value,
|
Some(value) => value,
|
||||||
None => &default_config_file_name
|
None => &default_config_file_name,
|
||||||
};
|
};
|
||||||
let builder = Config::builder();
|
let builder = Config::builder();
|
||||||
let config: Config = builder
|
let config: Config = builder
|
||||||
|
|
21
src/db.rs
21
src/db.rs
|
@ -3,20 +3,20 @@ use crate::config::Settings;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::event::Event;
|
use crate::event::Event;
|
||||||
use crate::notice::Notice;
|
use crate::notice::Notice;
|
||||||
|
use crate::repo::postgres::{PostgresPool, PostgresRepo};
|
||||||
|
use crate::repo::sqlite::SqliteRepo;
|
||||||
|
use crate::repo::NostrRepo;
|
||||||
use crate::server::NostrMetrics;
|
use crate::server::NostrMetrics;
|
||||||
use crate::nauthz;
|
use crate::nauthz;
|
||||||
use governor::clock::Clock;
|
use governor::clock::Clock;
|
||||||
use governor::{Quota, RateLimiter};
|
use governor::{Quota, RateLimiter};
|
||||||
use r2d2;
|
use r2d2;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::thread;
|
|
||||||
use sqlx::pool::PoolOptions;
|
use sqlx::pool::PoolOptions;
|
||||||
use sqlx::postgres::PgConnectOptions;
|
use sqlx::postgres::PgConnectOptions;
|
||||||
use sqlx::ConnectOptions;
|
use sqlx::ConnectOptions;
|
||||||
use crate::repo::sqlite::SqliteRepo;
|
use std::sync::Arc;
|
||||||
use crate::repo::postgres::{PostgresRepo,PostgresPool};
|
use std::thread;
|
||||||
use crate::repo::NostrRepo;
|
use std::time::{Duration, Instant};
|
||||||
use std::time::{Instant, Duration};
|
|
||||||
use tracing::log::LevelFilter;
|
use tracing::log::LevelFilter;
|
||||||
use tracing::{debug, info, trace, warn};
|
use tracing::{debug, info, trace, warn};
|
||||||
|
|
||||||
|
@ -42,8 +42,8 @@ pub const DB_FILE: &str = "nostr.db";
|
||||||
/// Will panic if the pool could not be created.
|
/// Will panic if the pool could not be created.
|
||||||
pub async fn build_repo(settings: &Settings, metrics: NostrMetrics) -> Arc<dyn NostrRepo> {
|
pub async fn build_repo(settings: &Settings, metrics: NostrMetrics) -> Arc<dyn NostrRepo> {
|
||||||
match settings.database.engine.as_str() {
|
match settings.database.engine.as_str() {
|
||||||
"sqlite" => {Arc::new(build_sqlite_pool(settings, metrics).await)},
|
"sqlite" => Arc::new(build_sqlite_pool(settings, metrics).await),
|
||||||
"postgres" => {Arc::new(build_postgres_pool(settings, metrics).await)},
|
"postgres" => Arc::new(build_postgres_pool(settings, metrics).await),
|
||||||
_ => panic!("Unknown database engine"),
|
_ => panic!("Unknown database engine"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -165,10 +165,7 @@ pub async fn db_writer(
|
||||||
&event.kind
|
&event.kind
|
||||||
);
|
);
|
||||||
notice_tx
|
notice_tx
|
||||||
.try_send(Notice::blocked(
|
.try_send(Notice::blocked(event.id, "event kind is blocked by relay"))
|
||||||
event.id,
|
|
||||||
"event kind is blocked by relay"
|
|
||||||
))
|
|
||||||
.ok();
|
.ok();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
|
@ -84,7 +84,8 @@ pub struct ConditionQuery {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ConditionQuery {
|
impl ConditionQuery {
|
||||||
#[must_use] pub fn allows_event(&self, event: &Event) -> bool {
|
#[must_use]
|
||||||
|
pub fn allows_event(&self, event: &Event) -> bool {
|
||||||
// check each condition, to ensure that the event complies
|
// check each condition, to ensure that the event complies
|
||||||
// with the restriction.
|
// with the restriction.
|
||||||
for c in &self.conditions {
|
for c in &self.conditions {
|
||||||
|
@ -101,7 +102,8 @@ impl ConditionQuery {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify that the delegator approved the delegation; return a ConditionQuery if so.
|
// Verify that the delegator approved the delegation; return a ConditionQuery if so.
|
||||||
#[must_use] pub fn validate_delegation(
|
#[must_use]
|
||||||
|
pub fn validate_delegation(
|
||||||
delegator: &str,
|
delegator: &str,
|
||||||
delegatee: &str,
|
delegatee: &str,
|
||||||
cond_query: &str,
|
cond_query: &str,
|
||||||
|
@ -144,7 +146,8 @@ pub struct Condition {
|
||||||
|
|
||||||
impl Condition {
|
impl Condition {
|
||||||
/// Check if this condition allows the given event to be delegated
|
/// Check if this condition allows the given event to be delegated
|
||||||
#[must_use] pub fn allows_event(&self, event: &Event) -> bool {
|
#[must_use]
|
||||||
|
pub fn allows_event(&self, event: &Event) -> bool {
|
||||||
// determine what the right-hand side of the operator is
|
// determine what the right-hand side of the operator is
|
||||||
let resolved_field = match &self.field {
|
let resolved_field = match &self.field {
|
||||||
Field::Kind => event.kind,
|
Field::Kind => event.kind,
|
||||||
|
|
84
src/event.rs
84
src/event.rs
|
@ -1,6 +1,9 @@
|
||||||
//! Event parsing and validation
|
//! Event parsing and validation
|
||||||
use crate::delegation::validate_delegation;
|
use crate::delegation::validate_delegation;
|
||||||
use crate::error::Error::{CommandUnknownError, EventCouldNotCanonicalize, EventInvalidId, EventInvalidSignature, EventMalformedPubkey};
|
use crate::error::Error::{
|
||||||
|
CommandUnknownError, EventCouldNotCanonicalize, EventInvalidId, EventInvalidSignature,
|
||||||
|
EventMalformedPubkey,
|
||||||
|
};
|
||||||
use crate::error::Result;
|
use crate::error::Result;
|
||||||
use crate::nip05;
|
use crate::nip05;
|
||||||
use crate::utils::unix_time;
|
use crate::utils::unix_time;
|
||||||
|
@ -30,7 +33,8 @@ pub struct EventCmd {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EventCmd {
|
impl EventCmd {
|
||||||
#[must_use] pub fn event_id(&self) -> &str {
|
#[must_use]
|
||||||
|
pub fn event_id(&self) -> &str {
|
||||||
&self.event.id
|
&self.event.id
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -67,7 +71,8 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Attempt to form a single-char tag name.
|
/// Attempt to form a single-char tag name.
|
||||||
#[must_use] pub fn single_char_tagname(tagname: &str) -> Option<char> {
|
#[must_use]
|
||||||
|
pub fn single_char_tagname(tagname: &str) -> Option<char> {
|
||||||
// We return the tag character if and only if the tagname consists
|
// We return the tag character if and only if the tagname consists
|
||||||
// of a single char.
|
// of a single char.
|
||||||
let mut tagnamechars = tagname.chars();
|
let mut tagnamechars = tagname.chars();
|
||||||
|
@ -114,7 +119,8 @@ impl From<EventCmd> for Result<EventWrapper> {
|
||||||
|
|
||||||
impl Event {
|
impl Event {
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
#[must_use] pub fn simple_event() -> Event {
|
#[must_use]
|
||||||
|
pub fn simple_event() -> Event {
|
||||||
Event {
|
Event {
|
||||||
id: "0".to_owned(),
|
id: "0".to_owned(),
|
||||||
pubkey: "0".to_owned(),
|
pubkey: "0".to_owned(),
|
||||||
|
@ -128,12 +134,14 @@ impl Event {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn is_kind_metadata(&self) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_kind_metadata(&self) -> bool {
|
||||||
self.kind == 0
|
self.kind == 0
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Should this event be persisted?
|
/// Should this event be persisted?
|
||||||
#[must_use] pub fn is_ephemeral(&self) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_ephemeral(&self) -> bool {
|
||||||
self.kind >= 20000 && self.kind < 30000
|
self.kind >= 20000 && self.kind < 30000
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -160,29 +168,37 @@ impl Event {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Should this event be replaced with newer timestamps from same author?
|
/// Should this event be replaced with newer timestamps from same author?
|
||||||
#[must_use] pub fn is_replaceable(&self) -> bool {
|
#[must_use]
|
||||||
self.kind == 0 || self.kind == 3 || self.kind == 41 || (self.kind >= 10000 && self.kind < 20000)
|
pub fn is_replaceable(&self) -> bool {
|
||||||
|
self.kind == 0
|
||||||
|
|| self.kind == 3
|
||||||
|
|| self.kind == 41
|
||||||
|
|| (self.kind >= 10000 && self.kind < 20000)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Should this event be replaced with newer timestamps from same author, for distinct `d` tag values?
|
/// Should this event be replaced with newer timestamps from same author, for distinct `d` tag values?
|
||||||
#[must_use] pub fn is_param_replaceable(&self) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_param_replaceable(&self) -> bool {
|
||||||
self.kind >= 30000 && self.kind < 40000
|
self.kind >= 30000 && self.kind < 40000
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Should this event be replaced with newer timestamps from same author, for distinct `d` tag values?
|
/// Should this event be replaced with newer timestamps from same author, for distinct `d` tag values?
|
||||||
#[must_use] pub fn distinct_param(&self) -> Option<String> {
|
#[must_use]
|
||||||
|
pub fn distinct_param(&self) -> Option<String> {
|
||||||
if self.is_param_replaceable() {
|
if self.is_param_replaceable() {
|
||||||
let default = "".to_string();
|
let default = "".to_string();
|
||||||
let dvals:Vec<&String> = self.tags
|
let dvals: Vec<&String> = self
|
||||||
|
.tags
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|x| !x.is_empty())
|
.filter(|x| !x.is_empty())
|
||||||
.filter(|x| x.get(0).unwrap() == "d")
|
.filter(|x| x.get(0).unwrap() == "d")
|
||||||
.map(|x| x.get(1).unwrap_or(&default)).take(1)
|
.map(|x| x.get(1).unwrap_or(&default))
|
||||||
|
.take(1)
|
||||||
.collect();
|
.collect();
|
||||||
let dval_first = dvals.get(0);
|
let dval_first = dvals.get(0);
|
||||||
match dval_first {
|
match dval_first {
|
||||||
Some(_) => {dval_first.map(|x| x.to_string())},
|
Some(_) => dval_first.map(|x| x.to_string()),
|
||||||
None => Some(default)
|
None => Some(default),
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
|
@ -190,7 +206,8 @@ impl Event {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Pull a NIP-05 Name out of the event, if one exists
|
/// Pull a NIP-05 Name out of the event, if one exists
|
||||||
#[must_use] pub fn get_nip05_addr(&self) -> Option<nip05::Nip05Name> {
|
#[must_use]
|
||||||
|
pub fn get_nip05_addr(&self) -> Option<nip05::Nip05Name> {
|
||||||
if self.is_kind_metadata() {
|
if self.is_kind_metadata() {
|
||||||
// very quick check if we should attempt to parse this json
|
// very quick check if we should attempt to parse this json
|
||||||
if self.content.contains("\"nip05\"") {
|
if self.content.contains("\"nip05\"") {
|
||||||
|
@ -207,7 +224,8 @@ impl Event {
|
||||||
// is this event delegated (properly)?
|
// is this event delegated (properly)?
|
||||||
// does the signature match, and are conditions valid?
|
// does the signature match, and are conditions valid?
|
||||||
// if so, return an alternate author for the event
|
// if so, return an alternate author for the event
|
||||||
#[must_use] pub fn delegated_author(&self) -> Option<String> {
|
#[must_use]
|
||||||
|
pub fn delegated_author(&self) -> Option<String> {
|
||||||
// is there a delegation tag?
|
// is there a delegation tag?
|
||||||
let delegation_tag: Vec<String> = self
|
let delegation_tag: Vec<String> = self
|
||||||
.tags
|
.tags
|
||||||
|
@ -215,7 +233,8 @@ impl Event {
|
||||||
.filter(|x| x.len() == 4)
|
.filter(|x| x.len() == 4)
|
||||||
.filter(|x| x.get(0).unwrap() == "delegation")
|
.filter(|x| x.get(0).unwrap() == "delegation")
|
||||||
.take(1)
|
.take(1)
|
||||||
.next()?.clone(); // get first tag
|
.next()?
|
||||||
|
.clone(); // get first tag
|
||||||
|
|
||||||
//let delegation_tag = self.tag_values_by_name("delegation");
|
//let delegation_tag = self.tag_values_by_name("delegation");
|
||||||
// delegation tags should have exactly 3 elements after the name (pubkey, condition, sig)
|
// delegation tags should have exactly 3 elements after the name (pubkey, condition, sig)
|
||||||
|
@ -275,15 +294,18 @@ impl Event {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a short event identifier, suitable for logging.
|
/// Create a short event identifier, suitable for logging.
|
||||||
#[must_use] pub fn get_event_id_prefix(&self) -> String {
|
#[must_use]
|
||||||
|
pub fn get_event_id_prefix(&self) -> String {
|
||||||
self.id.chars().take(8).collect()
|
self.id.chars().take(8).collect()
|
||||||
}
|
}
|
||||||
#[must_use] pub fn get_author_prefix(&self) -> String {
|
#[must_use]
|
||||||
|
pub fn get_author_prefix(&self) -> String {
|
||||||
self.pubkey.chars().take(8).collect()
|
self.pubkey.chars().take(8).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Retrieve tag initial values across all tags matching the name
|
/// Retrieve tag initial values across all tags matching the name
|
||||||
#[must_use] pub fn tag_values_by_name(&self, tag_name: &str) -> Vec<String> {
|
#[must_use]
|
||||||
|
pub fn tag_values_by_name(&self, tag_name: &str) -> Vec<String> {
|
||||||
self.tags
|
self.tags
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|x| x.len() > 1)
|
.filter(|x| x.len() > 1)
|
||||||
|
@ -292,7 +314,8 @@ impl Event {
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn is_valid_timestamp(&self, reject_future_seconds: Option<usize>) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_valid_timestamp(&self, reject_future_seconds: Option<usize>) -> bool {
|
||||||
if let Some(allowable_future) = reject_future_seconds {
|
if let Some(allowable_future) = reject_future_seconds {
|
||||||
let curr_time = unix_time();
|
let curr_time = unix_time();
|
||||||
// calculate difference, plus how far future we allow
|
// calculate difference, plus how far future we allow
|
||||||
|
@ -384,7 +407,8 @@ impl Event {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Determine if the given tag and value set intersect with tags in this event.
|
/// Determine if the given tag and value set intersect with tags in this event.
|
||||||
#[must_use] pub fn generic_tag_val_intersect(&self, tagname: char, check: &HashSet<String>) -> bool {
|
#[must_use]
|
||||||
|
pub fn generic_tag_val_intersect(&self, tagname: char, check: &HashSet<String>) -> bool {
|
||||||
match &self.tagidx {
|
match &self.tagidx {
|
||||||
// check if this is indexable tagname
|
// check if this is indexable tagname
|
||||||
Some(idx) => match idx.get(&tagname) {
|
Some(idx) => match idx.get(&tagname) {
|
||||||
|
@ -614,8 +638,7 @@ mod tests {
|
||||||
// NIP case #1: "tags":[["d",""]]
|
// NIP case #1: "tags":[["d",""]]
|
||||||
let mut event = Event::simple_event();
|
let mut event = Event::simple_event();
|
||||||
event.kind = 30000;
|
event.kind = 30000;
|
||||||
event.tags = vec![
|
event.tags = vec![vec!["d".to_owned(), "".to_owned()]];
|
||||||
vec!["d".to_owned(), "".to_owned()]];
|
|
||||||
assert_eq!(event.distinct_param(), Some("".to_string()));
|
assert_eq!(event.distinct_param(), Some("".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -632,8 +655,7 @@ mod tests {
|
||||||
// NIP case #3: "tags":[["d"]]: implicit empty value ""
|
// NIP case #3: "tags":[["d"]]: implicit empty value ""
|
||||||
let mut event = Event::simple_event();
|
let mut event = Event::simple_event();
|
||||||
event.kind = 30000;
|
event.kind = 30000;
|
||||||
event.tags = vec![
|
event.tags = vec![vec!["d".to_owned()]];
|
||||||
vec!["d".to_owned()]];
|
|
||||||
assert_eq!(event.distinct_param(), Some("".to_string()));
|
assert_eq!(event.distinct_param(), Some("".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -644,7 +666,7 @@ mod tests {
|
||||||
event.kind = 30000;
|
event.kind = 30000;
|
||||||
event.tags = vec![
|
event.tags = vec![
|
||||||
vec!["d".to_owned(), "".to_string()],
|
vec!["d".to_owned(), "".to_string()],
|
||||||
vec!["d".to_owned(), "not empty".to_string()]
|
vec!["d".to_owned(), "not empty".to_string()],
|
||||||
];
|
];
|
||||||
assert_eq!(event.distinct_param(), Some("".to_string()));
|
assert_eq!(event.distinct_param(), Some("".to_string()));
|
||||||
}
|
}
|
||||||
|
@ -657,7 +679,7 @@ mod tests {
|
||||||
event.kind = 30000;
|
event.kind = 30000;
|
||||||
event.tags = vec![
|
event.tags = vec![
|
||||||
vec!["d".to_owned(), "not empty".to_string()],
|
vec!["d".to_owned(), "not empty".to_string()],
|
||||||
vec!["d".to_owned(), "".to_string()]
|
vec!["d".to_owned(), "".to_string()],
|
||||||
];
|
];
|
||||||
assert_eq!(event.distinct_param(), Some("not empty".to_string()));
|
assert_eq!(event.distinct_param(), Some("not empty".to_string()));
|
||||||
}
|
}
|
||||||
|
@ -670,7 +692,7 @@ mod tests {
|
||||||
event.tags = vec![
|
event.tags = vec![
|
||||||
vec!["d".to_owned()],
|
vec!["d".to_owned()],
|
||||||
vec!["d".to_owned(), "second value".to_string()],
|
vec!["d".to_owned(), "second value".to_string()],
|
||||||
vec!["d".to_owned(), "third value".to_string()]
|
vec!["d".to_owned(), "third value".to_string()],
|
||||||
];
|
];
|
||||||
assert_eq!(event.distinct_param(), Some("".to_string()));
|
assert_eq!(event.distinct_param(), Some("".to_string()));
|
||||||
}
|
}
|
||||||
|
@ -680,9 +702,7 @@ mod tests {
|
||||||
// NIP case #6: "tags":[["e"]]: same as no tags
|
// NIP case #6: "tags":[["e"]]: same as no tags
|
||||||
let mut event = Event::simple_event();
|
let mut event = Event::simple_event();
|
||||||
event.kind = 30000;
|
event.kind = 30000;
|
||||||
event.tags = vec![
|
event.tags = vec![vec!["e".to_owned()]];
|
||||||
vec!["e".to_owned()],
|
|
||||||
];
|
|
||||||
assert_eq!(event.distinct_param(), Some("".to_string()));
|
assert_eq!(event.distinct_param(), Some("".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
//! Utilities for searching hexadecimal
|
//! Utilities for searching hexadecimal
|
||||||
use crate::utils::{is_hex};
|
use crate::utils::is_hex;
|
||||||
use hex;
|
use hex;
|
||||||
|
|
||||||
/// Types of hexadecimal queries.
|
/// Types of hexadecimal queries.
|
||||||
|
@ -19,7 +19,8 @@ fn is_all_fs(s: &str) -> bool {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find the next hex sequence greater than the argument.
|
/// Find the next hex sequence greater than the argument.
|
||||||
#[must_use] pub fn hex_range(s: &str) -> Option<HexSearch> {
|
#[must_use]
|
||||||
|
pub fn hex_range(s: &str) -> Option<HexSearch> {
|
||||||
let mut hash_base = s.to_owned();
|
let mut hash_base = s.to_owned();
|
||||||
if !is_hex(&hash_base) || hash_base.len() > 64 {
|
if !is_hex(&hash_base) || hash_base.len() > 64 {
|
||||||
return None;
|
return None;
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
//! Server process
|
//! Server process
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
|
use console_subscriber::ConsoleLayer;
|
||||||
use nostr_rs_relay::cli::CLIArgs;
|
use nostr_rs_relay::cli::CLIArgs;
|
||||||
use nostr_rs_relay::config;
|
use nostr_rs_relay::config;
|
||||||
use nostr_rs_relay::server::start_server;
|
use nostr_rs_relay::server::start_server;
|
||||||
|
@ -7,7 +8,6 @@ use std::sync::mpsc as syncmpsc;
|
||||||
use std::sync::mpsc::{Receiver as MpscReceiver, Sender as MpscSender};
|
use std::sync::mpsc::{Receiver as MpscReceiver, Sender as MpscSender};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
use console_subscriber::ConsoleLayer;
|
|
||||||
|
|
||||||
/// Start running a Nostr relay server.
|
/// Start running a Nostr relay server.
|
||||||
fn main() {
|
fn main() {
|
||||||
|
|
30
src/nip05.rs
30
src/nip05.rs
|
@ -8,11 +8,11 @@ use crate::config::VerifiedUsers;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::event::Event;
|
use crate::event::Event;
|
||||||
use crate::repo::NostrRepo;
|
use crate::repo::NostrRepo;
|
||||||
use std::sync::Arc;
|
|
||||||
use hyper::body::HttpBody;
|
use hyper::body::HttpBody;
|
||||||
use hyper::client::connect::HttpConnector;
|
use hyper::client::connect::HttpConnector;
|
||||||
use hyper::Client;
|
use hyper::Client;
|
||||||
use hyper_tls::HttpsConnector;
|
use hyper_tls::HttpsConnector;
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use std::time::SystemTime;
|
use std::time::SystemTime;
|
||||||
|
@ -48,7 +48,8 @@ pub struct Nip05Name {
|
||||||
|
|
||||||
impl Nip05Name {
|
impl Nip05Name {
|
||||||
/// Does this name represent the entire domain?
|
/// Does this name represent the entire domain?
|
||||||
#[must_use] pub fn is_domain_only(&self) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_domain_only(&self) -> bool {
|
||||||
self.local == "_"
|
self.local == "_"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -73,7 +74,10 @@ impl std::convert::TryFrom<&str> for Nip05Name {
|
||||||
// check if local name is valid
|
// check if local name is valid
|
||||||
let local = components[0];
|
let local = components[0];
|
||||||
let domain = components[1];
|
let domain = components[1];
|
||||||
if local.chars().all(|x| x.is_alphanumeric() || x == '_' || x == '-' || x == '.') {
|
if local
|
||||||
|
.chars()
|
||||||
|
.all(|x| x.is_alphanumeric() || x == '_' || x == '-' || x == '.')
|
||||||
|
{
|
||||||
if domain
|
if domain
|
||||||
.chars()
|
.chars()
|
||||||
.all(|x| x.is_alphanumeric() || x == '-' || x == '.')
|
.all(|x| x.is_alphanumeric() || x == '-' || x == '.')
|
||||||
|
@ -349,10 +353,8 @@ impl Verifier {
|
||||||
UserWebVerificationStatus::Verified => {
|
UserWebVerificationStatus::Verified => {
|
||||||
// freshly verified account, update the
|
// freshly verified account, update the
|
||||||
// timestamp.
|
// timestamp.
|
||||||
self.repo.update_verification_timestamp(v.rowid)
|
self.repo.update_verification_timestamp(v.rowid).await?;
|
||||||
.await?;
|
|
||||||
info!("verification updated for {}", v.to_string());
|
info!("verification updated for {}", v.to_string());
|
||||||
|
|
||||||
}
|
}
|
||||||
UserWebVerificationStatus::DomainNotAllowed
|
UserWebVerificationStatus::DomainNotAllowed
|
||||||
| UserWebVerificationStatus::Unknown => {
|
| UserWebVerificationStatus::Unknown => {
|
||||||
|
@ -367,8 +369,7 @@ impl Verifier {
|
||||||
"giving up on verifying {:?} after {} failures",
|
"giving up on verifying {:?} after {} failures",
|
||||||
v.name, v.failure_count
|
v.name, v.failure_count
|
||||||
);
|
);
|
||||||
self.repo.delete_verification(v.rowid)
|
self.repo.delete_verification(v.rowid).await?;
|
||||||
.await?;
|
|
||||||
} else {
|
} else {
|
||||||
// record normal failure, incrementing failure count
|
// record normal failure, incrementing failure count
|
||||||
info!("verification failed for {}", v.to_string());
|
info!("verification failed for {}", v.to_string());
|
||||||
|
@ -379,8 +380,7 @@ impl Verifier {
|
||||||
// domain has removed the verification, drop
|
// domain has removed the verification, drop
|
||||||
// the record on our side.
|
// the record on our side.
|
||||||
info!("verification rescinded for {}", v.to_string());
|
info!("verification rescinded for {}", v.to_string());
|
||||||
self.repo.delete_verification(v.rowid)
|
self.repo.delete_verification(v.rowid).await?;
|
||||||
.await?;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -433,7 +433,9 @@ impl Verifier {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// write the verification record
|
// write the verification record
|
||||||
self.repo.create_verification_record(&event.id, name).await?;
|
self.repo
|
||||||
|
.create_verification_record(&event.id, name)
|
||||||
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -463,7 +465,8 @@ pub struct VerificationRecord {
|
||||||
|
|
||||||
/// Check with settings to determine if a given domain is allowed to
|
/// Check with settings to determine if a given domain is allowed to
|
||||||
/// publish.
|
/// publish.
|
||||||
#[must_use] pub fn is_domain_allowed(
|
#[must_use]
|
||||||
|
pub fn is_domain_allowed(
|
||||||
domain: &str,
|
domain: &str,
|
||||||
whitelist: &Option<Vec<String>>,
|
whitelist: &Option<Vec<String>>,
|
||||||
blacklist: &Option<Vec<String>>,
|
blacklist: &Option<Vec<String>>,
|
||||||
|
@ -483,7 +486,8 @@ pub struct VerificationRecord {
|
||||||
impl VerificationRecord {
|
impl VerificationRecord {
|
||||||
/// Check if the record is recent enough to be considered valid,
|
/// Check if the record is recent enough to be considered valid,
|
||||||
/// and the domain is allowed.
|
/// and the domain is allowed.
|
||||||
#[must_use] pub fn is_valid(&self, verified_users_settings: &VerifiedUsers) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_valid(&self, verified_users_settings: &VerifiedUsers) -> bool {
|
||||||
//let settings = SETTINGS.read().unwrap();
|
//let settings = SETTINGS.read().unwrap();
|
||||||
// how long a verification record is good for
|
// how long a verification record is good for
|
||||||
let nip05_expiration = &verified_users_settings.verify_expiration_duration;
|
let nip05_expiration = &verified_users_settings.verify_expiration_duration;
|
||||||
|
|
|
@ -20,14 +20,16 @@ pub enum Notice {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EventResultStatus {
|
impl EventResultStatus {
|
||||||
#[must_use] pub fn to_bool(&self) -> bool {
|
#[must_use]
|
||||||
|
pub fn to_bool(&self) -> bool {
|
||||||
match self {
|
match self {
|
||||||
Self::Duplicate | Self::Saved => true,
|
Self::Duplicate | Self::Saved => true,
|
||||||
Self::Invalid | Self::Blocked | Self::RateLimited | Self::Error => false,
|
Self::Invalid | Self::Blocked | Self::RateLimited | Self::Error => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn prefix(&self) -> &'static str {
|
#[must_use]
|
||||||
|
pub fn prefix(&self) -> &'static str {
|
||||||
match self {
|
match self {
|
||||||
Self::Saved => "saved",
|
Self::Saved => "saved",
|
||||||
Self::Duplicate => "duplicate",
|
Self::Duplicate => "duplicate",
|
||||||
|
@ -44,7 +46,8 @@ impl Notice {
|
||||||
// Notice::err_msg(format!("{}", err), id)
|
// Notice::err_msg(format!("{}", err), id)
|
||||||
//}
|
//}
|
||||||
|
|
||||||
#[must_use] pub fn message(msg: String) -> Notice {
|
#[must_use]
|
||||||
|
pub fn message(msg: String) -> Notice {
|
||||||
Notice::Message(msg)
|
Notice::Message(msg)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -53,27 +56,33 @@ impl Notice {
|
||||||
Notice::EventResult(EventResult { id, msg, status })
|
Notice::EventResult(EventResult { id, msg, status })
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn invalid(id: String, msg: &str) -> Notice {
|
#[must_use]
|
||||||
|
pub fn invalid(id: String, msg: &str) -> Notice {
|
||||||
Notice::prefixed(id, msg, EventResultStatus::Invalid)
|
Notice::prefixed(id, msg, EventResultStatus::Invalid)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn blocked(id: String, msg: &str) -> Notice {
|
#[must_use]
|
||||||
|
pub fn blocked(id: String, msg: &str) -> Notice {
|
||||||
Notice::prefixed(id, msg, EventResultStatus::Blocked)
|
Notice::prefixed(id, msg, EventResultStatus::Blocked)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn rate_limited(id: String, msg: &str) -> Notice {
|
#[must_use]
|
||||||
|
pub fn rate_limited(id: String, msg: &str) -> Notice {
|
||||||
Notice::prefixed(id, msg, EventResultStatus::RateLimited)
|
Notice::prefixed(id, msg, EventResultStatus::RateLimited)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn duplicate(id: String) -> Notice {
|
#[must_use]
|
||||||
|
pub fn duplicate(id: String) -> Notice {
|
||||||
Notice::prefixed(id, "", EventResultStatus::Duplicate)
|
Notice::prefixed(id, "", EventResultStatus::Duplicate)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn error(id: String, msg: &str) -> Notice {
|
#[must_use]
|
||||||
|
pub fn error(id: String, msg: &str) -> Notice {
|
||||||
Notice::prefixed(id, msg, EventResultStatus::Error)
|
Notice::prefixed(id, msg, EventResultStatus::Error)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use] pub fn saved(id: String) -> Notice {
|
#[must_use]
|
||||||
|
pub fn saved(id: String) -> Notice {
|
||||||
Notice::EventResult(EventResult {
|
Notice::EventResult(EventResult {
|
||||||
id,
|
id,
|
||||||
msg: "".into(),
|
msg: "".into(),
|
||||||
|
|
|
@ -7,10 +7,10 @@ use crate::utils::unix_time;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
|
|
||||||
pub mod sqlite;
|
|
||||||
pub mod sqlite_migration;
|
|
||||||
pub mod postgres;
|
pub mod postgres;
|
||||||
pub mod postgres_migration;
|
pub mod postgres_migration;
|
||||||
|
pub mod sqlite;
|
||||||
|
pub mod sqlite_migration;
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
pub trait NostrRepo: Send + Sync {
|
pub trait NostrRepo: Send + Sync {
|
||||||
|
|
|
@ -8,10 +8,11 @@ use async_std::stream::StreamExt;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use chrono::{DateTime, TimeZone, Utc};
|
use chrono::{DateTime, TimeZone, Utc};
|
||||||
use sqlx::postgres::PgRow;
|
use sqlx::postgres::PgRow;
|
||||||
|
use sqlx::Error::RowNotFound;
|
||||||
use sqlx::{Error, Execute, FromRow, Postgres, QueryBuilder, Row};
|
use sqlx::{Error, Execute, FromRow, Postgres, QueryBuilder, Row};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
use sqlx::Error::RowNotFound;
|
|
||||||
|
|
||||||
|
use crate::error;
|
||||||
use crate::hexrange::{hex_range, HexSearch};
|
use crate::hexrange::{hex_range, HexSearch};
|
||||||
use crate::repo::postgres_migration::run_migrations;
|
use crate::repo::postgres_migration::run_migrations;
|
||||||
use crate::server::NostrMetrics;
|
use crate::server::NostrMetrics;
|
||||||
|
@ -19,8 +20,7 @@ use crate::utils::{is_hex, is_lower_hex, self};
|
||||||
use tokio::sync::mpsc::Sender;
|
use tokio::sync::mpsc::Sender;
|
||||||
use tokio::sync::oneshot::Receiver;
|
use tokio::sync::oneshot::Receiver;
|
||||||
use tracing::log::trace;
|
use tracing::log::trace;
|
||||||
use tracing::{debug, error, warn, info};
|
use tracing::{debug, info, warn, error};
|
||||||
use crate::error;
|
|
||||||
|
|
||||||
pub type PostgresPool = sqlx::pool::Pool<Postgres>;
|
pub type PostgresPool = sqlx::pool::Pool<Postgres>;
|
||||||
|
|
||||||
|
@ -78,7 +78,6 @@ async fn delete_expired(conn:PostgresPool) -> Result<u64> {
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl NostrRepo for PostgresRepo {
|
impl NostrRepo for PostgresRepo {
|
||||||
|
|
||||||
async fn start(&self) -> Result<()> {
|
async fn start(&self) -> Result<()> {
|
||||||
// begin a cleanup task for expired events.
|
// begin a cleanup task for expired events.
|
||||||
cleanup_expired(self.conn.clone(), Duration::from_secs(600)).await?;
|
cleanup_expired(self.conn.clone(), Duration::from_secs(600)).await?;
|
||||||
|
@ -139,7 +138,7 @@ impl NostrRepo for PostgresRepo {
|
||||||
// the same author/kind/tag value exist, and we can ignore
|
// the same author/kind/tag value exist, and we can ignore
|
||||||
// this event.
|
// this event.
|
||||||
if repl_count > 0 {
|
if repl_count > 0 {
|
||||||
return Ok(0)
|
return Ok(0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// ignore if the event hash is a duplicate.
|
// ignore if the event hash is a duplicate.
|
||||||
|
@ -159,7 +158,6 @@ ON CONFLICT (id) DO NOTHING"#,
|
||||||
.execute(&mut tx)
|
.execute(&mut tx)
|
||||||
.await?
|
.await?
|
||||||
.rows_affected();
|
.rows_affected();
|
||||||
|
|
||||||
if ins_count == 0 {
|
if ins_count == 0 {
|
||||||
// if the event was a duplicate, no need to insert event or
|
// if the event was a duplicate, no need to insert event or
|
||||||
// pubkey references. This will abort the txn.
|
// pubkey references. This will abort the txn.
|
||||||
|
@ -380,7 +378,10 @@ ON CONFLICT (id) DO NOTHING"#,
|
||||||
|
|
||||||
// check if this is still active; every 100 rows
|
// check if this is still active; every 100 rows
|
||||||
if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
|
if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
|
||||||
debug!("query cancelled by client (cid: {}, sub: {:?})", client_id, sub.id);
|
debug!(
|
||||||
|
"query cancelled by client (cid: {}, sub: {:?})",
|
||||||
|
client_id, sub.id
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -396,7 +397,10 @@ ON CONFLICT (id) DO NOTHING"#,
|
||||||
if last_successful_send + abort_cutoff < Instant::now() {
|
if last_successful_send + abort_cutoff < Instant::now() {
|
||||||
// the queue has been full for too long, abort
|
// the queue has been full for too long, abort
|
||||||
info!("aborting database query due to slow client");
|
info!("aborting database query due to slow client");
|
||||||
metrics.query_aborts.with_label_values(&["slowclient"]).inc();
|
metrics
|
||||||
|
.query_aborts
|
||||||
|
.with_label_values(&["slowclient"])
|
||||||
|
.inc();
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
// give the queue a chance to clear before trying again
|
// give the queue a chance to clear before trying again
|
||||||
|
@ -467,9 +471,7 @@ ON CONFLICT (id) DO NOTHING"#,
|
||||||
let verify_time = now_jitter(600);
|
let verify_time = now_jitter(600);
|
||||||
|
|
||||||
// update verification time and reset any failure count
|
// update verification time and reset any failure count
|
||||||
sqlx::query(
|
sqlx::query("UPDATE user_verification SET verified_at = $1, fail_count = 0 WHERE id = $2")
|
||||||
"UPDATE user_verification SET verified_at = $1, fail_count = 0 WHERE id = $2",
|
|
||||||
)
|
|
||||||
.bind(Utc.timestamp_opt(verify_time as i64, 0).unwrap())
|
.bind(Utc.timestamp_opt(verify_time as i64, 0).unwrap())
|
||||||
.bind(id as i64)
|
.bind(id as i64)
|
||||||
.execute(&self.conn)
|
.execute(&self.conn)
|
||||||
|
@ -767,8 +769,7 @@ fn query_from_filter(f: &ReqFilter) -> Option<QueryBuilder<Postgres>> {
|
||||||
|
|
||||||
impl FromRow<'_, PgRow> for VerificationRecord {
|
impl FromRow<'_, PgRow> for VerificationRecord {
|
||||||
fn from_row(row: &'_ PgRow) -> std::result::Result<Self, Error> {
|
fn from_row(row: &'_ PgRow) -> std::result::Result<Self, Error> {
|
||||||
let name =
|
let name = Nip05Name::try_from(row.get::<'_, &str, &str>("name")).or(Err(RowNotFound))?;
|
||||||
Nip05Name::try_from(row.get::<'_, &str, &str>("name")).or(Err(RowNotFound))?;
|
|
||||||
Ok(VerificationRecord {
|
Ok(VerificationRecord {
|
||||||
rowid: row.get::<'_, i64, &str>("id") as u64,
|
rowid: row.get::<'_, i64, &str>("id") as u64,
|
||||||
name,
|
name,
|
||||||
|
|
|
@ -1,32 +1,33 @@
|
||||||
//! Event persistence and querying
|
//! Event persistence and querying
|
||||||
//use crate::config::SETTINGS;
|
//use crate::config::SETTINGS;
|
||||||
use crate::config::Settings;
|
use crate::config::Settings;
|
||||||
use crate::error::{Result,Error::SqlError};
|
use crate::db::QueryResult;
|
||||||
|
use crate::error::Result;
|
||||||
|
use crate::error::Error::SqlError;
|
||||||
use crate::event::{single_char_tagname, Event};
|
use crate::event::{single_char_tagname, Event};
|
||||||
use crate::hexrange::hex_range;
|
use crate::hexrange::hex_range;
|
||||||
use crate::hexrange::HexSearch;
|
use crate::hexrange::HexSearch;
|
||||||
use crate::repo::sqlite_migration::{STARTUP_SQL,upgrade_db};
|
|
||||||
use crate::utils::{is_hex,unix_time};
|
|
||||||
use crate::nip05::{Nip05Name, VerificationRecord};
|
use crate::nip05::{Nip05Name, VerificationRecord};
|
||||||
use crate::subscription::{ReqFilter, Subscription};
|
use crate::repo::sqlite_migration::{upgrade_db, STARTUP_SQL};
|
||||||
use crate::server::NostrMetrics;
|
use crate::server::NostrMetrics;
|
||||||
|
use crate::subscription::{ReqFilter, Subscription};
|
||||||
|
use crate::utils::{is_hex, unix_time};
|
||||||
|
use async_trait::async_trait;
|
||||||
use hex;
|
use hex;
|
||||||
use r2d2;
|
use r2d2;
|
||||||
use r2d2_sqlite::SqliteConnectionManager;
|
use r2d2_sqlite::SqliteConnectionManager;
|
||||||
use rusqlite::params;
|
use rusqlite::params;
|
||||||
use rusqlite::types::ToSql;
|
use rusqlite::types::ToSql;
|
||||||
use rusqlite::OpenFlags;
|
use rusqlite::OpenFlags;
|
||||||
use tokio::sync::{Mutex, MutexGuard, Semaphore};
|
|
||||||
use std::fmt::Write as _;
|
use std::fmt::Write as _;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
use tokio::sync::{Mutex, MutexGuard, Semaphore};
|
||||||
use tokio::task;
|
use tokio::task;
|
||||||
use tracing::{debug, info, trace, warn};
|
use tracing::{debug, info, trace, warn};
|
||||||
use async_trait::async_trait;
|
|
||||||
use crate::db::QueryResult;
|
|
||||||
|
|
||||||
use crate::repo::{now_jitter, NostrRepo};
|
use crate::repo::{now_jitter, NostrRepo};
|
||||||
|
|
||||||
|
@ -54,7 +55,8 @@ pub struct SqliteRepo {
|
||||||
|
|
||||||
impl SqliteRepo {
|
impl SqliteRepo {
|
||||||
// build all the pools needed
|
// build all the pools needed
|
||||||
#[must_use] pub fn new(settings: &Settings, metrics: NostrMetrics) -> SqliteRepo {
|
#[must_use]
|
||||||
|
pub fn new(settings: &Settings, metrics: NostrMetrics) -> SqliteRepo {
|
||||||
let write_pool = build_pool(
|
let write_pool = build_pool(
|
||||||
"writer",
|
"writer",
|
||||||
settings,
|
settings,
|
||||||
|
@ -110,7 +112,8 @@ impl SqliteRepo {
|
||||||
// get relevant fields from event and convert to blobs.
|
// get relevant fields from event and convert to blobs.
|
||||||
let id_blob = hex::decode(&e.id).ok();
|
let id_blob = hex::decode(&e.id).ok();
|
||||||
let pubkey_blob: Option<Vec<u8>> = hex::decode(&e.pubkey).ok();
|
let pubkey_blob: Option<Vec<u8>> = hex::decode(&e.pubkey).ok();
|
||||||
let delegator_blob: Option<Vec<u8>> = e.delegated_by.as_ref().and_then(|d| hex::decode(d).ok());
|
let delegator_blob: Option<Vec<u8>> =
|
||||||
|
e.delegated_by.as_ref().and_then(|d| hex::decode(d).ok());
|
||||||
let event_str = serde_json::to_string(&e).ok();
|
let event_str = serde_json::to_string(&e).ok();
|
||||||
// check for replaceable events that would hide this one; we won't even attempt to insert these.
|
// check for replaceable events that would hide this one; we won't even attempt to insert these.
|
||||||
if e.is_replaceable() {
|
if e.is_replaceable() {
|
||||||
|
@ -130,7 +133,7 @@ impl SqliteRepo {
|
||||||
// the same author/kind/tag value exist, and we can ignore
|
// the same author/kind/tag value exist, and we can ignore
|
||||||
// this event.
|
// this event.
|
||||||
if repl_count.ok().is_some() {
|
if repl_count.ok().is_some() {
|
||||||
return Ok(0)
|
return Ok(0);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// ignore if the event hash is a duplicate.
|
// ignore if the event hash is a duplicate.
|
||||||
|
@ -249,21 +252,24 @@ impl SqliteRepo {
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl NostrRepo for SqliteRepo {
|
impl NostrRepo for SqliteRepo {
|
||||||
|
|
||||||
async fn start(&self) -> Result<()> {
|
async fn start(&self) -> Result<()> {
|
||||||
db_checkpoint_task(self.maint_pool.clone(), Duration::from_secs(60),
|
db_checkpoint_task(
|
||||||
|
self.maint_pool.clone(),
|
||||||
|
Duration::from_secs(60),
|
||||||
self.write_in_progress.clone(),
|
self.write_in_progress.clone(),
|
||||||
self.checkpoint_in_progress.clone()).await?;
|
self.checkpoint_in_progress.clone()
|
||||||
cleanup_expired(self.maint_pool.clone(), Duration::from_secs(600),
|
).await?;
|
||||||
self.write_in_progress.clone()).await
|
cleanup_expired(
|
||||||
|
self.maint_pool.clone(),
|
||||||
|
Duration::from_secs(600),
|
||||||
|
self.write_in_progress.clone()
|
||||||
|
).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn migrate_up(&self) -> Result<usize> {
|
async fn migrate_up(&self) -> Result<usize> {
|
||||||
let _write_guard = self.write_in_progress.lock().await;
|
let _write_guard = self.write_in_progress.lock().await;
|
||||||
let mut conn = self.write_pool.get()?;
|
let mut conn = self.write_pool.get()?;
|
||||||
task::spawn_blocking(move || {
|
task::spawn_blocking(move || upgrade_db(&mut conn)).await?
|
||||||
upgrade_db(&mut conn)
|
|
||||||
}).await?
|
|
||||||
}
|
}
|
||||||
/// Persist event to database
|
/// Persist event to database
|
||||||
async fn write_event(&self, e: &Event) -> Result<u64> {
|
async fn write_event(&self, e: &Event) -> Result<u64> {
|
||||||
|
@ -322,7 +328,12 @@ impl NostrRepo for SqliteRepo {
|
||||||
// thread pool waiting for queries to finish under high load.
|
// thread pool waiting for queries to finish under high load.
|
||||||
// Instead, don't bother spawning threads when they will just
|
// Instead, don't bother spawning threads when they will just
|
||||||
// block on a database connection.
|
// block on a database connection.
|
||||||
let sem = self.reader_threads_ready.clone().acquire_owned().await.unwrap();
|
let sem = self
|
||||||
|
.reader_threads_ready
|
||||||
|
.clone()
|
||||||
|
.acquire_owned()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
let self = self.clone();
|
let self = self.clone();
|
||||||
let metrics = self.metrics.clone();
|
let metrics = self.metrics.clone();
|
||||||
task::spawn_blocking(move || {
|
task::spawn_blocking(move || {
|
||||||
|
@ -349,7 +360,10 @@ impl NostrRepo for SqliteRepo {
|
||||||
}
|
}
|
||||||
// check before getting a DB connection if the client still wants the results
|
// check before getting a DB connection if the client still wants the results
|
||||||
if abandon_query_rx.try_recv().is_ok() {
|
if abandon_query_rx.try_recv().is_ok() {
|
||||||
debug!("query cancelled by client (before execution) (cid: {}, sub: {:?})", client_id, sub.id);
|
debug!(
|
||||||
|
"query cancelled by client (before execution) (cid: {}, sub: {:?})",
|
||||||
|
client_id, sub.id
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -362,7 +376,9 @@ impl NostrRepo for SqliteRepo {
|
||||||
if let Ok(mut conn) = self.read_pool.get() {
|
if let Ok(mut conn) = self.read_pool.get() {
|
||||||
{
|
{
|
||||||
let pool_state = self.read_pool.state();
|
let pool_state = self.read_pool.state();
|
||||||
metrics.db_connections.set((pool_state.connections - pool_state.idle_connections).into());
|
metrics
|
||||||
|
.db_connections
|
||||||
|
.set((pool_state.connections - pool_state.idle_connections).into());
|
||||||
}
|
}
|
||||||
for filter in sub.filters.iter() {
|
for filter in sub.filters.iter() {
|
||||||
let filter_start = Instant::now();
|
let filter_start = Instant::now();
|
||||||
|
@ -379,7 +395,7 @@ impl NostrRepo for SqliteRepo {
|
||||||
let mut last_successful_send = Instant::now();
|
let mut last_successful_send = Instant::now();
|
||||||
// execute the query.
|
// execute the query.
|
||||||
// make the actual SQL query (with parameters inserted) available
|
// make the actual SQL query (with parameters inserted) available
|
||||||
conn.trace(Some(|x| {trace!("SQL trace: {:?}", x)}));
|
conn.trace(Some(|x| trace!("SQL trace: {:?}", x)));
|
||||||
let mut stmt = conn.prepare_cached(&q)?;
|
let mut stmt = conn.prepare_cached(&q)?;
|
||||||
let mut event_rows = stmt.query(rusqlite::params_from_iter(p))?;
|
let mut event_rows = stmt.query(rusqlite::params_from_iter(p))?;
|
||||||
|
|
||||||
|
@ -397,7 +413,10 @@ impl NostrRepo for SqliteRepo {
|
||||||
if slow_first_event && client_id.starts_with('0') {
|
if slow_first_event && client_id.starts_with('0') {
|
||||||
debug!(
|
debug!(
|
||||||
"filter first result in {:?} (slow): {} (cid: {}, sub: {:?})",
|
"filter first result in {:?} (slow): {} (cid: {}, sub: {:?})",
|
||||||
first_event_elapsed, serde_json::to_string(&filter)?, client_id, sub.id
|
first_event_elapsed,
|
||||||
|
serde_json::to_string(&filter)?,
|
||||||
|
client_id,
|
||||||
|
sub.id
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
first_result = false;
|
first_result = false;
|
||||||
|
@ -407,8 +426,14 @@ impl NostrRepo for SqliteRepo {
|
||||||
{
|
{
|
||||||
if self.checkpoint_in_progress.try_lock().is_err() {
|
if self.checkpoint_in_progress.try_lock().is_err() {
|
||||||
// lock was held, abort this query
|
// lock was held, abort this query
|
||||||
debug!("query aborted due to checkpoint (cid: {}, sub: {:?})", client_id, sub.id);
|
debug!(
|
||||||
metrics.query_aborts.with_label_values(&["checkpoint"]).inc();
|
"query aborted due to checkpoint (cid: {}, sub: {:?})",
|
||||||
|
client_id, sub.id
|
||||||
|
);
|
||||||
|
metrics
|
||||||
|
.query_aborts
|
||||||
|
.with_label_values(&["checkpoint"])
|
||||||
|
.inc();
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -416,7 +441,10 @@ impl NostrRepo for SqliteRepo {
|
||||||
|
|
||||||
// check if this is still active; every 100 rows
|
// check if this is still active; every 100 rows
|
||||||
if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
|
if row_count % 100 == 0 && abandon_query_rx.try_recv().is_ok() {
|
||||||
debug!("query cancelled by client (cid: {}, sub: {:?})", client_id, sub.id);
|
debug!(
|
||||||
|
"query cancelled by client (cid: {}, sub: {:?})",
|
||||||
|
client_id, sub.id
|
||||||
|
);
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
row_count += 1;
|
row_count += 1;
|
||||||
|
@ -432,19 +460,31 @@ impl NostrRepo for SqliteRepo {
|
||||||
// the queue has been full for too long, abort
|
// the queue has been full for too long, abort
|
||||||
info!("aborting database query due to slow client (cid: {}, sub: {:?})",
|
info!("aborting database query due to slow client (cid: {}, sub: {:?})",
|
||||||
client_id, sub.id);
|
client_id, sub.id);
|
||||||
metrics.query_aborts.with_label_values(&["slowclient"]).inc();
|
metrics
|
||||||
|
.query_aborts
|
||||||
|
.with_label_values(&["slowclient"])
|
||||||
|
.inc();
|
||||||
let ok: Result<()> = Ok(());
|
let ok: Result<()> = Ok(());
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
// check if a checkpoint is trying to run, and abort
|
// check if a checkpoint is trying to run, and abort
|
||||||
if self.checkpoint_in_progress.try_lock().is_err() {
|
if self.checkpoint_in_progress.try_lock().is_err() {
|
||||||
// lock was held, abort this query
|
// lock was held, abort this query
|
||||||
debug!("query aborted due to checkpoint (cid: {}, sub: {:?})", client_id, sub.id);
|
debug!(
|
||||||
metrics.query_aborts.with_label_values(&["checkpoint"]).inc();
|
"query aborted due to checkpoint (cid: {}, sub: {:?})",
|
||||||
|
client_id, sub.id
|
||||||
|
);
|
||||||
|
metrics
|
||||||
|
.query_aborts
|
||||||
|
.with_label_values(&["checkpoint"])
|
||||||
|
.inc();
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
// give the queue a chance to clear before trying again
|
// give the queue a chance to clear before trying again
|
||||||
debug!("query thread sleeping due to full query_tx (cid: {}, sub: {:?})", client_id, sub.id);
|
debug!(
|
||||||
|
"query thread sleeping due to full query_tx (cid: {}, sub: {:?})",
|
||||||
|
client_id, sub.id
|
||||||
|
);
|
||||||
thread::sleep(Duration::from_millis(500));
|
thread::sleep(Duration::from_millis(500));
|
||||||
}
|
}
|
||||||
// TODO: we could use try_send, but we'd have to juggle
|
// TODO: we could use try_send, but we'd have to juggle
|
||||||
|
@ -465,10 +505,12 @@ impl NostrRepo for SqliteRepo {
|
||||||
if filter_start.elapsed() > slow_cutoff && client_id.starts_with('0') {
|
if filter_start.elapsed() > slow_cutoff && client_id.starts_with('0') {
|
||||||
debug!(
|
debug!(
|
||||||
"query filter req (slow): {} (cid: {}, sub: {:?}, filter: {})",
|
"query filter req (slow): {} (cid: {}, sub: {:?}, filter: {})",
|
||||||
serde_json::to_string(&filter)?, client_id, sub.id, filter_count
|
serde_json::to_string(&filter)?,
|
||||||
|
client_id,
|
||||||
|
sub.id,
|
||||||
|
filter_count
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
warn!("Could not get a database connection for querying");
|
warn!("Could not get a database connection for querying");
|
||||||
|
@ -504,7 +546,8 @@ impl NostrRepo for SqliteRepo {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
conn.execute_batch("PRAGMA optimize;").ok();
|
conn.execute_batch("PRAGMA optimize;").ok();
|
||||||
info!("optimize ran in {:?}", start.elapsed());
|
info!("optimize ran in {:?}", start.elapsed());
|
||||||
}).await?;
|
})
|
||||||
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -558,7 +601,6 @@ impl NostrRepo for SqliteRepo {
|
||||||
ok
|
ok
|
||||||
})
|
})
|
||||||
.await?
|
.await?
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Update verification record as failed
|
/// Update verification record as failed
|
||||||
|
@ -684,12 +726,13 @@ fn override_index(f: &ReqFilter) -> Option<String> {
|
||||||
// queries for multiple kinds default to kind_index, which is
|
// queries for multiple kinds default to kind_index, which is
|
||||||
// significantly slower than kind_created_at_index.
|
// significantly slower than kind_created_at_index.
|
||||||
if let Some(ks) = &f.kinds {
|
if let Some(ks) = &f.kinds {
|
||||||
if f.ids.is_none() &&
|
if f.ids.is_none()
|
||||||
ks.len() > 1 &&
|
&& ks.len() > 1
|
||||||
f.since.is_none() &&
|
&& f.since.is_none()
|
||||||
f.until.is_none() &&
|
&& f.until.is_none()
|
||||||
f.tags.is_none() &&
|
&& f.tags.is_none()
|
||||||
f.authors.is_none() {
|
&& f.authors.is_none()
|
||||||
|
{
|
||||||
return Some("kind_created_at_index".into());
|
return Some("kind_created_at_index".into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -726,7 +769,9 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
|
||||||
|
|
||||||
// check if the index needs to be overriden
|
// check if the index needs to be overriden
|
||||||
let idx_name = override_index(f);
|
let idx_name = override_index(f);
|
||||||
let idx_stmt = idx_name.as_ref().map_or_else(|| "".to_owned(), |i| format!("INDEXED BY {i}"));
|
let idx_stmt = idx_name
|
||||||
|
.as_ref()
|
||||||
|
.map_or_else(|| "".to_owned(), |i| format!("INDEXED BY {i}"));
|
||||||
let mut query = format!("SELECT e.content FROM event e {idx_stmt}");
|
let mut query = format!("SELECT e.content FROM event e {idx_stmt}");
|
||||||
// query parameters for SQLite
|
// query parameters for SQLite
|
||||||
let mut params: Vec<Box<dyn ToSql>> = vec![];
|
let mut params: Vec<Box<dyn ToSql>> = vec![];
|
||||||
|
@ -744,9 +789,7 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
|
||||||
params.push(Box::new(ex));
|
params.push(Box::new(ex));
|
||||||
}
|
}
|
||||||
Some(HexSearch::Range(lower, upper)) => {
|
Some(HexSearch::Range(lower, upper)) => {
|
||||||
auth_searches.push(
|
auth_searches.push("(author>? AND author<?)".to_owned());
|
||||||
"(author>? AND author<?)".to_owned(),
|
|
||||||
);
|
|
||||||
params.push(Box::new(lower));
|
params.push(Box::new(lower));
|
||||||
params.push(Box::new(upper));
|
params.push(Box::new(upper));
|
||||||
}
|
}
|
||||||
|
@ -1003,7 +1046,11 @@ pub fn delete_expired(conn: &mut PooledConnection) -> Result<usize> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Perform database WAL checkpoint on a regular basis
|
/// Perform database WAL checkpoint on a regular basis
|
||||||
pub async fn db_checkpoint_task(pool: SqlitePool, frequency: Duration, write_in_progress: Arc<Mutex<u64>>, checkpoint_in_progress: Arc<Mutex<u64>>) -> Result<()> {
|
pub async fn db_checkpoint_task(
|
||||||
|
pool: SqlitePool,
|
||||||
|
frequency: Duration,
|
||||||
|
write_in_progress: Arc<Mutex<u64>>,
|
||||||
|
checkpoint_in_progress: Arc<Mutex<u64>>) -> Result<()> {
|
||||||
// TODO; use acquire_many on the reader semaphore to stop them from interrupting this.
|
// TODO; use acquire_many on the reader semaphore to stop them from interrupting this.
|
||||||
tokio::task::spawn(async move {
|
tokio::task::spawn(async move {
|
||||||
// WAL size in pages.
|
// WAL size in pages.
|
||||||
|
@ -1079,7 +1126,6 @@ pub fn checkpoint_db(conn: &mut PooledConnection) -> Result<usize> {
|
||||||
Ok(wal_size as usize)
|
Ok(wal_size as usize)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Produce a arbitrary list of '?' parameters.
|
/// Produce a arbitrary list of '?' parameters.
|
||||||
fn repeat_vars(count: usize) -> String {
|
fn repeat_vars(count: usize) -> String {
|
||||||
if count == 0 {
|
if count == 0 {
|
||||||
|
@ -1113,7 +1159,6 @@ fn log_pool_stats(name: &str, pool: &SqlitePool) {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// Check if the pool is fully utilized
|
/// Check if the pool is fully utilized
|
||||||
fn _pool_at_capacity(pool: &SqlitePool) -> bool {
|
fn _pool_at_capacity(pool: &SqlitePool) -> bool {
|
||||||
let state: r2d2::State = pool.state();
|
let state: r2d2::State = pool.state();
|
||||||
|
|
|
@ -293,8 +293,6 @@ pub fn rebuild_tags(conn: &mut PooledConnection) -> Result<()> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
//// Migration Scripts
|
//// Migration Scripts
|
||||||
|
|
||||||
fn mig_1_to_2(conn: &mut PooledConnection) -> Result<usize> {
|
fn mig_1_to_2(conn: &mut PooledConnection) -> Result<usize> {
|
||||||
|
@ -586,11 +584,17 @@ fn mig_11_to_12(conn: &mut PooledConnection) -> Result<usize> {
|
||||||
tx.execute("PRAGMA user_version = 12;", [])?;
|
tx.execute("PRAGMA user_version = 12;", [])?;
|
||||||
}
|
}
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
info!("database schema upgraded v11 -> v12 in {:?}", start.elapsed());
|
info!(
|
||||||
|
"database schema upgraded v11 -> v12 in {:?}",
|
||||||
|
start.elapsed()
|
||||||
|
);
|
||||||
// vacuum after large table modification
|
// vacuum after large table modification
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
conn.execute("VACUUM;", [])?;
|
conn.execute("VACUUM;", [])?;
|
||||||
info!("vacuumed DB after hidden event cleanup in {:?}", start.elapsed());
|
info!(
|
||||||
|
"vacuumed DB after hidden event cleanup in {:?}",
|
||||||
|
start.elapsed()
|
||||||
|
);
|
||||||
Ok(12)
|
Ok(12)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -656,7 +660,7 @@ PRAGMA user_version = 15;
|
||||||
match conn.execute_batch(clear_hidden_sql) {
|
match conn.execute_batch(clear_hidden_sql) {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
info!("all hidden events removed");
|
info!("all hidden events removed");
|
||||||
},
|
}
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
error!("delete failed: {}", err);
|
error!("delete failed: {}", err);
|
||||||
panic!("could not remove hidden events");
|
panic!("could not remove hidden events");
|
||||||
|
|
|
@ -3,7 +3,6 @@ use crate::close::Close;
|
||||||
use crate::close::CloseCmd;
|
use crate::close::CloseCmd;
|
||||||
use crate::config::{Settings, VerifiedUsersMode};
|
use crate::config::{Settings, VerifiedUsersMode};
|
||||||
use crate::conn;
|
use crate::conn;
|
||||||
use crate::repo::NostrRepo;
|
|
||||||
use crate::db;
|
use crate::db;
|
||||||
use crate::db::SubmittedEvent;
|
use crate::db::SubmittedEvent;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
|
@ -14,10 +13,8 @@ use crate::event::EventCmd;
|
||||||
use crate::info::RelayInfo;
|
use crate::info::RelayInfo;
|
||||||
use crate::nip05;
|
use crate::nip05;
|
||||||
use crate::notice::Notice;
|
use crate::notice::Notice;
|
||||||
|
use crate::repo::NostrRepo;
|
||||||
use crate::subscription::Subscription;
|
use crate::subscription::Subscription;
|
||||||
use prometheus::IntCounterVec;
|
|
||||||
use prometheus::IntGauge;
|
|
||||||
use prometheus::{Encoder, Histogram, IntCounter, HistogramOpts, Opts, Registry, TextEncoder};
|
|
||||||
use futures::SinkExt;
|
use futures::SinkExt;
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
use governor::{Jitter, Quota, RateLimiter};
|
use governor::{Jitter, Quota, RateLimiter};
|
||||||
|
@ -28,6 +25,9 @@ use hyper::upgrade::Upgraded;
|
||||||
use hyper::{
|
use hyper::{
|
||||||
header, server::conn::AddrStream, upgrade, Body, Request, Response, Server, StatusCode,
|
header, server::conn::AddrStream, upgrade, Body, Request, Response, Server, StatusCode,
|
||||||
};
|
};
|
||||||
|
use prometheus::IntCounterVec;
|
||||||
|
use prometheus::IntGauge;
|
||||||
|
use prometheus::{Encoder, Histogram, HistogramOpts, IntCounter, Opts, Registry, TextEncoder};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
|
@ -37,9 +37,9 @@ use std::io::BufReader;
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
use std::sync::mpsc::Receiver as MpscReceiver;
|
use std::sync::mpsc::Receiver as MpscReceiver;
|
||||||
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use tokio::runtime::Builder;
|
use tokio::runtime::Builder;
|
||||||
|
@ -256,50 +256,42 @@ fn create_metrics() -> (Registry, NostrMetrics) {
|
||||||
let query_sub = Histogram::with_opts(HistogramOpts::new(
|
let query_sub = Histogram::with_opts(HistogramOpts::new(
|
||||||
"nostr_query_seconds",
|
"nostr_query_seconds",
|
||||||
"Subscription response times",
|
"Subscription response times",
|
||||||
)).unwrap();
|
))
|
||||||
|
.unwrap();
|
||||||
let query_db = Histogram::with_opts(HistogramOpts::new(
|
let query_db = Histogram::with_opts(HistogramOpts::new(
|
||||||
"nostr_filter_seconds",
|
"nostr_filter_seconds",
|
||||||
"Filter SQL query times",
|
"Filter SQL query times",
|
||||||
)).unwrap();
|
))
|
||||||
|
.unwrap();
|
||||||
let write_events = Histogram::with_opts(HistogramOpts::new(
|
let write_events = Histogram::with_opts(HistogramOpts::new(
|
||||||
"nostr_events_write_seconds",
|
"nostr_events_write_seconds",
|
||||||
"Event writing response times",
|
"Event writing response times",
|
||||||
)).unwrap();
|
))
|
||||||
|
.unwrap();
|
||||||
let sent_events = IntCounterVec::new(
|
let sent_events = IntCounterVec::new(
|
||||||
Opts::new("nostr_events_sent_total", "Events sent to clients"),
|
Opts::new("nostr_events_sent_total", "Events sent to clients"),
|
||||||
vec!["source"].as_slice(),
|
vec!["source"].as_slice(),
|
||||||
).unwrap();
|
)
|
||||||
let connections = IntCounter::with_opts(Opts::new(
|
.unwrap();
|
||||||
"nostr_connections_total",
|
let connections =
|
||||||
"New connections",
|
IntCounter::with_opts(Opts::new("nostr_connections_total", "New connections")).unwrap();
|
||||||
)).unwrap();
|
|
||||||
let db_connections = IntGauge::with_opts(Opts::new(
|
let db_connections = IntGauge::with_opts(Opts::new(
|
||||||
"nostr_db_connections", "Active database connections"
|
"nostr_db_connections",
|
||||||
)).unwrap();
|
"Active database connections",
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
let query_aborts = IntCounterVec::new(
|
let query_aborts = IntCounterVec::new(
|
||||||
Opts::new("nostr_query_abort_total", "Aborted queries"),
|
Opts::new("nostr_query_abort_total", "Aborted queries"),
|
||||||
vec!["reason"].as_slice(),
|
vec!["reason"].as_slice(),
|
||||||
).unwrap();
|
).unwrap();
|
||||||
let cmd_req = IntCounter::with_opts(Opts::new(
|
let cmd_req = IntCounter::with_opts(Opts::new("nostr_cmd_req_total", "REQ commands")).unwrap();
|
||||||
"nostr_cmd_req_total",
|
let cmd_event =
|
||||||
"REQ commands",
|
IntCounter::with_opts(Opts::new("nostr_cmd_event_total", "EVENT commands")).unwrap();
|
||||||
)).unwrap();
|
let cmd_close =
|
||||||
let cmd_event = IntCounter::with_opts(Opts::new(
|
IntCounter::with_opts(Opts::new("nostr_cmd_close_total", "CLOSE commands")).unwrap();
|
||||||
"nostr_cmd_event_total",
|
let cmd_auth = IntCounter::with_opts(Opts::new("nostr_cmd_auth_total", "AUTH commands")).unwrap();
|
||||||
"EVENT commands",
|
let disconnects = IntCounterVec::new(Opts::new("nostr_disconnects_total", "Client disconnects"),
|
||||||
)).unwrap();
|
vec!["reason"].as_slice()).unwrap();
|
||||||
let cmd_close = IntCounter::with_opts(Opts::new(
|
|
||||||
"nostr_cmd_close_total",
|
|
||||||
"CLOSE commands",
|
|
||||||
)).unwrap();
|
|
||||||
let cmd_auth = IntCounter::with_opts(Opts::new(
|
|
||||||
"nostr_cmd_auth_total",
|
|
||||||
"AUTH commands",
|
|
||||||
)).unwrap();
|
|
||||||
let disconnects = IntCounterVec::new(
|
|
||||||
Opts::new("nostr_disconnects_total", "Client disconnects"),
|
|
||||||
vec!["reason"].as_slice(),
|
|
||||||
).unwrap();
|
|
||||||
registry.register(Box::new(query_sub.clone())).unwrap();
|
registry.register(Box::new(query_sub.clone())).unwrap();
|
||||||
registry.register(Box::new(query_db.clone())).unwrap();
|
registry.register(Box::new(query_db.clone())).unwrap();
|
||||||
registry.register(Box::new(write_events.clone())).unwrap();
|
registry.register(Box::new(write_events.clone())).unwrap();
|
||||||
|
@ -383,7 +375,8 @@ pub fn start_server(settings: &Settings, shutdown_rx: MpscReceiver<()>) -> Resul
|
||||||
.enable_all()
|
.enable_all()
|
||||||
.thread_name_fn(|| {
|
.thread_name_fn(|| {
|
||||||
// give each thread a unique numeric name
|
// give each thread a unique numeric name
|
||||||
static ATOMIC_ID: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
static ATOMIC_ID: std::sync::atomic::AtomicUsize =
|
||||||
|
std::sync::atomic::AtomicUsize::new(0);
|
||||||
let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);
|
let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);
|
||||||
format!("tokio-ws-{id}")
|
format!("tokio-ws-{id}")
|
||||||
})
|
})
|
||||||
|
@ -431,8 +424,7 @@ pub fn start_server(settings: &Settings, shutdown_rx: MpscReceiver<()>) -> Resul
|
||||||
// start the database writer task. Give it a channel for
|
// start the database writer task. Give it a channel for
|
||||||
// writing events, and for publishing events that have been
|
// writing events, and for publishing events that have been
|
||||||
// written (to all connected clients).
|
// written (to all connected clients).
|
||||||
tokio::task::spawn(
|
tokio::task::spawn(db::db_writer(
|
||||||
db::db_writer(
|
|
||||||
repo.clone(),
|
repo.clone(),
|
||||||
settings.clone(),
|
settings.clone(),
|
||||||
event_rx,
|
event_rx,
|
||||||
|
@ -444,8 +436,12 @@ pub fn start_server(settings: &Settings, shutdown_rx: MpscReceiver<()>) -> Resul
|
||||||
|
|
||||||
// create a nip-05 verifier thread; if enabled.
|
// create a nip-05 verifier thread; if enabled.
|
||||||
if settings.verified_users.mode != VerifiedUsersMode::Disabled {
|
if settings.verified_users.mode != VerifiedUsersMode::Disabled {
|
||||||
let verifier_opt =
|
let verifier_opt = nip05::Verifier::new(
|
||||||
nip05::Verifier::new(repo.clone(), metadata_rx, bcast_tx.clone(), settings.clone());
|
repo.clone(),
|
||||||
|
metadata_rx,
|
||||||
|
bcast_tx.clone(),
|
||||||
|
settings.clone(),
|
||||||
|
);
|
||||||
if let Ok(mut v) = verifier_opt {
|
if let Ok(mut v) = verifier_opt {
|
||||||
if verified_users_active {
|
if verified_users_active {
|
||||||
tokio::task::spawn(async move {
|
tokio::task::spawn(async move {
|
||||||
|
@ -464,7 +460,7 @@ pub fn start_server(settings: &Settings, shutdown_rx: MpscReceiver<()>) -> Resul
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
info!("control message requesting shutdown");
|
info!("control message requesting shutdown");
|
||||||
controlled_shutdown.send(()).ok();
|
controlled_shutdown.send(()).ok();
|
||||||
},
|
}
|
||||||
Err(std::sync::mpsc::RecvError) => {
|
Err(std::sync::mpsc::RecvError) => {
|
||||||
trace!("shutdown requestor is disconnected (this is normal)");
|
trace!("shutdown requestor is disconnected (this is normal)");
|
||||||
}
|
}
|
||||||
|
@ -545,7 +541,8 @@ pub enum NostrMessage {
|
||||||
|
|
||||||
/// Convert Message to `NostrMessage`
|
/// Convert Message to `NostrMessage`
|
||||||
fn convert_to_msg(msg: &str, max_bytes: Option<usize>) -> Result<NostrMessage> {
|
fn convert_to_msg(msg: &str, max_bytes: Option<usize>) -> Result<NostrMessage> {
|
||||||
let parsed_res: Result<NostrMessage> = serde_json::from_str(msg).map_err(std::convert::Into::into);
|
let parsed_res: Result<NostrMessage> =
|
||||||
|
serde_json::from_str(msg).map_err(std::convert::Into::into);
|
||||||
match parsed_res {
|
match parsed_res {
|
||||||
Ok(m) => {
|
Ok(m) => {
|
||||||
if let NostrMessage::SubMsg(_) = m {
|
if let NostrMessage::SubMsg(_) = m {
|
||||||
|
@ -786,6 +783,7 @@ async fn nostr_server(
|
||||||
// handle each type of message
|
// handle each type of message
|
||||||
let evid = ec.event_id().to_owned();
|
let evid = ec.event_id().to_owned();
|
||||||
let parsed : Result<EventWrapper> = Result::<EventWrapper>::from(ec);
|
let parsed : Result<EventWrapper> = Result::<EventWrapper>::from(ec);
|
||||||
|
metrics.cmd_event.inc();
|
||||||
match parsed {
|
match parsed {
|
||||||
Ok(WrappedEvent(e)) => {
|
Ok(WrappedEvent(e)) => {
|
||||||
metrics.cmd_event.inc();
|
metrics.cmd_event.inc();
|
||||||
|
@ -955,5 +953,4 @@ pub struct NostrMetrics {
|
||||||
pub cmd_event: IntCounter, // count of EVENT commands received
|
pub cmd_event: IntCounter, // count of EVENT commands received
|
||||||
pub cmd_close: IntCounter, // count of CLOSE commands received
|
pub cmd_close: IntCounter, // count of CLOSE commands received
|
||||||
pub cmd_auth: IntCounter, // count of AUTH commands received
|
pub cmd_auth: IntCounter, // count of AUTH commands received
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -45,7 +45,8 @@ pub struct ReqFilter {
|
||||||
|
|
||||||
impl Serialize for ReqFilter {
|
impl Serialize for ReqFilter {
|
||||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||||
where S:Serializer,
|
where
|
||||||
|
S: Serializer,
|
||||||
{
|
{
|
||||||
let mut map = serializer.serialize_map(None)?;
|
let mut map = serializer.serialize_map(None)?;
|
||||||
if let Some(ids) = &self.ids {
|
if let Some(ids) = &self.ids {
|
||||||
|
@ -110,7 +111,8 @@ impl<'de> Deserialize<'de> for ReqFilter {
|
||||||
if a.contains(&empty_string) {
|
if a.contains(&empty_string) {
|
||||||
return Err(serde::de::Error::invalid_type(
|
return Err(serde::de::Error::invalid_type(
|
||||||
Unexpected::Other("prefix matches must not be empty strings"),
|
Unexpected::Other("prefix matches must not be empty strings"),
|
||||||
&"a json object"));
|
&"a json object",
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rf.ids = raw_ids;
|
rf.ids = raw_ids;
|
||||||
|
@ -128,7 +130,8 @@ impl<'de> Deserialize<'de> for ReqFilter {
|
||||||
if a.contains(&empty_string) {
|
if a.contains(&empty_string) {
|
||||||
return Err(serde::de::Error::invalid_type(
|
return Err(serde::de::Error::invalid_type(
|
||||||
Unexpected::Other("prefix matches must not be empty strings"),
|
Unexpected::Other("prefix matches must not be empty strings"),
|
||||||
&"a json object"));
|
&"a json object",
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rf.authors = raw_authors;
|
rf.authors = raw_authors;
|
||||||
|
@ -232,19 +235,22 @@ impl<'de> Deserialize<'de> for Subscription {
|
||||||
|
|
||||||
impl Subscription {
|
impl Subscription {
|
||||||
/// Get a copy of the subscription identifier.
|
/// Get a copy of the subscription identifier.
|
||||||
#[must_use] pub fn get_id(&self) -> String {
|
#[must_use]
|
||||||
|
pub fn get_id(&self) -> String {
|
||||||
self.id.clone()
|
self.id.clone()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Determine if any filter is requesting historical (database)
|
/// Determine if any filter is requesting historical (database)
|
||||||
/// queries. If every filter has limit:0, we do not need to query the DB.
|
/// queries. If every filter has limit:0, we do not need to query the DB.
|
||||||
#[must_use] pub fn needs_historical_events(&self) -> bool {
|
#[must_use]
|
||||||
|
pub fn needs_historical_events(&self) -> bool {
|
||||||
self.filters.iter().any(|f| f.limit != Some(0))
|
self.filters.iter().any(|f| f.limit != Some(0))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Determine if this subscription matches a given [`Event`]. Any
|
/// Determine if this subscription matches a given [`Event`]. Any
|
||||||
/// individual filter match is sufficient.
|
/// individual filter match is sufficient.
|
||||||
#[must_use] pub fn interested_in_event(&self, event: &Event) -> bool {
|
#[must_use]
|
||||||
|
pub fn interested_in_event(&self, event: &Event) -> bool {
|
||||||
for f in &self.filters {
|
for f in &self.filters {
|
||||||
if f.interested_in_event(event) {
|
if f.interested_in_event(event) {
|
||||||
return true;
|
return true;
|
||||||
|
@ -305,13 +311,12 @@ impl ReqFilter {
|
||||||
|
|
||||||
/// Check if this filter either matches, or does not care about the kind.
|
/// Check if this filter either matches, or does not care about the kind.
|
||||||
fn kind_match(&self, kind: u64) -> bool {
|
fn kind_match(&self, kind: u64) -> bool {
|
||||||
self.kinds
|
self.kinds.as_ref().map_or(true, |ks| ks.contains(&kind))
|
||||||
.as_ref()
|
|
||||||
.map_or(true, |ks| ks.contains(&kind))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Determine if all populated fields in this filter match the provided event.
|
/// Determine if all populated fields in this filter match the provided event.
|
||||||
#[must_use] pub fn interested_in_event(&self, event: &Event) -> bool {
|
#[must_use]
|
||||||
|
pub fn interested_in_event(&self, event: &Event) -> bool {
|
||||||
// self.id.as_ref().map(|v| v == &event.id).unwrap_or(true)
|
// self.id.as_ref().map(|v| v == &event.id).unwrap_or(true)
|
||||||
self.ids_match(event)
|
self.ids_match(event)
|
||||||
&& self.since.map_or(true, |t| event.created_at > t)
|
&& self.since.map_or(true, |t| event.created_at > t)
|
||||||
|
@ -625,7 +630,9 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn serialize_filter() -> Result<()> {
|
fn serialize_filter() -> Result<()> {
|
||||||
let s: Subscription = serde_json::from_str(r##"["REQ","xyz",{"authors":["abc", "bcd"], "since": 10, "until": 20, "limit":100, "#e": ["foo", "bar"], "#d": ["test"]}]"##)?;
|
let s: Subscription = serde_json::from_str(
|
||||||
|
r##"["REQ","xyz",{"authors":["abc", "bcd"], "since": 10, "until": 20, "limit":100, "#e": ["foo", "bar"], "#d": ["test"]}]"##,
|
||||||
|
)?;
|
||||||
let f = s.filters.get(0);
|
let f = s.filters.get(0);
|
||||||
let serialized = serde_json::to_string(&f)?;
|
let serialized = serde_json::to_string(&f)?;
|
||||||
let serialized_wrapped = format!(r##"["REQ", "xyz",{}]"##, serialized);
|
let serialized_wrapped = format!(r##"["REQ", "xyz",{}]"##, serialized);
|
||||||
|
|
|
@ -4,7 +4,8 @@ use std::time::SystemTime;
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
/// Seconds since 1970.
|
/// Seconds since 1970.
|
||||||
#[must_use] pub fn unix_time() -> u64 {
|
#[must_use]
|
||||||
|
pub fn unix_time() -> u64 {
|
||||||
SystemTime::now()
|
SystemTime::now()
|
||||||
.duration_since(SystemTime::UNIX_EPOCH)
|
.duration_since(SystemTime::UNIX_EPOCH)
|
||||||
.map(|x| x.as_secs())
|
.map(|x| x.as_secs())
|
||||||
|
@ -12,7 +13,8 @@ use url::Url;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a string contains only hex characters.
|
/// Check if a string contains only hex characters.
|
||||||
#[must_use] pub fn is_hex(s: &str) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_hex(s: &str) -> bool {
|
||||||
s.chars().all(|x| char::is_ascii_hexdigit(&x))
|
s.chars().all(|x| char::is_ascii_hexdigit(&x))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -28,7 +30,8 @@ pub fn nip19_to_hex(s: &str) -> Result<String, bech32::Error> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a string contains only lower-case hex chars.
|
/// Check if a string contains only lower-case hex chars.
|
||||||
#[must_use] pub fn is_lower_hex(s: &str) -> bool {
|
#[must_use]
|
||||||
|
pub fn is_lower_hex(s: &str) -> bool {
|
||||||
s.chars().all(|x| {
|
s.chars().all(|x| {
|
||||||
(char::is_ascii_lowercase(&x) || char::is_ascii_digit(&x)) && char::is_ascii_hexdigit(&x)
|
(char::is_ascii_lowercase(&x) || char::is_ascii_digit(&x)) && char::is_ascii_hexdigit(&x)
|
||||||
})
|
})
|
||||||
|
|
Loading…
Reference in New Issue
Block a user