fix: postgresql tag filtering for odd-length hex-looking values

The tag filtering code misses odd-length strings that contains only hex digits [0-9a-f].
This fix makes the condition for `has_plain_values` the inverse of the condition for `has_hex_values`.

Fixes #191

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nostr-rs-relay"
-version = "0.8.10"
+version = "0.8.13"
 edition = "2021"
 authors = ["Greg Heartsfield <scsibug@imap.cc>"]
 description = "A relay implementation for the Nostr protocol"
@@ -39,7 +39,7 @@ lazy_static = "1.4"
 governor = "0.4"
 nonzero_ext = "0.3"
 hyper = { version="0.14", features=["client", "server","http1","http2","tcp"] }
-hyper-tls = "0.5"
+hyper-rustls = { version = "0.24" }
 http = { version = "0.2" }
 parse_duration = "2"
 rand = "0.8"
@@ -57,6 +57,7 @@ qrcode = { version = "0.12.0", default-features = false, features = ["svg"] }
 nostr = { version = "0.18.0", default-features = false, features = ["base", "nip04", "nip19"] }
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = "0.5"
+log = "0.4"
 
 [dev-dependencies]
 anyhow = "1"

README.md

@@ -49,7 +49,7 @@ The examples below start a rootless podman container, mapping a local
 data directory and config file.
 
 ```console
-$ podman build -t nostr-rs-relay .
+$ podman build --pull -t nostr-rs-relay .
 
 $ mkdir data
 

config.toml

@@ -10,7 +10,7 @@ name = "nostr-rs-relay"
 # Description
 description = "A newly created nostr-rs-relay.\n\nCustomize this with your own info."
 
-# Administrative contact pubkey
+# Administrative contact pubkey (32-byte hex, not npub)
 #pubkey = "0c2d168a4ae8ca58c9f1ab237b5df682599c6c7ab74307ea8b05684b60405d41"
 
 # Administrative contact URI
@@ -40,7 +40,7 @@ description = "A newly created nostr-rs-relay.\n\nCustomize this with your own i
 # Use an in-memory database instead of 'nostr.db'.
 # Requires sqlite engine.
 # Caution; this will not survive a process restart!
-in_memory = true
+#in_memory = false
 
 # Database connection pool settings for subscribers:
 
@@ -75,6 +75,11 @@ in_memory = true
 # `proto/nauthz.proto`.
 # event_admission_server = "http://[::1]:50051"
 
+# If the event admission server denies writes
+# in any case (excluding spam filtering).
+# This is reflected in the relay information document.
+# restricts_write = true
+
 [network]
 # Bind to this network address
 address = "0.0.0.0"
@@ -150,6 +155,11 @@ reject_future_seconds = 1800
 #    0, 1, 2, 3, 7, 40, 41, 42, 43, 44, 30023,
 #]
 
+# Rejects imprecise requests (kind only and author only etc)
+# This is a temperary measure to improve the adoption of outbox model
+# Its recommended to have this enabled
+limit_scrapers = false
+
 [authorization]
 # Pubkey addresses in this array are whitelisted for event publishing.
 # Only valid events by these authors will be accepted, if the variable
@@ -160,7 +170,7 @@ reject_future_seconds = 1800
 #]
 # Enable NIP-42 authentication
 #nip42_auth = false
-# Send DMs events (kind 4) only to their authenticated recipients
+# Send DMs (kind 4 and 44) and gift wraps (kind 1059) only to their authenticated recipients
 #nip42_dms = false
 
 [verified_users]
@@ -206,7 +216,7 @@ reject_future_seconds = 1800
 #api_secret = "<ln bits api>"
 
 # Nostr direct message on signup
-#direct_message=true
+#direct_message=false
 
 # Terms of service
 #terms_message = """

docs/run-as-linux-system-process.md

@@ -30,7 +30,8 @@ To get the service running, we need to reload the systemd daemon and enable the
 
 1. `sudo systemctl daemon-reload`
 2. `sudo systemctl start nostr-rs-relay.service`
-3. `sudo systemctl status nostr-rs-relay.service`
+3. `sudo systemctl enable nostr-rs-relay.service`
+4. `sudo systemctl status nostr-rs-relay.service`
 
 
 ### Tips

examples/nauthz/build.rs

@@ -1,6 +1,6 @@
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     tonic_build::configure()
-        .build_server(false)
+        .build_server(true)
         .protoc_arg("--experimental_allow_proto3_optional")
         .compile(&["../../proto/nauthz.proto"], &["../../proto"])?;
     Ok(())

src/bin/bulkloader.rs

@@ -143,7 +143,7 @@ fn write_event(tx: &Transaction, e: Event) -> Result<usize> {
     let event_id = tx.last_insert_rowid();
     // look at each event, and each tag, creating new tag entries if appropriate.
     for t in e.tags.iter().filter(|x| x.len() > 1) {
-        let tagname = t.get(0).unwrap();
+        let tagname = t.first().unwrap();
         let tagnamechar_opt = single_char_tagname(tagname);
         if tagnamechar_opt.is_none() {
             continue;

src/config.rs

@@ -32,6 +32,7 @@ pub struct Database {
 #[allow(unused)]
 pub struct Grpc {
     pub event_admission_server: Option<String>,
+    pub restricts_write: bool,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -73,6 +74,7 @@ pub struct Limits {
     pub event_persist_buffer: usize, // events to buffer for database commits (block senders if database writes are too slow)
     pub event_kind_blacklist: Option<Vec<u64>>,
     pub event_kind_allowlist: Option<Vec<u64>>,
+    pub limit_scrapers: bool,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -287,6 +289,7 @@ impl Default for Settings {
             },
             grpc: Grpc {
                 event_admission_server: None,
+                restricts_write: false,
             },
             network: Network {
                 port: 8080,
@@ -306,6 +309,7 @@ impl Default for Settings {
                 event_persist_buffer: 4096,
                 event_kind_blacklist: None,
                 event_kind_allowlist: None,
+                limit_scrapers: false
             },
             authorization: Authorization {
                 pubkey_whitelist: None, // Allow any address to publish
@@ -320,7 +324,7 @@ impl Default for Settings {
                 node_url: "".to_string(),
                 api_secret: "".to_string(),
                 sign_ups: false,
-                direct_message: true,
+                direct_message: false,
                 secret_key: None,
                 processor: Processor::LNBits,
             },

src/conn.rs

@@ -156,7 +156,7 @@ impl ClientConn {
         self.auth = Challenge(Uuid::new_v4().to_string());
     }
 
-    pub fn authenticate(&mut self, event: &Event, relay_url: &String) -> Result<()> {
+    pub fn authenticate(&mut self, event: &Event, relay_url: &str) -> Result<()> {
         match &self.auth {
             Challenge(_) => (),
             AuthPubkey(_) => {
@@ -181,15 +181,15 @@ impl ClientConn {
                     return Err(Error::AuthFailure);
                 }
 
-                let mut challenge: Option<&String> = None;
-                let mut relay: Option<&String> = None;
+                let mut challenge: Option<&str> = None;
+                let mut relay: Option<&str> = None;
 
                 for tag in &event.tags {
-                    if tag.len() == 2 && tag.get(0) == Some(&"challenge".into()) {
-                        challenge = tag.get(1);
+                    if tag.len() == 2 && tag.first() == Some(&"challenge".into()) {
+                        challenge = tag.get(1).map(|x| x.as_str());
                     }
-                    if tag.len() == 2 && tag.get(0) == Some(&"relay".into()) {
-                        relay = tag.get(1);
+                    if tag.len() == 2 && tag.first() == Some(&"relay".into()) {
+                        relay = tag.get(1).map(|x| x.as_str());
                     }
                 }
 

src/db.rs

@@ -11,6 +11,7 @@ use crate::repo::NostrRepo;
 use crate::server::NostrMetrics;
 use governor::clock::Clock;
 use governor::{Quota, RateLimiter};
+use log::LevelFilter;
 use nostr::key::FromPkStr;
 use nostr::key::Keys;
 use r2d2;
@@ -20,7 +21,6 @@ use sqlx::ConnectOptions;
 use std::sync::Arc;
 use std::thread;
 use std::time::{Duration, Instant};
-use tracing::log::LevelFilter;
 use tracing::{debug, info, trace, warn};
 
 pub type SqlitePool = r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>;
@@ -261,7 +261,7 @@ pub async fn db_writer(
                     ) => {
                         // User does not exist
                         info!("Unregistered user");
-                        if settings.pay_to_relay.sign_ups {
+                        if settings.pay_to_relay.sign_ups && settings.pay_to_relay.direct_message {
                             payment_tx
                                 .send(PaymentMessage::NewAccount(event.pubkey))
                                 .ok();
@@ -401,6 +401,9 @@ pub async fn db_writer(
                 start.elapsed()
             );
             event_write = true;
+
+            // send OK message
+            notice_tx.try_send(Notice::saved(event.id)).ok();
         } else {
             match repo.write_event(&event).await {
                 Ok(updated) => {

src/event.rs

@@ -160,11 +160,11 @@ impl Event {
             .tags
             .iter()
             .filter(|x| !x.is_empty())
-            .filter(|x| x.get(0).unwrap() == "expiration")
+            .filter(|x| x.first().unwrap() == "expiration")
             .map(|x| x.get(1).unwrap_or(&default))
             .take(1)
             .collect();
-        let val_first = dvals.get(0);
+        let val_first = dvals.first();
         val_first.and_then(|t| t.parse::<u64>().ok())
     }
 
@@ -192,11 +192,11 @@ impl Event {
                 .tags
                 .iter()
                 .filter(|x| !x.is_empty())
-                .filter(|x| x.get(0).unwrap() == "d")
+                .filter(|x| x.first().unwrap() == "d")
                 .map(|x| x.get(1).unwrap_or(&default))
                 .take(1)
                 .collect();
-            let dval_first = dvals.get(0);
+            let dval_first = dvals.first();
             match dval_first {
                 Some(_) => dval_first.map(|x| x.to_string()),
                 None => Some(default),
@@ -232,7 +232,7 @@ impl Event {
             .tags
             .iter()
             .filter(|x| x.len() == 4)
-            .filter(|x| x.get(0).unwrap() == "delegation")
+            .filter(|x| x.first().unwrap() == "delegation")
             .take(1)
             .next()?
             .clone(); // get first tag
@@ -277,7 +277,7 @@ impl Event {
         let mut idx: HashMap<char, HashSet<String>> = HashMap::new();
         // iterate over tags that have at least 2 elements
         for t in self.tags.iter().filter(|x| x.len() > 1) {
-            let tagname = t.get(0).unwrap();
+            let tagname = t.first().unwrap();
             let tagnamechar_opt = single_char_tagname(tagname);
             if tagnamechar_opt.is_none() {
                 continue;
@@ -285,7 +285,7 @@ impl Event {
             let tagnamechar = tagnamechar_opt.unwrap();
             let tagval = t.get(1).unwrap();
             // ensure a vector exists for this tag
-            idx.entry(tagnamechar).or_insert_with(HashSet::new);
+            idx.entry(tagnamechar).or_default();
             // get the tag vec and insert entry
             let idx_tag_vec = idx.get_mut(&tagnamechar).expect("could not get tag vector");
             idx_tag_vec.insert(tagval.clone());
@@ -310,7 +310,7 @@ impl Event {
         self.tags
             .iter()
             .filter(|x| x.len() > 1)
-            .filter(|x| x.get(0).unwrap() == tag_name)
+            .filter(|x| x.first().unwrap() == tag_name)
             .map(|x| x.get(1).unwrap().clone())
             .collect()
     }
@@ -355,7 +355,7 @@ impl Event {
             return Err(EventInvalidId);
         }
         // * validate the message digest (sig) using the pubkey & computed sha256 message hash.
-        let sig = schnorr::Signature::from_str(&self.sig).unwrap();
+        let sig = schnorr::Signature::from_str(&self.sig).map_err(|_| EventInvalidSignature)?;
         if let Ok(msg) = secp256k1::Message::from_slice(digest.as_ref()) {
             if let Ok(pubkey) = XOnlyPublicKey::from_str(&self.pubkey) {
                 SECP.verify_schnorr(&sig, &msg, &pubkey)

src/hexrange.rs

@@ -1,159 +0,0 @@
-//! Utilities for searching hexadecimal
-use crate::utils::is_hex;
-use hex;
-
-/// Types of hexadecimal queries.
-#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone)]
-pub enum HexSearch {
-    // when no range is needed, exact 32-byte
-    Exact(Vec<u8>),
-    // lower (inclusive) and upper range (exclusive)
-    Range(Vec<u8>, Vec<u8>),
-    // lower bound only, upper bound is MAX inclusive
-    LowerOnly(Vec<u8>),
-}
-
-/// Check if a string contains only f chars
-fn is_all_fs(s: &str) -> bool {
-    s.chars().all(|x| x == 'f' || x == 'F')
-}
-
-/// Find the next hex sequence greater than the argument.
-#[must_use]
-pub fn hex_range(s: &str) -> Option<HexSearch> {
-    let mut hash_base = s.to_owned();
-    if !is_hex(&hash_base) || hash_base.len() > 64 {
-        return None;
-    }
-    if hash_base.len() == 64 {
-        return Some(HexSearch::Exact(hex::decode(&hash_base).ok()?));
-    }
-    // if s is odd, add a zero
-    let mut odd = hash_base.len() % 2 != 0;
-    if odd {
-        // extend the string to make it even
-        hash_base.push('0');
-    }
-    let base = hex::decode(hash_base).ok()?;
-    // check for all ff's
-    if is_all_fs(s) {
-        // there is no higher bound, we only want to search for blobs greater than this.
-        return Some(HexSearch::LowerOnly(base));
-    }
-
-    // return a range
-    let mut upper = base.clone();
-    let mut byte_len = upper.len();
-
-    // for odd strings, we made them longer, but we want to increment the upper char (+16).
-    // we know we can do this without overflowing because we explicitly set the bottom half to 0's.
-    while byte_len > 0 {
-        byte_len -= 1;
-        // check if byte can be incremented, or if we need to carry.
-        let b = upper[byte_len];
-        if b == u8::MAX {
-            // reset and carry
-            upper[byte_len] = 0;
-        } else if odd {
-            // check if first char in this byte is NOT 'f'
-            if b < 240 {
-                // bump up the first character in this byte
-                upper[byte_len] = b + 16;
-                // increment done, stop iterating through the vec
-                break;
-            }
-            // if it is 'f', reset the byte to 0 and do a carry
-            // reset and carry
-            upper[byte_len] = 0;
-            // done with odd logic, so don't repeat this
-            odd = false;
-        } else {
-            // bump up the first character in this byte
-            upper[byte_len] = b + 1;
-            // increment done, stop iterating
-            break;
-        }
-    }
-    Some(HexSearch::Range(base, upper))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::error::Result;
-
-    #[test]
-    fn hex_range_exact() -> Result<()> {
-        let hex = "abcdef00abcdef00abcdef00abcdef00abcdef00abcdef00abcdef00abcdef00";
-        let r = hex_range(hex);
-        assert_eq!(
-            r,
-            Some(HexSearch::Exact(hex::decode(hex).expect("invalid hex")))
-        );
-        Ok(())
-    }
-    #[test]
-    fn hex_full_range() -> Result<()> {
-        let hex = "aaaa";
-        let hex_upper = "aaab";
-        let r = hex_range(hex);
-        assert_eq!(
-            r,
-            Some(HexSearch::Range(
-                hex::decode(hex).expect("invalid hex"),
-                hex::decode(hex_upper).expect("invalid hex")
-            ))
-        );
-        Ok(())
-    }
-
-    #[test]
-    fn hex_full_range_odd() -> Result<()> {
-        let r = hex_range("abc");
-        assert_eq!(
-            r,
-            Some(HexSearch::Range(
-                hex::decode("abc0").expect("invalid hex"),
-                hex::decode("abd0").expect("invalid hex")
-            ))
-        );
-        Ok(())
-    }
-
-    #[test]
-    fn hex_full_range_odd_end_f() -> Result<()> {
-        let r = hex_range("abf");
-        assert_eq!(
-            r,
-            Some(HexSearch::Range(
-                hex::decode("abf0").expect("invalid hex"),
-                hex::decode("ac00").expect("invalid hex")
-            ))
-        );
-        Ok(())
-    }
-
-    #[test]
-    fn hex_no_upper() -> Result<()> {
-        let r = hex_range("ffff");
-        assert_eq!(
-            r,
-            Some(HexSearch::LowerOnly(
-                hex::decode("ffff").expect("invalid hex")
-            ))
-        );
-        Ok(())
-    }
-
-    #[test]
-    fn hex_no_upper_odd() -> Result<()> {
-        let r = hex_range("fff");
-        assert_eq!(
-            r,
-            Some(HexSearch::LowerOnly(
-                hex::decode("fff0").expect("invalid hex")
-            ))
-        );
-        Ok(())
-    }
-}

src/info.rs

@@ -4,7 +4,7 @@ use crate::config::Settings;
 use serde::{Deserialize, Serialize};
 
 pub const CARGO_PKG_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION");
-pub const UNIT: &str = "sats";
+pub const UNIT: &str = "msats";
 
 /// Limitations of the relay as specified in NIP-111
 /// (This nip isn't finalized so may change)
@@ -13,6 +13,9 @@ pub const UNIT: &str = "sats";
 pub struct Limitation {
     #[serde(skip_serializing_if = "Option::is_none")]
     payment_required: Option<bool>,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    restricted_writes: Option<bool>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -45,7 +48,7 @@ pub struct RelayInfo {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub contact: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub icon: Option<String>,        
+    pub icon: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub supported_nips: Option<Vec<i64>>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -63,7 +66,7 @@ pub struct RelayInfo {
 /// Convert an Info configuration into public Relay Info
 impl From<Settings> for RelayInfo {
     fn from(c: Settings) -> Self {
-        let mut supported_nips = vec![1, 2, 9, 11, 12, 15, 16, 20, 22, 33, 40, 42];
+        let mut supported_nips = vec![1, 2, 9, 11, 12, 15, 16, 20, 22, 33, 40];
 
         if c.authorization.nip42_auth {
             supported_nips.push(42);
@@ -75,12 +78,18 @@ impl From<Settings> for RelayInfo {
 
         let limitations = Limitation {
             payment_required: Some(p.enabled),
+            restricted_writes: Some(
+                p.enabled
+                    || c.verified_users.is_enabled()
+                    || c.authorization.pubkey_whitelist.is_some()
+                    || c.grpc.restricts_write,
+            ),
         };
 
         let (payment_url, fees) = if p.enabled {
             let admission_fee = if p.admission_cost > 0 {
                 Some(vec![Fee {
-                    amount: p.admission_cost,
+                    amount: p.admission_cost * 1000,
                     unit: UNIT.to_string(),
                 }])
             } else {
@@ -89,7 +98,7 @@ impl From<Settings> for RelayInfo {
 
             let post_fee = if p.cost_per_event > 0 {
                 Some(vec![Fee {
-                    amount: p.cost_per_event,
+                    amount: p.cost_per_event * 1000,
                     unit: UNIT.to_string(),
                 }])
             } else {

src/lib.rs

@@ -6,7 +6,6 @@ pub mod db;
 pub mod delegation;
 pub mod error;
 pub mod event;
-pub mod hexrange;
 pub mod info;
 pub mod nauthz;
 pub mod nip05;

src/nauthz.rs

@@ -41,7 +41,7 @@ impl std::convert::From<Nip05Name> for nauthz_grpc::event_request::Nip05Name {
 }
 
 // conversion of event tags into gprc struct
-fn tags_to_protobuf(tags: &Vec<Vec<String>>) -> Vec<TagEntry> {
+fn tags_to_protobuf(tags: &[Vec<String>]) -> Vec<TagEntry> {
     tags.iter()
         .map(|x| TagEntry { values: x.clone() })
         .collect()

src/nip05.rs

@@ -11,7 +11,7 @@ use crate::repo::NostrRepo;
 use hyper::body::HttpBody;
 use hyper::client::connect::HttpConnector;
 use hyper::Client;
-use hyper_tls::HttpsConnector;
+use hyper_rustls::HttpsConnector;
 use std::sync::Arc;
 use std::time::Duration;
 use std::time::Instant;
@@ -133,7 +133,12 @@ impl Verifier {
     ) -> Result<Self> {
         info!("creating NIP-05 verifier");
         // setup hyper client
-        let https = HttpsConnector::new();
+        let https = hyper_rustls::HttpsConnectorBuilder::new()
+            .with_native_roots()
+            .https_or_http()
+            .enable_http1()
+            .build();
+
         let client = Client::builder().build::<_, hyper::Body>(https);
 
         // After all accounts have been re-verified, don't check again

src/payment/lnbits.rs

@@ -2,7 +2,7 @@
 use http::Uri;
 use hyper::client::connect::HttpConnector;
 use hyper::Client;
-use hyper_tls::HttpsConnector;
+use hyper_rustls::HttpsConnector;
 use nostr::Keys;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -72,7 +72,11 @@ pub struct LNBitsPaymentProcessor {
 impl LNBitsPaymentProcessor {
     pub fn new(settings: &Settings) -> Self {
         // setup hyper client
-        let https = HttpsConnector::new();
+        let https = hyper_rustls::HttpsConnectorBuilder::new()
+            .with_native_roots()
+            .https_only()
+            .enable_http1()
+            .build();
         let client = Client::builder().build::<_, hyper::Body>(https);
 
         Self {

src/repo/postgres.rs

@@ -14,15 +14,13 @@ use sqlx::{Error, Execute, FromRow, Postgres, QueryBuilder, Row};
 use std::time::{Duration, Instant};
 
 use crate::error;
-use crate::hexrange::{hex_range, HexSearch};
 use crate::repo::postgres_migration::run_migrations;
 use crate::server::NostrMetrics;
 use crate::utils::{self, is_hex, is_lower_hex};
 use nostr::key::Keys;
 use tokio::sync::mpsc::Sender;
 use tokio::sync::oneshot::Receiver;
-use tracing::log::trace;
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info, trace, warn};
 
 pub type PostgresPool = sqlx::pool::Pool<Postgres>;
 
@@ -638,14 +636,14 @@ ON CONFLICT (id) DO NOTHING"#,
         sqlx::query(
             "INSERT INTO invoice (pubkey, payment_hash, amount, status, description, created_at, invoice) VALUES ($1, $2, $3, $4, $5, now(), $6)",
         )
-        .bind(pub_key)
-        .bind(invoice_info.payment_hash)
-        .bind(invoice_info.amount as i64)
-        .bind(invoice_info.status)
-        .bind(invoice_info.memo)
-        .bind(invoice_info.bolt11)
-        .execute(&mut tx)
-        .await.unwrap();
+            .bind(pub_key)
+            .bind(invoice_info.payment_hash)
+            .bind(invoice_info.amount as i64)
+            .bind(invoice_info.status)
+            .bind(invoice_info.memo)
+            .bind(invoice_info.bolt11)
+            .execute(&mut tx)
+            .await.unwrap();
 
         debug!("Invoice added");
 
@@ -734,140 +732,62 @@ fn query_from_filter(f: &ReqFilter) -> Option<QueryBuilder<Postgres>> {
         // filter out non-hex values
         let auth_vec: Vec<&String> = auth_vec.iter().filter(|a| is_hex(a)).collect();
 
-        if !auth_vec.is_empty() {
-            query.push("(");
-
-            // shortcut authors into "IN" query
-            let any_is_range = auth_vec.iter().any(|pk| pk.len() != 64);
-            if !any_is_range {
-                query.push("e.pub_key in (");
-                let mut pk_sep = query.separated(", ");
-                for pk in auth_vec.iter() {
-                    pk_sep.push_bind(hex::decode(pk).ok());
-                }
-                query.push(") OR e.delegated_by in (");
-                let mut pk_delegated_sep = query.separated(", ");
-                for pk in auth_vec.iter() {
-                    pk_delegated_sep.push_bind(hex::decode(pk).ok());
-                }
-                query.push(")");
-                push_and = true;
-            } else {
-                let mut range_authors = query.separated(" OR ");
-                for auth in auth_vec {
-                    match hex_range(auth) {
-                        Some(HexSearch::Exact(ex)) => {
-                            range_authors
-                                .push("(e.pub_key = ")
-                                .push_bind_unseparated(ex.clone())
-                                .push_unseparated(" OR e.delegated_by = ")
-                                .push_bind_unseparated(ex)
-                                .push_unseparated(")");
-                        }
-                        Some(HexSearch::Range(lower, upper)) => {
-                            range_authors
-                                .push("((e.pub_key > ")
-                                .push_bind_unseparated(lower.clone())
-                                .push_unseparated(" AND e.pub_key < ")
-                                .push_bind_unseparated(upper.clone())
-                                .push_unseparated(") OR (e.delegated_by > ")
-                                .push_bind_unseparated(lower)
-                                .push_unseparated(" AND e.delegated_by < ")
-                                .push_bind_unseparated(upper)
-                                .push_unseparated("))");
-                        }
-                        Some(HexSearch::LowerOnly(lower)) => {
-                            range_authors
-                                .push("(e.pub_key > ")
-                                .push_bind_unseparated(lower.clone())
-                                .push_unseparated(" OR e.delegated_by > ")
-                                .push_bind_unseparated(lower)
-                                .push_unseparated(")");
-                        }
-                        None => {
-                            info!("Could not parse hex range from author {:?}", auth);
-                        }
-                    }
-                    push_and = true;
-                }
-            }
-            query.push(")");
+        if auth_vec.is_empty() {
+            return None;
         }
+        query.push("(e.pub_key in (");
+
+        let mut pk_sep = query.separated(", ");
+        for pk in auth_vec.iter() {
+            pk_sep.push_bind(hex::decode(pk).ok());
+        }
+        query.push(") OR e.delegated_by in (");
+        let mut pk_delegated_sep = query.separated(", ");
+        for pk in auth_vec.iter() {
+            pk_delegated_sep.push_bind(hex::decode(pk).ok());
+        }
+        push_and = true;
+        query.push("))");
     }
 
     // Query for Kind
     if let Some(ks) = &f.kinds {
-        if !ks.is_empty() {
-            if push_and {
-                query.push(" AND ");
-            }
-            push_and = true;
-
-            query.push("e.kind in (");
-            let mut list_query = query.separated(", ");
-            for k in ks.iter() {
-                list_query.push_bind(*k as i64);
-            }
-            query.push(")");
+        if ks.is_empty() {
+            return None;
         }
+        if push_and {
+            query.push(" AND ");
+        }
+        push_and = true;
+
+        query.push("e.kind in (");
+        let mut list_query = query.separated(", ");
+        for k in ks.iter() {
+            list_query.push_bind(*k as i64);
+        }
+        query.push(")");
     }
 
-    // Query for event, allowing prefix matches
+    // Query for event,
     if let Some(id_vec) = &f.ids {
         // filter out non-hex values
         let id_vec: Vec<&String> = id_vec.iter().filter(|a| is_hex(a)).collect();
-
-        if !id_vec.is_empty() {
-            if push_and {
-                query.push(" AND (");
-            } else {
-                query.push("(");
-            }
-            push_and = true;
-
-            // shortcut ids into "IN" query
-            let any_is_range = id_vec.iter().any(|pk| pk.len() != 64);
-            if !any_is_range {
-                query.push("id in (");
-                let mut sep = query.separated(", ");
-                for id in id_vec.iter() {
-                    sep.push_bind(hex::decode(id).ok());
-                }
-                query.push(")");
-            } else {
-                // take each author and convert to a hex search
-                let mut id_query = query.separated(" OR ");
-                for id in id_vec {
-                    match hex_range(id) {
-                        Some(HexSearch::Exact(ex)) => {
-                            id_query
-                                .push("(id = ")
-                                .push_bind_unseparated(ex)
-                                .push_unseparated(")");
-                        }
-                        Some(HexSearch::Range(lower, upper)) => {
-                            id_query
-                                .push("(id > ")
-                                .push_bind_unseparated(lower)
-                                .push_unseparated(" AND id < ")
-                                .push_bind_unseparated(upper)
-                                .push_unseparated(")");
-                        }
-                        Some(HexSearch::LowerOnly(lower)) => {
-                            id_query
-                                .push("(id > ")
-                                .push_bind_unseparated(lower)
-                                .push_unseparated(")");
-                        }
-                        None => {
-                            info!("Could not parse hex range from id {:?}", id);
-                        }
-                    }
-                }
-            }
-
-            query.push(")");
+        if id_vec.is_empty() {
+            return None;
         }
+        if push_and {
+            query.push(" AND (");
+        } else {
+            query.push("(");
+        }
+        push_and = true;
+
+        query.push("id in (");
+        let mut sep = query.separated(", ");
+        for id in id_vec.iter() {
+            sep.push_bind(hex::decode(id).ok());
+        }
+        query.push("))");
     }
 
     // Query for tags
@@ -878,22 +798,46 @@ fn query_from_filter(f: &ReqFilter) -> Option<QueryBuilder<Postgres>> {
             }
             push_and = true;
 
+            let mut push_or = false;
+            query.push("e.id IN (SELECT ee.id FROM \"event\" ee LEFT JOIN tag t on ee.id = t.event_id WHERE ee.hidden != 1::bit(1) and ");
             for (key, val) in map.iter() {
-                query.push("e.id IN (SELECT ee.id FROM \"event\" ee LEFT JOIN tag t on ee.id = t.event_id WHERE ee.hidden != 1::bit(1) and (t.\"name\" = ")
+                if val.is_empty() {
+                    return None;
+                }
+                if push_or {
+                    query.push(" OR ");
+                }
+                query
+                    .push("(t.\"name\" = ")
                     .push_bind(key.to_string())
-                    .push(" AND (value in (");
+                    .push(" AND (");
 
-                // plain value match first
-                let mut tag_query = query.separated(", ");
-                for v in val.iter() {
-                    if (v.len() % 2 != 0) && !is_lower_hex(v) {
+                let has_plain_values = val.iter().any(|v| (v.len() % 2 != 0 || !is_lower_hex(v)));
+                let has_hex_values = val.iter().any(|v| v.len() % 2 == 0 && is_lower_hex(v));
+                if has_plain_values {
+                    query.push("value in (");
+                    // plain value match first
+                    let mut tag_query = query.separated(", ");
+                    for v in val.iter().filter(|v| v.len() % 2 != 0 || !is_lower_hex(v)) {
                         tag_query.push_bind(v.as_bytes());
-                    } else {
+                    }
+                }
+                if has_plain_values && has_hex_values {
+                    query.push(") OR ");
+                }
+                if has_hex_values {
+                    query.push("value_hex in (");
+                    // plain value match first
+                    let mut tag_query = query.separated(", ");
+                    for v in val.iter().filter(|v| v.len() % 2 == 0 && is_lower_hex(v)) {
                         tag_query.push_bind(hex::decode(v).ok());
                     }
                 }
-                query.push("))))");
+
+                query.push(")))");
+                push_or = true;
             }
+            query.push(")");
         }
     }
 
@@ -926,10 +870,7 @@ fn query_from_filter(f: &ReqFilter) -> Option<QueryBuilder<Postgres>> {
         query.push("e.hidden != 1::bit(1)");
     }
     // never display expired events
-    query
-        .push(" AND (e.expires_at IS NULL OR e.expires_at > ")
-        .push_bind(Utc.timestamp_opt(utils::unix_time() as i64, 0).unwrap())
-        .push(")");
+    query.push(" AND (e.expires_at IS NULL OR e.expires_at > now())");
 
     // Apply per-filter limit to this query.
     // The use of a LIMIT implies a DESC order, to capture only the most recent events.
@@ -964,3 +905,111 @@ impl FromRow<'_, PgRow> for VerificationRecord {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::{HashMap, HashSet};
+
+    #[test]
+    fn test_query_gen_tag_value_hex() {
+        let filter = ReqFilter {
+            ids: None,
+            kinds: Some(vec![1000]),
+            since: None,
+            until: None,
+            authors: Some(vec![
+                "84de35e2584d2b144aae823c9ed0b0f3deda09648530b93d1a2a146d1dea9864".to_owned(),
+            ]),
+            limit: None,
+            tags: Some(HashMap::from([(
+                'p',
+                HashSet::from([
+                    "63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed".to_owned(),
+                ]),
+            )])),
+            force_no_match: false,
+        };
+
+        let q = query_from_filter(&filter).unwrap();
+        assert_eq!(q.sql(), "SELECT e.\"content\", e.created_at FROM \"event\" e WHERE (e.pub_key in ($1) OR e.delegated_by in ($2)) AND e.kind in ($3) AND e.id IN (SELECT ee.id FROM \"event\" ee LEFT JOIN tag t on ee.id = t.event_id WHERE ee.hidden != 1::bit(1) and (t.\"name\" = $4 AND (value_hex in ($5)))) AND e.hidden != 1::bit(1) AND (e.expires_at IS NULL OR e.expires_at > now()) ORDER BY e.created_at ASC LIMIT 1000")
+    }
+
+    #[test]
+    fn test_query_gen_tag_value() {
+        let filter = ReqFilter {
+            ids: None,
+            kinds: Some(vec![1000]),
+            since: None,
+            until: None,
+            authors: Some(vec![
+                "84de35e2584d2b144aae823c9ed0b0f3deda09648530b93d1a2a146d1dea9864".to_owned(),
+            ]),
+            limit: None,
+            tags: Some(HashMap::from([('d', HashSet::from(["test".to_owned()]))])),
+            force_no_match: false,
+        };
+
+        let q = query_from_filter(&filter).unwrap();
+        assert_eq!(q.sql(), "SELECT e.\"content\", e.created_at FROM \"event\" e WHERE (e.pub_key in ($1) OR e.delegated_by in ($2)) AND e.kind in ($3) AND e.id IN (SELECT ee.id FROM \"event\" ee LEFT JOIN tag t on ee.id = t.event_id WHERE ee.hidden != 1::bit(1) and (t.\"name\" = $4 AND (value in ($5)))) AND e.hidden != 1::bit(1) AND (e.expires_at IS NULL OR e.expires_at > now()) ORDER BY e.created_at ASC LIMIT 1000")
+    }
+
+    #[test]
+    fn test_query_gen_tag_value_and_value_hex() {
+        let filter = ReqFilter {
+            ids: None,
+            kinds: Some(vec![1000]),
+            since: None,
+            until: None,
+            authors: Some(vec![
+                "84de35e2584d2b144aae823c9ed0b0f3deda09648530b93d1a2a146d1dea9864".to_owned(),
+            ]),
+            limit: None,
+            tags: Some(HashMap::from([(
+                'd',
+                HashSet::from([
+                    "test".to_owned(),
+                    "63fe6318dc58583cfe16810f86dd09e18bfd76aabc24a0081ce2856f330504ed".to_owned(),
+                ]),
+            )])),
+            force_no_match: false,
+        };
+
+        let q = query_from_filter(&filter).unwrap();
+        assert_eq!(q.sql(), "SELECT e.\"content\", e.created_at FROM \"event\" e WHERE (e.pub_key in ($1) OR e.delegated_by in ($2)) AND e.kind in ($3) AND e.id IN (SELECT ee.id FROM \"event\" ee LEFT JOIN tag t on ee.id = t.event_id WHERE ee.hidden != 1::bit(1) and (t.\"name\" = $4 AND (value in ($5) OR value_hex in ($6)))) AND e.hidden != 1::bit(1) AND (e.expires_at IS NULL OR e.expires_at > now()) ORDER BY e.created_at ASC LIMIT 1000")
+    }
+
+    #[test]
+    fn test_query_multiple_tags() {
+        let filter = ReqFilter {
+            ids: None,
+            kinds: Some(vec![30_001]),
+            since: None,
+            until: None,
+            authors: None,
+            limit: None,
+            tags: Some(HashMap::from([
+                ('d', HashSet::from(["follow".to_owned()])),
+                ('t', HashSet::from(["siamstr".to_owned()])),
+            ])),
+            force_no_match: false,
+        };
+        let q = query_from_filter(&filter).unwrap();
+        assert_eq!(q.sql(), "SELECT e.\"content\", e.created_at FROM \"event\" e WHERE e.kind in ($1) AND e.id IN (SELECT ee.id FROM \"event\" ee LEFT JOIN tag t on ee.id = t.event_id WHERE ee.hidden != 1::bit(1) and (t.\"name\" = $2 AND (value in ($3))) OR (t.\"name\" = $4 AND (value in ($5)))) AND e.hidden != 1::bit(1) AND (e.expires_at IS NULL OR e.expires_at > now()) ORDER BY e.created_at ASC LIMIT 1000")
+    }
+
+    #[test]
+    fn test_query_empty_tags() {
+        let filter = ReqFilter {
+            ids: None,
+            kinds: Some(vec![1, 6, 16, 30023, 1063, 6969]),
+            since: Some(1700697846),
+            until: None,
+            authors: None,
+            limit: None,
+            tags: Some(HashMap::from([('a', HashSet::new())])),
+            force_no_match: false,
+        };
+        assert!(query_from_filter(&filter).is_none());
+    }
+}

src/repo/postgres_migration.rs

@@ -205,7 +205,7 @@ CREATE INDEX tag_value_hex_idx ON tag USING btree (value_hex);
                 let event: Event = serde_json::from_str(&String::from_utf8(event_bytes).unwrap())?;
 
                 for t in event.tags.iter().filter(|x| x.len() > 1) {
-                    let tagname = t.get(0).unwrap();
+                    let tagname = t.first().unwrap();
                     let tagnamechar_opt = single_char_tagname(tagname);
                     if tagnamechar_opt.is_none() {
                         continue;

src/repo/sqlite.rs

@@ -4,8 +4,6 @@ use crate::config::Settings;
 use crate::db::QueryResult;
 use crate::error::{Error::SqlError, Result};
 use crate::event::{single_char_tagname, Event};
-use crate::hexrange::hex_range;
-use crate::hexrange::HexSearch;
 use crate::nip05::{Nip05Name, VerificationRecord};
 use crate::payment::{InvoiceInfo, InvoiceStatus};
 use crate::repo::sqlite_migration::{upgrade_db, STARTUP_SQL};
@@ -994,24 +992,9 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
         // take each author and convert to a hexsearch
         let mut auth_searches: Vec<String> = vec![];
         for auth in authvec {
-            match hex_range(auth) {
-                Some(HexSearch::Exact(ex)) => {
-                    auth_searches.push("author=?".to_owned());
-                    params.push(Box::new(ex));
-                }
-                Some(HexSearch::Range(lower, upper)) => {
-                    auth_searches.push("(author>? AND author<?)".to_owned());
-                    params.push(Box::new(lower));
-                    params.push(Box::new(upper));
-                }
-                Some(HexSearch::LowerOnly(lower)) => {
-                    auth_searches.push("author>?".to_owned());
-                    params.push(Box::new(lower));
-                }
-                None => {
-                    trace!("Could not parse hex range from author {:?}", auth);
-                }
-            }
+            auth_searches.push("author=?".to_owned());
+            let auth_bin = hex::decode(auth).ok();
+            params.push(Box::new(auth_bin));
         }
         if !authvec.is_empty() {
             let auth_clause = format!("({})", auth_searches.join(" OR "));
@@ -1032,24 +1015,8 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
         // take each author and convert to a hexsearch
         let mut id_searches: Vec<String> = vec![];
         for id in idvec {
-            match hex_range(id) {
-                Some(HexSearch::Exact(ex)) => {
-                    id_searches.push("event_hash=?".to_owned());
-                    params.push(Box::new(ex));
-                }
-                Some(HexSearch::Range(lower, upper)) => {
-                    id_searches.push("(event_hash>? AND event_hash<?)".to_owned());
-                    params.push(Box::new(lower));
-                    params.push(Box::new(upper));
-                }
-                Some(HexSearch::LowerOnly(lower)) => {
-                    id_searches.push("event_hash>?".to_owned());
-                    params.push(Box::new(lower));
-                }
-                None => {
-                    info!("Could not parse hex range from id {:?}", id);
-                }
-            }
+            id_searches.push("event_hash=?".to_owned());
+            params.push(Box::new(id.clone()));
         }
         if idvec.is_empty() {
             // if the ids list was empty, we should never return
@@ -1072,8 +1039,6 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
             // find evidence of the target tag name/value existing for this event.
             // Query for Kind/Since/Until additionally, to reduce the number of tags that come back.
             let kind_clause;
-            let since_clause;
-            let until_clause;
             if let Some(ks) = &f.kinds {
                 // kind is number, no escaping needed
                 let str_kinds: Vec<String> =
@@ -1082,16 +1047,16 @@ fn query_from_filter(f: &ReqFilter) -> (String, Vec<Box<dyn ToSql>>, Option<Stri
             } else {
                 kind_clause = String::new();
             };
-            if f.since.is_some() {
-                since_clause = format!("AND created_at >= {}", f.since.unwrap());
+            let since_clause = if f.since.is_some() {
+                format!("AND created_at >= {}", f.since.unwrap())
             } else {
-                since_clause = String::new();
+                String::new()
             };
             // Query for timestamp
-            if f.until.is_some() {
-                until_clause = format!("AND created_at <= {}", f.until.unwrap());
+            let until_clause = if f.until.is_some() {
+                format!("AND created_at <= {}", f.until.unwrap())
             } else {
-                until_clause = String::new();
+                String::new()
             };
 
             let tag_clause = format!(
@@ -1317,6 +1282,7 @@ pub async fn db_checkpoint_task(
 }
 
 #[derive(Debug)]
+#[allow(dead_code)]
 enum SqliteStatus {
     Ok,
     Busy,

src/repo/sqlite_migration.rs

@@ -159,7 +159,7 @@ fn mig_init(conn: &mut PooledConnection) -> usize {
             );
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (init) failed: {}", err);
             panic!("database could not be initialized");
         }
     }
@@ -295,7 +295,7 @@ pub fn rebuild_tags(conn: &mut PooledConnection) -> Result<()> {
             let event: Event = serde_json::from_str(&event_json)?;
             // look at each event, and each tag, creating new tag entries if appropriate.
             for t in event.tags.iter().filter(|x| x.len() > 1) {
-                let tagname = t.get(0).unwrap();
+                let tagname = t.first().unwrap();
                 let tagnamechar_opt = single_char_tagname(tagname);
                 if tagnamechar_opt.is_none() {
                     continue;
@@ -325,7 +325,7 @@ pub fn rebuild_tags(conn: &mut PooledConnection) -> Result<()> {
     Ok(())
 }
 
-//// Migration Scripts
+// Migration Scripts
 
 fn mig_1_to_2(conn: &mut PooledConnection) -> Result<usize> {
     // only change is adding a hidden column to events.
@@ -339,7 +339,7 @@ PRAGMA user_version = 2;
             info!("database schema upgraded v1 -> v2");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v1->v2) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -366,7 +366,7 @@ PRAGMA user_version = 3;
             info!("database schema upgraded v2 -> v3");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v2->v3) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -416,7 +416,7 @@ PRAGMA user_version = 4;
             info!("database schema upgraded v3 -> v4");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v3->v4) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -435,7 +435,7 @@ PRAGMA user_version=5;
             info!("database schema upgraded v4 -> v5");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v4->v5) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -461,7 +461,7 @@ fn mig_5_to_6(conn: &mut PooledConnection) -> Result<usize> {
             let event: Event = serde_json::from_str(&event_json)?;
             // look at each event, and each tag, creating new tag entries if appropriate.
             for t in event.tags.iter().filter(|x| x.len() > 1) {
-                let tagname = t.get(0).unwrap();
+                let tagname = t.first().unwrap();
                 let tagnamechar_opt = single_char_tagname(tagname);
                 if tagnamechar_opt.is_none() {
                     continue;
@@ -507,7 +507,7 @@ PRAGMA user_version = 7;
             info!("database schema upgraded v6 -> v7");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v6->v7) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -528,7 +528,7 @@ PRAGMA user_version = 8;
             info!("database schema upgraded v7 -> v8");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v7->v8) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -548,7 +548,7 @@ PRAGMA user_version = 9;
             info!("database schema upgraded v8 -> v9");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v8->v9) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -567,7 +567,7 @@ PRAGMA user_version = 10;
             info!("database schema upgraded v9 -> v10");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v9->v10) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -588,7 +588,7 @@ PRAGMA user_version = 11;
             info!("database schema upgraded v10 -> v11");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v10->v11) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -643,7 +643,7 @@ PRAGMA user_version = 13;
             info!("database schema upgraded v12 -> v13");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v12->v13) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -663,7 +663,7 @@ PRAGMA user_version = 14;
             info!("database schema upgraded v13 -> v14");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v13->v14) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -682,7 +682,7 @@ PRAGMA user_version = 15;
             info!("database schema upgraded v14 -> v15");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v14->v15) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -749,7 +749,7 @@ CREATE INDEX IF NOT EXISTS tag_covering_index ON tag(name,kind,value,created_at,
             let event: Event = serde_json::from_str(&event_json)?;
             // look at each event, and each tag, creating new tag entries if appropriate.
             for t in event.tags.iter().filter(|x| x.len() > 1) {
-                let tagname = t.get(0).unwrap();
+                let tagname = t.first().unwrap();
                 let tagnamechar_opt = single_char_tagname(tagname);
                 if tagnamechar_opt.is_none() {
                     continue;
@@ -786,7 +786,7 @@ PRAGMA user_version = 17;
             info!("database schema upgraded v16 -> v17");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v16->v17) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }
@@ -833,7 +833,7 @@ PRAGMA user_version = 18;
             info!("database schema upgraded v17 -> v18");
         }
         Err(err) => {
-            error!("update failed: {}", err);
+            error!("update (v17->v18) failed: {}", err);
             panic!("database could not be upgraded");
         }
     }

src/server.rs

@@ -30,6 +30,8 @@ use hyper::upgrade::Upgraded;
 use hyper::{
     header, server::conn::AddrStream, upgrade, Body, Request, Response, Server, StatusCode,
 };
+use nostr::key::FromPkStr;
+use nostr::key::Keys;
 use prometheus::IntCounterVec;
 use prometheus::IntGauge;
 use prometheus::{Encoder, Histogram, HistogramOpts, IntCounter, Opts, Registry, TextEncoder};
@@ -60,8 +62,6 @@ use tungstenite::error::Error as WsError;
 use tungstenite::handshake;
 use tungstenite::protocol::Message;
 use tungstenite::protocol::WebSocketConfig;
-use nostr::key::FromPkStr;
-use nostr::key::Keys;
 
 /// Handle arbitrary HTTP requests, including for `WebSocket` upgrades.
 #[allow(clippy::too_many_arguments)]
@@ -653,6 +653,7 @@ fn get_header_string(header: &str, headers: &HeaderMap) -> Option<String> {
 async fn ctrl_c_or_signal(mut shutdown_signal: Receiver<()>) {
     let mut term_signal = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
         .expect("could not define signal");
+    #[allow(clippy::never_loop)]
     loop {
         tokio::select! {
             _ = shutdown_signal.recv() => {
@@ -1029,25 +1030,23 @@ fn make_notice_message(notice: &Notice) -> Message {
     Message::text(json.to_string())
 }
 
-fn allowed_to_send(event_str: &String, conn: &conn::ClientConn, settings: &Settings) -> bool {
+fn allowed_to_send(event_str: &str, conn: &conn::ClientConn, settings: &Settings) -> bool {
     // TODO: pass in kind so that we can avoid deserialization for most events
     if settings.authorization.nip42_dms {
         match serde_json::from_str::<Event>(event_str) {
             Ok(event) => {
-                if event.kind == 4 {
+                if event.kind == 4 || event.kind == 44 || event.kind == 1059 {
                     match (conn.auth_pubkey(), event.tag_values_by_name("p").first()) {
                         (Some(auth_pubkey), Some(recipient_pubkey)) => {
                             recipient_pubkey == auth_pubkey || &event.pubkey == auth_pubkey
-                        },
-                        (_, _) => {
-                            false
-                        },
+                        }
+                        (_, _) => false,
                     }
                 } else {
                     true
                 }
-            },
-            Err(_) => false
+            }
+            Err(_) => false,
         }
     } else {
         true
@@ -1263,7 +1262,6 @@ async fn nostr_server(
                         // handle each type of message
                         let evid = ec.event_id().to_owned();
                         let parsed : Result<EventWrapper> = Result::<EventWrapper>::from(ec);
-                        metrics.cmd_event.inc();
                         match parsed {
                             Ok(WrappedEvent(e)) => {
                                 metrics.cmd_event.inc();
@@ -1344,10 +1342,15 @@ async fn nostr_server(
                         if conn.has_subscription(&s) {
                             info!("client sent duplicate subscription, ignoring (cid: {}, sub: {:?})", cid, s.id);
                         } else {
-                metrics.cmd_req.inc();
+                            metrics.cmd_req.inc();
                             if let Some(ref lim) = sub_lim_opt {
                                 lim.until_ready_with_jitter(jitter).await;
                             }
+                            if settings.limits.limit_scrapers && s.is_scraper() {
+                                info!("subscription was scraper, ignoring (cid: {}, sub: {:?})", cid, s.id);
+                                ws_stream.send(Message::Text(format!("[\"EOSE\",\"{}\"]", s.id))).await.ok();
+                                continue
+                            }
                             let (abandon_query_tx, abandon_query_rx) = oneshot::channel::<()>();
                             match conn.subscribe(s.clone()) {
                                 Ok(()) => {
@@ -1371,7 +1374,7 @@ async fn nostr_server(
                         // closing a request simply removes the subscription.
                         let parsed : Result<Close> = Result::<Close>::from(cc);
                         if let Ok(c) = parsed {
-                metrics.cmd_close.inc();
+                            metrics.cmd_close.inc();
                             // check if a query is currently
                             // running, and remove it if so.
                             let stop_tx = running_queries.remove(&c.id);

src/subscription.rs

@@ -45,8 +45,8 @@ pub struct ReqFilter {
 
 impl Serialize for ReqFilter {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
+        where
+            S: Serializer,
     {
         let mut map = serializer.serialize_map(None)?;
         if let Some(ids) = &self.ids {
@@ -80,8 +80,8 @@ impl Serialize for ReqFilter {
 
 impl<'de> Deserialize<'de> for ReqFilter {
     fn deserialize<D>(deserializer: D) -> Result<ReqFilter, D::Error>
-    where
-        D: Deserializer<'de>,
+        where
+            D: Deserializer<'de>,
     {
         let received: Value = Deserialize::deserialize(deserializer)?;
         let filter = received.as_object().ok_or_else(|| {
@@ -184,8 +184,8 @@ impl<'de> Deserialize<'de> for Subscription {
     /// Custom deserializer for subscriptions, which have a more
     /// complex structure than the other message types.
     fn deserialize<D>(deserializer: D) -> Result<Subscription, D::Error>
-    where
-        D: Deserializer<'de>,
+        where
+            D: Deserializer<'de>,
     {
         let mut v: Value = Deserialize::deserialize(deserializer)?;
         // this should be a 3-or-more element array.
@@ -258,6 +258,29 @@ impl Subscription {
         }
         false
     }
+
+    /// Is this subscription defined as a scraper query
+    pub fn is_scraper(&self) -> bool {
+        for f in &self.filters {
+            let mut precision = 0;
+            if f.ids.is_some() {
+                precision += 2;
+            }
+            if f.authors.is_some() {
+                precision += 1;
+            }
+            if f.kinds.is_some() {
+                precision += 1;
+            }
+            if f.tags.is_some() {
+                precision += 1;
+            }
+            if precision < 2 {
+                return true;
+            }
+        }
+        false
+    }
 }
 
 fn prefix_match(prefixes: &[String], target: &str) -> bool {
@@ -338,7 +361,7 @@ mod tests {
         let s: Subscription = serde_json::from_str(raw_json)?;
         assert_eq!(s.id, "some-id");
         assert_eq!(s.filters.len(), 1);
-        assert_eq!(s.filters.get(0).unwrap().authors, None);
+        assert_eq!(s.filters.first().unwrap().authors, None);
         Ok(())
     }
 
@@ -402,7 +425,7 @@ mod tests {
         let s: Subscription = serde_json::from_str(raw_json)?;
         assert_eq!(s.id, "some-id");
         assert_eq!(s.filters.len(), 1);
-        let first_filter = s.filters.get(0).unwrap();
+        let first_filter = s.filters.first().unwrap();
         assert_eq!(
             first_filter.authors,
             Some(vec!("test-author-id".to_owned()))
@@ -633,11 +656,11 @@ mod tests {
         let s: Subscription = serde_json::from_str(
             r##"["REQ","xyz",{"authors":["abc", "bcd"], "since": 10, "until": 20, "limit":100, "#e": ["foo", "bar"], "#d": ["test"]}]"##,
         )?;
-        let f = s.filters.get(0);
+        let f = s.filters.first();
         let serialized = serde_json::to_string(&f)?;
         let serialized_wrapped = format!(r##"["REQ", "xyz",{}]"##, serialized);
         let parsed: Subscription = serde_json::from_str(&serialized_wrapped)?;
-        let parsed_filter = parsed.filters.get(0);
+        let parsed_filter = parsed.filters.first();
         if let Some(pf) = parsed_filter {
             assert_eq!(pf.since, Some(10));
             assert_eq!(pf.until, Some(20));
@@ -647,4 +670,14 @@ mod tests {
         }
         Ok(())
     }
+
+    #[test]
+    fn is_scraper() -> Result<()> {
+        assert!(serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"kinds": [1984],"since": 123,"limit":1}]"#)?.is_scraper());
+        assert!(serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"kinds": [1984]},{"kinds": [1984],"authors":["aaaa"]}]"#)?.is_scraper());
+        assert!(!serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"kinds": [1984],"authors":["aaaa"]}]"#)?.is_scraper());
+        assert!(!serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"ids": ["aaaa"]}]"#)?.is_scraper());
+        assert!(!serde_json::from_str::<Subscription>(r##"["REQ","some-id",{"#p": ["aaaa"],"kinds":[1,4]}]"##)?.is_scraper());
+        Ok(())
+    }
 }

src/utils.rs

@@ -37,7 +37,7 @@ pub fn is_lower_hex(s: &str) -> bool {
     })
 }
 
-pub fn host_str(url: &String) -> Option<String> {
+pub fn host_str(url: &str) -> Option<String> {
     Url::parse(url)
         .ok()
         .and_then(|u| u.host_str().map(|s| s.to_string()))

tests/common/mod.rs

@@ -103,8 +103,5 @@ fn get_available_port() -> Option<u16> {
 }
 pub fn port_is_available(port: u16) -> bool {
     info!("checking on port {}", port);
-    match TcpListener::bind(("127.0.0.1", port)) {
-        Ok(_) => true,
-        Err(_) => false,
-    }
+    TcpListener::bind(("127.0.0.1", port)).is_ok()
 }

tests/conn.rs

@@ -52,7 +52,7 @@ mod tests {
         let challenge = client_conn.auth_challenge().unwrap();
         let event = auth_event(challenge);
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Ok(())));
         assert_eq!(client_conn.auth_challenge(), None);
@@ -67,7 +67,7 @@ mod tests {
         assert_eq!(client_conn.auth_pubkey(), None);
 
         let event = auth_event(&"challenge".into());
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -87,14 +87,14 @@ mod tests {
         let challenge = client_conn.auth_challenge().unwrap().clone();
 
         let event = auth_event(&challenge);
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Ok(())));
         assert_eq!(client_conn.auth_challenge(), None);
         assert_eq!(client_conn.auth_pubkey(), Some(&event.pubkey));
 
         let event1 = auth_event(&challenge);
-        let result1 = client_conn.authenticate(&event1, &RELAY.into());
+        let result1 = client_conn.authenticate(&event1, RELAY);
 
         assert!(matches!(result1, Ok(())));
         assert_eq!(client_conn.auth_challenge(), None);
@@ -118,7 +118,7 @@ mod tests {
         let mut event = auth_event(challenge);
         event.sig = event.sig.chars().rev().collect::<String>();
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -138,7 +138,7 @@ mod tests {
         let challenge = client_conn.auth_challenge().unwrap();
         let event = auth_event_with_kind(challenge, 9999999999999999);
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -158,7 +158,7 @@ mod tests {
         let challenge = client_conn.auth_challenge().unwrap();
         let event = auth_event_with_created_at(challenge, unix_time() - 1200); // 20 minutes
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -178,7 +178,7 @@ mod tests {
         let challenge = client_conn.auth_challenge().unwrap();
         let event = auth_event_with_created_at(challenge, unix_time() + 1200); // 20 minutes
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -197,7 +197,7 @@ mod tests {
 
         let event = auth_event_without_tags();
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -216,7 +216,7 @@ mod tests {
 
         let event = auth_event_without_challenge();
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -236,7 +236,7 @@ mod tests {
         let challenge = client_conn.auth_challenge().unwrap();
         let event = auth_event_without_relay(challenge);
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -255,7 +255,7 @@ mod tests {
 
         let event = auth_event(&"invalid challenge".into());
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }
@@ -275,7 +275,7 @@ mod tests {
         let challenge = client_conn.auth_challenge().unwrap();
         let event = auth_event_with_relay(challenge, &"xyz".into());
 
-        let result = client_conn.authenticate(&event, &RELAY.into());
+        let result = client_conn.authenticate(&event, RELAY);
 
         assert!(matches!(result, Err(Error::AuthFailure)));
     }

tests/integration_test.rs

@@ -73,7 +73,7 @@ async fn publish_test() -> Result<()> {
     let event_sub = r#"["REQ", "simple", {}]"#;
     sub_ws.send(event_sub.into()).await?;
     // read from subscription
-    let ws_next = sub_ws.next().await;
+    let _ws_next = sub_ws.next().await;
     let _res = relay.shutdown_tx.send(());
     Ok(())
 }