From 8bf52646c73c209c97c28821e0e0f85c9f7badc2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 10:30:18 +0200 Subject: [PATCH] Fork Sync: Update from parent repository (#3) * improvement: use appropriate paths for systemd example * improvement: add a configurable postgres write conn string This adds a new configurable connection string for postgres writes. * improvement: document pg connection_write config * build: upgrade checkout action for github ci * perf: use standard allocator, limit sqlite mmap to 4GB This is an experimental change to see if we can reduce memory usage with large SQLite databases. If successful, we'll do this again and further reduce the database mmap size. This will cause greater use of the page cache, but that is more easily reclaimed by the kernel, and should reduce memory pressure, as well as making it clearer how much memory the application is actually using for connections, subscriptions, etc. * docs: reformatting * docs: allow host header prefix matching, required for Damus compatibility * perf: disable sqlite mmap to reduce memory pressure * perf: switch to jemalloc allocator * docs: helpful ubuntu packages for building * perf: reduce SQLite connection count and idle lifetime On lightly loaded relays, we free up memory faster by letting idle connections be reclaimed in 10 seconds instead of the default 10 minutes. This also sets the minimum to zero connections, instead of always trying to hold one open. --------- Co-authored-by: Petr Kracik Co-authored-by: Kieran Co-authored-by: Greg Heartsfield --- README.md | 5 +++++ config.toml | 2 +- docs/reverse-proxy.md | 6 +++--- src/main.rs | 1 - src/repo/sqlite.rs | 5 +++-- src/repo/sqlite_migration.rs | 2 +- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ec3cff4..09a66f7 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,11 @@ https://hub.docker.com/r/scsibug/nostr-rs-relay Building `nostr-rs-relay` requires an installation of Cargo & Rust: https://www.rust-lang.org/tools/install +The following OS packages will be helpful; on Debian/Ubuntu: +```console +$ sudo apt-get install build-essentials cmake protobuf-compiler pkg-config libssl-dev +``` + Clone this repository, and then build a release version of the relay: ```console diff --git a/config.toml b/config.toml index a3edb91..3dea8ee 100644 --- a/config.toml +++ b/config.toml @@ -42,7 +42,7 @@ description = "A newly created nostr-rs-relay.\n\nCustomize this with your own i # Database connection pool settings for subscribers: # Minimum number of SQLite reader connections -#min_conn = 4 +#min_conn = 0 # Maximum number of SQLite reader connections. Recommend setting this # to approx the number of cores. diff --git a/docs/reverse-proxy.md b/docs/reverse-proxy.md index 157ba5c..e6c6d0d 100644 --- a/docs/reverse-proxy.md +++ b/docs/reverse-proxy.md @@ -29,7 +29,7 @@ frontend fe_prod bind :80 http-request set-header X-Forwarded-Proto https if { ssl_fc } redirect scheme https code 301 if !{ ssl_fc } - acl host_relay hdr(host) -i relay.example.com + acl host_relay hdr(host) -i -m beg relay.example.com use_backend relay if host_relay # HSTS (1 year) http-response set-header Strict-Transport-Security max-age=31536000 @@ -120,7 +120,7 @@ Assumptions: * `Traefik` is running in `Docker`, using `docker compose` and labels for the static configuration. An equivalent setup useing a Traefik config file is possible too (but not covered here). * Strict Transport Security is enabled. * Hostname for the relay is `relay.example.com`, email adres for ACME certificates provider is `name@example.com`. -* ipv6 is enabled, a viable private ipv6 subnet is specified in the example below. +* ipv6 is enabled, a viable private ipv6 subnet is specified in the example below. * Relay is running on port `8080`. ``` @@ -196,4 +196,4 @@ services: ### Traefik Notes -Traefik will take care of the provisioning and renewal of certificates. In case of an ipv4-only relay, simply detele the `enable_ipv6:` and `ipam:` entries in the `networks:` section of the docker-compose file. \ No newline at end of file +Traefik will take care of the provisioning and renewal of certificates. In case of an ipv4-only relay, simply detele the `enable_ipv6:` and `ipam:` entries in the `networks:` section of the docker-compose file. diff --git a/src/main.rs b/src/main.rs index 97aa032..6a9116d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,7 +8,6 @@ use std::sync::mpsc as syncmpsc; use std::sync::mpsc::{Receiver as MpscReceiver, Sender as MpscSender}; use std::thread; use tracing::info; - #[cfg(not(target_env = "msvc"))] use tikv_jemallocator::Jemalloc; diff --git a/src/repo/sqlite.rs b/src/repo/sqlite.rs index 91b48e6..b156f0c 100644 --- a/src/repo/sqlite.rs +++ b/src/repo/sqlite.rs @@ -62,7 +62,7 @@ impl SqliteRepo { "writer", settings, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, - 1, + 0, 2, false, ); @@ -70,7 +70,7 @@ impl SqliteRepo { "maintenance", settings, OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, - 1, + 0, 2, true, ); @@ -1199,6 +1199,7 @@ pub fn build_pool( .test_on_check_out(true) // no noticeable performance hit .min_idle(Some(min_size)) .max_size(max_size) + .idle_timeout(Some(Duration::from_secs(10))) .max_lifetime(Some(Duration::from_secs(30))) .build(manager) .unwrap(); diff --git a/src/repo/sqlite_migration.rs b/src/repo/sqlite_migration.rs index 709a572..f2ccab0 100644 --- a/src/repo/sqlite_migration.rs +++ b/src/repo/sqlite_migration.rs @@ -19,7 +19,7 @@ PRAGMA foreign_keys = ON; PRAGMA journal_size_limit = 32768; PRAGMA temp_store = 2; -- use memory, not temp files PRAGMA main.cache_size = 20000; -- 80MB max cache size per conn -pragma mmap_size = 17179869184; -- cap mmap at 16GB +pragma mmap_size = 0; -- disable mmap (default) "##; /// Latest database version