Merge 21bf4e4c26 into 971889f9a6

This is a good feature, but will limit valid requests from being
served.  Defaulting this to off will be less surprising to relay ops.

.github/workflows/ci.yml

@@ -1,8 +1,9 @@
 name: Test and build
 
 on:
+  workflow_call:
   push:
-    branches:
+    branches-ignore:
       - master
 
 jobs:

.github/workflows/docker-image.yml

@@ -0,0 +1,67 @@
+name: Docker Build and Push
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - master
+
+env:
+  REGISTRY_NAME: ghcr.io
+  IMAGE_NAME: nostr-rs-relay
+
+jobs:
+  # ci:
+  #   name: CI
+  #   uses: ./.github/workflows/ci.yml
+
+  build_and_push:
+    # needs: [ci]
+    name: Build and Push
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Collecting Metadata
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY_NAME }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+      
+      - name: Building And Pushing Image
+        id: docker_build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # - name: Build and push
+      #   id: docker_build
+      #   uses: docker/build-push-action@v4
+      #   with:
+      #     context: .
+      #     push: true
+      #     tags: ghcr.io/${{ github.repository_owner }}/nostr-rs-relay:latest
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/sync-fork.yml

@@ -0,0 +1,19 @@
+name: Sync Fork
+
+on:
+  schedule:
+    - cron: '0 2 * * *' # once a day at 2 am
+  workflow_dispatch: # on button click
+
+jobs:
+  sync:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: tgymnich/fork-sync@v1.8
+        with:
+          owner: scsibug
+          base: master
+          head: master
+          ignore_fail: true

config.toml

@@ -155,6 +155,11 @@ reject_future_seconds = 1800
 #    0, 1, 2, 3, 7, 40, 41, 42, 43, 44, 30023,
 #]
 
+# Rejects imprecise requests (kind only and author only etc)
+# This is a temperary measure to improve the adoption of outbox model
+# Its recommended to have this enabled
+limit_scrapers = false
+
 [authorization]
 # Pubkey addresses in this array are whitelisted for event publishing.
 # Only valid events by these authors will be accepted, if the variable

src/config.rs

@@ -74,6 +74,7 @@ pub struct Limits {
     pub event_persist_buffer: usize, // events to buffer for database commits (block senders if database writes are too slow)
     pub event_kind_blacklist: Option<Vec<u64>>,
     pub event_kind_allowlist: Option<Vec<u64>>,
+    pub limit_scrapers: bool,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -308,6 +309,7 @@ impl Default for Settings {
                 event_persist_buffer: 4096,
                 event_kind_blacklist: None,
                 event_kind_allowlist: None,
+                limit_scrapers: false
             },
             authorization: Authorization {
                 pubkey_whitelist: None, // Allow any address to publish

src/info.rs

@@ -58,7 +58,7 @@ pub struct RelayInfo {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub limitation: Option<Limitation>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub payment_url: Option<String>,
+    pub payments_url: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub fees: Option<Fees>,
 }
@@ -86,7 +86,7 @@ impl From<Settings> for RelayInfo {
             ),
         };
 
-        let (payment_url, fees) = if p.enabled {
+        let (payments_url, fees) = if p.enabled {
             let admission_fee = if p.admission_cost > 0 {
                 Some(vec![Fee {
                     amount: p.admission_cost * 1000,
@@ -110,7 +110,7 @@ impl From<Settings> for RelayInfo {
                 publication: post_fee,
             };
 
-            let payment_url = if p.enabled && i.relay_url.is_some() {
+            let payments_url = if p.enabled && i.relay_url.is_some() {
                 Some(format!(
                     "{}join",
                     i.relay_url.clone().unwrap().replace("ws", "http")
@@ -118,7 +118,7 @@ impl From<Settings> for RelayInfo {
             } else {
                 None
             };
-            (payment_url, Some(fees))
+            (payments_url, Some(fees))
         } else {
             (None, None)
         };
@@ -133,7 +133,7 @@ impl From<Settings> for RelayInfo {
             software: Some("https://git.sr.ht/~gheartsfield/nostr-rs-relay".to_owned()),
             version: CARGO_PKG_VERSION.map(std::borrow::ToOwned::to_owned),
             limitation: Some(limitations),
-            payment_url,
+            payments_url,
             fees,
             icon: i.relay_icon,
         }

src/server.rs

@@ -266,73 +266,82 @@ async fn handle_web_request(
             }
 
             let html = r#"
-<!doctype HTML>
-<head>
-  <meta charset="UTF-8">
-  <style>
-    body {
-      display: flex;
-      flex-direction: column;
-      align-items: center;
-      text-align: center;
-      font-family: Arial, sans-serif;
-      background-color: #6320a7;
-      color: white;
-    }
-
-    .container {
-      display: flex;
-      justify-content: center;
-      align-items: center;
-      height: 400px;
-    }
-
-    a {
-      color: pink;
-    }
-
-    input[type="text"] {
-        width: 100%;
-        max-width: 500px;
-        box-sizing: border-box;
-        overflow-x: auto;
-        white-space: nowrap;
-    }
-  </style>
-</head>
-<body>
-  <div style="width:75%;">
-    <h1>Enter your pubkey</h1>
-    <form action="/invoice" onsubmit="return checkForm(this);">
-      <input type="text" name="pubkey" id="pubkey-input"><br><br>
-      <input type="checkbox" id="terms" required>
-      <label for="terms">I agree to the <a href="/terms">terms and conditions</a></label><br><br>
-      <button type="submit">Submit</button>
-    </form>
-    <button id="get-public-key-btn">Get Public Key</button>
-  </div>
-  <script>
-    function checkForm(form) {
-      if (!form.terms.checked) {
-        alert("Please agree to the terms and conditions");
-        return false;
-      }
-      return true;
-    }
-
-    const pubkeyInput = document.getElementById('pubkey-input');
-      const getPublicKeyBtn = document.getElementById('get-public-key-btn');
-      getPublicKeyBtn.addEventListener('click', async function() {
-        try {
-          const publicKey = await window.nostr.getPublicKey();
-          pubkeyInput.value = publicKey;
-        } catch (error) {
-          console.error(error);
-        }
-      });
-  </script>
-</body>
-</html>
+            <!DOCTYPE html>
+            <html lang="en">
+            
+            <head>
+                <meta charset="utf-8">
+                <meta name="viewport" content="width=device-width, initial-scale=1">
+                <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" rel="stylesheet">
+                <style>
+                    body {
+                        background-color: #6320a7 !important; /* Make sure this override takes precedence */
+                        color: white;
+                    }
+                
+                    a {
+                        color: pink !important;
+                    }
+                
+                    .btn-primary {
+                        background-color: #ff4081 !important;
+                    }
+                </style>
+                <title>Join Nostr - Enter your pubkey</title>
+            </head>
+            
+            <body>
+                <div class="container py-5">
+                    <div class="row">
+                        <div class="col-lg-6 mx-auto">
+                            <h1 class="text-center mb-4">Enter your pubkey</h1>
+                            <form action="/invoice" onsubmit="return checkForm(this);">
+                                <div class="form-group">
+                                    <input type="text" name="pubkey" class="form-control" id="pubkey-input" placeholder="Public Key" required>
+                                </div>
+                                <div class="form-check">
+                                    <input class="form-check-input" type="checkbox" id="terms" required>
+                                    <label class="form-check-label" for="terms">I agree to the <a href="/terms">terms and conditions</a></label>
+                                </div>
+                                <div class="form-group mt-4">
+                                    <button type="submit" class="btn btn-primary btn-block">Submit</button>
+                                </div>
+                            </form>
+                            <div class="form-group">
+                                <button id="get-public-key-btn" class="btn btn-primary btn-block">Get Public Key</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            
+                <script>
+                    function checkForm(form) {
+                        if (!form.terms.checked) {
+                            alert("Please agree to the terms and conditions");
+                            return false;
+                        }
+                        return true;
+                    }
+            
+                    const pubkeyInput = document.getElementById('pubkey-input');
+                    const getPublicKeyBtn = document.getElementById('get-public-key-btn');
+                    getPublicKeyBtn.addEventListener('click', async function() {
+                        try {
+                            const publicKey = await window.nostr.getPublicKey();
+                            pubkeyInput.value = publicKey;
+                        } catch (error) {
+                            console.error(error);
+                        }
+                    });
+                </script>
+            
+                <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script>
+                <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"></script>
+                <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"></script>
+            
+            </body>
+            
+            </html>
             "#;
             Ok(Response::builder()
                 .status(StatusCode::OK)
@@ -445,89 +454,94 @@ async fn handle_web_request(
 
             let html_result = format!(
                 r#"
-<!DOCTYPE html>
-<html>
-  <head>
-  <meta charset="UTF-8">
-    <style>
-      body {{
-        display: flex;
-        flex-direction: column;
-        align-items: center;
-        text-align: center;
-        font-family: Arial, sans-serif;
-        background-color:  #6320a7 ;
-        color: white;
-      }}
-      #copy-button {{
-        background-color: #bb5f0d ;
-        color: white;
-        padding: 10px 20px;
-        border-radius: 5px;
-        border: none;
-        cursor: pointer;
-      }}
-      #copy-button:hover {{
-        background-color: #8f29f4;
-      }}
-    .container {{
-        display: flex;
-        justify-content: center;
-        align-items: center;
-        height: 400px;
-    }}
-    a {{
-        color: pink;
-    }}
-    </style>
-  </head>
-  <body>
-    <div style="width:75%;">
-      <h3>
-        To use this relay, an admission fee of {} sats is required. By paying the fee, you agree to the <a href='terms'>terms</a>.
-      </h3>
-    </div>
-    <div>
-        <div style="max-height: 300px;">
-            {}
-        </div>
-    </div>
-    <div>
-    <div style="width: 75%;">
-        <p style="overflow-wrap: break-word; width: 500px;">{}</p>
-        <button id="copy-button">Copy</button>
-    </div>
-    <div>
-        <p> This page will not refresh </p>
-        <p> Verify admission <a href=/account?pubkey={}>here</a> once you have paid</p>
-    </div>
-    </div>
-  </body>
-</html>
-
-
-<script>
-  const copyButton = document.getElementById("copy-button");
-  if (navigator.clipboard) {{
-    copyButton.addEventListener("click", function() {{
-      const textToCopy = "{}";
-      navigator.clipboard.writeText(textToCopy).then(function() {{
-        console.log("Text copied to clipboard");
-      }}, function(err) {{
-        console.error("Could not copy text: ", err);
-      }});
-    }});
-  }} else {{
-    copyButton.style.display = "none";
-    console.warn("Clipboard API is not supported in this browser");
-  }}
-</script>
+                <!DOCTYPE html>
+                <html lang="en">
+                
+                <head>
+                    <meta charset="utf-8">
+                    <meta name="viewport" content="width=device-width, initial-scale=1">
+                    <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" rel="stylesheet">
+                    <style>
+                        body {{
+                            background-color: #6320a7;
+                            color: white;
+                        }}
+                
+                        a {{
+                            color: pink;
+                        }}
+                
+                        .btn {{
+                            background-color: #bb5f0d;
+                            color: white;
+                        }}
+                
+                        .btn:hover {{
+                            background-color: #8f29f4;
+                        }}
+                
+                        #copy-button {{
+                            margin-top: 10px;
+                        }}
+                    </style>
+                    <title>Join Nostr - Invoice</title>
+                </head>
+                
+                <body>
+                    <div class="container py-5">
+                        <div class="row">
+                            <div class="col-lg-6 mx-auto">
+                                <h3 class="text-center mb-4">
+                                    To use this relay, an admission fee of {} sats is required. By paying the fee, you agree to the <a href='terms'>terms</a>.
+                                </h3>
+                                
+                                <div class="text-center" style="max-height: 300px;">
+                                    {}
+                                </div>
+                
+                                <div class="text-center mt-4">
+                                    <p id="text-area" style="overflow-wrap: break-word;">{}</p>
+                                    <button id="copy-button" class="btn">Copy</button>
+                                </div>
+                
+                                <div class="text-center mt-4">
+                                    <p>This page will not refresh</p>
+                                    <p>Verify admission <a href="/account?pubkey={}">here</a> once you have paid</p>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                
+                    <script>
+                        const copyButton = document.getElementById("copy-button");
+                        const textArea = document.getElementById("text-area");
+                        if (navigator.clipboard) {{
+                            copyButton.addEventListener("click", function() {{
+                                const textToCopy = textArea.textContent;
+                                navigator.clipboard.writeText(textToCopy).then(function() {{
+                                    console.log("Text copied to clipboard");
+                                }}, function(err) {{
+                                    console.error("Could not copy text: ", err);
+                                }});
+                            }});
+                        }} else {{
+                            copyButton.style.display = "none";
+                            console.warn("Clipboard API is not supported in this browser");
+                        }}
+                    </script>
+                
+                    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script>
+                    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"></script>
+                    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"></script>
+                
+                </body>
+                
+                </html>
 "#,
                 settings.pay_to_relay.admission_cost,
                 qr_code,
                 invoice_info.bolt11,
-                pubkey,
-                invoice_info.bolt11
+                pubkey
             );
 
             Ok(Response::builder()
@@ -582,30 +596,41 @@ async fn handle_web_request(
 
             let html_result = format!(
                 r#"
-            <!DOCTYPE html>
-<html>
-  <head>
-    <meta charset="UTF-8">
-    <style>
-      body {{
-        display: flex;
-        flex-direction: column;
-        align-items: center;
-        text-align: center;
-        font-family: Arial, sans-serif;
-        background-color: #6320a7;
-        color: white;
-        height: 100vh;
-      }}
-    </style>
-  </head>
-  <body>
-    <div>
-      <h5>{} {} admitted</h5>
-    </div>
-  </body>
-</html>
-
+                <!DOCTYPE html>
+                <html lang="en">
+                
+                <head>
+                    <meta charset="utf-8">
+                    <meta name="viewport" content="width=device-width, initial-scale=1">
+                    <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" rel="stylesheet">
+                    <style>
+                        body {{
+                            background-color: #6320a7;
+                            color: white;
+                        }}
+                
+                        .vh-100 {{
+                            height: 100vh;
+                        }}
+                
+                        h5 {{
+                            color: white;
+                        }}
+                     </style>
+                    <title>Bootstrap Admissions</title>
+                </head>
+                
+                <body>
+                    <div class="d-flex justify-content-center align-items-center vh-100">
+                        <h5 class="text-break">{ } { } admitted</h5>
+                    </div>
+                
+                    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script>
+                    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.7/umd/popper.min.js"></script>
+                    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/js/bootstrap.min.js"></script>
+                </body>
+                
+                </html>
 
             "#,
                 pubkey, text
@@ -1261,7 +1286,6 @@ async fn nostr_server(
                         // handle each type of message
                         let evid = ec.event_id().to_owned();
                         let parsed : Result<EventWrapper> = Result::<EventWrapper>::from(ec);
-                        metrics.cmd_event.inc();
                         match parsed {
                             Ok(WrappedEvent(e)) => {
                                 metrics.cmd_event.inc();
@@ -1342,10 +1366,15 @@ async fn nostr_server(
                         if conn.has_subscription(&s) {
                             info!("client sent duplicate subscription, ignoring (cid: {}, sub: {:?})", cid, s.id);
                         } else {
-                metrics.cmd_req.inc();
+                            metrics.cmd_req.inc();
                             if let Some(ref lim) = sub_lim_opt {
                                 lim.until_ready_with_jitter(jitter).await;
                             }
+                            if settings.limits.limit_scrapers && s.is_scraper() {
+                                info!("subscription was scraper, ignoring (cid: {}, sub: {:?})", cid, s.id);
+                                ws_stream.send(Message::Text(format!("[\"EOSE\",\"{}\"]", s.id))).await.ok();
+                                continue
+                            }
                             let (abandon_query_tx, abandon_query_rx) = oneshot::channel::<()>();
                             match conn.subscribe(s.clone()) {
                                 Ok(()) => {
@@ -1369,7 +1398,7 @@ async fn nostr_server(
                         // closing a request simply removes the subscription.
                         let parsed : Result<Close> = Result::<Close>::from(cc);
                         if let Ok(c) = parsed {
-                metrics.cmd_close.inc();
+                            metrics.cmd_close.inc();
                             // check if a query is currently
                             // running, and remove it if so.
                             let stop_tx = running_queries.remove(&c.id);

src/subscription.rs

@@ -45,8 +45,8 @@ pub struct ReqFilter {
 
 impl Serialize for ReqFilter {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
+        where
+            S: Serializer,
     {
         let mut map = serializer.serialize_map(None)?;
         if let Some(ids) = &self.ids {
@@ -80,8 +80,8 @@ impl Serialize for ReqFilter {
 
 impl<'de> Deserialize<'de> for ReqFilter {
     fn deserialize<D>(deserializer: D) -> Result<ReqFilter, D::Error>
-    where
-        D: Deserializer<'de>,
+        where
+            D: Deserializer<'de>,
     {
         let received: Value = Deserialize::deserialize(deserializer)?;
         let filter = received.as_object().ok_or_else(|| {
@@ -184,8 +184,8 @@ impl<'de> Deserialize<'de> for Subscription {
     /// Custom deserializer for subscriptions, which have a more
     /// complex structure than the other message types.
     fn deserialize<D>(deserializer: D) -> Result<Subscription, D::Error>
-    where
-        D: Deserializer<'de>,
+        where
+            D: Deserializer<'de>,
     {
         let mut v: Value = Deserialize::deserialize(deserializer)?;
         // this should be a 3-or-more element array.
@@ -258,6 +258,29 @@ impl Subscription {
         }
         false
     }
+
+    /// Is this subscription defined as a scraper query
+    pub fn is_scraper(&self) -> bool {
+        for f in &self.filters {
+            let mut precision = 0;
+            if f.ids.is_some() {
+                precision += 2;
+            }
+            if f.authors.is_some() {
+                precision += 1;
+            }
+            if f.kinds.is_some() {
+                precision += 1;
+            }
+            if f.tags.is_some() {
+                precision += 1;
+            }
+            if precision < 2 {
+                return true;
+            }
+        }
+        false
+    }
 }
 
 fn prefix_match(prefixes: &[String], target: &str) -> bool {
@@ -647,4 +670,14 @@ mod tests {
         }
         Ok(())
     }
+
+    #[test]
+    fn is_scraper() -> Result<()> {
+        assert_eq!(true, serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"kinds": [1984],"since": 123,"limit":1}]"#)?.is_scraper());
+        assert_eq!(true, serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"kinds": [1984]},{"kinds": [1984],"authors":["aaaa"]}]"#)?.is_scraper());
+        assert_eq!(false, serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"kinds": [1984],"authors":["aaaa"]}]"#)?.is_scraper());
+        assert_eq!(false, serde_json::from_str::<Subscription>(r#"["REQ","some-id",{"ids": ["aaaa"]}]"#)?.is_scraper());
+        assert_eq!(false, serde_json::from_str::<Subscription>(r##"["REQ","some-id",{"#p": ["aaaa"],"kinds":[1,4]}]"##)?.is_scraper());
+        Ok(())
+    }
 }