big update

Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 rss = "2.0"
 reqwest = {version = "0.11", default-features = false, features = ["rustls"]}
 tokio = {version = "1.28", features = ["rt-multi-thread", "signal"]}
-lazy-regex = "2.5.0"
+lazy-regex = "3.0.2"
 #time = {version = "0.3", features = ["local-offset" ,"macros", "parsing"]}
 chrono = "0.4.25"
 chrono-tz = "0.8.2"
@@ -17,6 +17,8 @@ anyhow = "1.0.71"
 log = "0.4.18"
 env_logger = "0.10.0"
 dotenvy = "0.15.7"
+sled = { version = "0.34.7", features = ["compression"] }
+bincode = "1.3.3"
 
 [dependencies.serenity]
 version = "0.11"

src/main.rs

@@ -1,101 +1,146 @@
 use anyhow::{anyhow, Context};
-use chrono::{DateTime, NaiveDateTime, TimeZone};
+use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
 use lazy_regex::regex;
-use log::{debug, error, info};
+use log::{debug, info};
 use reqwest::IntoUrl;
-use rss::{Channel, Item};
 use serenity::http::Http;
 use serenity::json::Value;
 use serenity::model::channel::Embed;
-use serenity::model::prelude::Webhook;
+use serenity::model::webhook::Webhook;
-use std::collections::HashSet;
+use std::borrow::Cow;
 use std::env;
-use std::fs::{File, OpenOptions};
-use std::io::{BufRead, BufReader, Write};
-use std::time::Duration;
 
-const REFRESH_INTERVAL: Duration = Duration::from_secs(10 * 60);
+struct RefreshDb(sled::Db);
 
-async fn get_rss_channel<T: IntoUrl>(url: T) -> anyhow::Result<Channel> {
+impl RefreshDb {
-    let content = reqwest::get(url).await?.bytes().await?;
+    fn get(&self, guid: &str) -> anyhow::Result<Option<DateTime<Utc>>> {
-    let channel = Channel::read_from(&content[..])?;
+        self.0
-    Ok(channel)
+            .get(guid)
-}
+            .with_context(|| "db error: unable to get value")?
+            .map(|b| {
+                bincode::deserialize::<i64>(&b)
+                    .with_context(|| "db error: unable to deserialize i64")
+            })
+            .transpose()?
+            .map(|i| {
+                Utc.timestamp_opt(i, 0)
+                    .single()
+                    .with_context(|| "db error: unable to parse timestamp from i64")
+            })
+            .transpose()
+    }
 
-fn validity_time_to_timestamp(input: &str) -> anyhow::Result<i64> {
+    fn insert(&self, guid: String, datetime: DateTime<Utc>) -> anyhow::Result<()> {
-    let naive = NaiveDateTime::parse_from_str(input, "%Y-%m-%d %H:%M:%S")?;
+        let timestamp = datetime.timestamp();
-    let timestamp = chrono_tz::Europe::Berlin
+        let bytes = bincode::serialize(&timestamp).with_context(|| "db error")?;
-        .from_local_datetime(&naive)
+        self.0.insert(guid, bytes).with_context(|| "db error")?;
-        .unwrap()
+        Ok(())
-        .timestamp();
-    Ok(timestamp)
-}
-
-fn html_to_discord_markdown(input: &str) -> String {
-    let re_times = regex!(r#"^.*<br\s*/>\s*<br\s*/>"#i);
-    let re_bold = regex!(r#"<b\s*>((.|\n)*?)</b\s*>"#i);
-    let re_italic = regex!(r#"<i\s*>((.|\n)*?)</i\s*>"#i);
-    let re_strikethrough = regex!(r#"<s\s*>((.|\n)*?)</s\s*>"#i);
-    let re_newline = regex!(r#"<br\s*/?>"#i);
-
-    let input = re_times.replace(input, "");
-    let input = re_bold.replace_all(&input, "**$1**");
-    let input = re_italic.replace_all(&input, "*$1*");
-    let input = re_strikethrough.replace_all(&input, "~~$1~~");
-    let input = re_newline.replace_all(&input, "\n");
-    input.to_string()
-}
-
-fn icon_name_to_url(name: &str) -> &str {
-    match name {
-        "HIM1" => "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a9/Zeichen_123_-_Arbeitsstelle%2C_StVO_2013.svg/273px-Zeichen_123_-_Arbeitsstelle%2C_StVO_2013.svg.png",
-        "HIM2" => "https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/Zeichen_101_-_Gefahrstelle%2C_StVO_1970.svg/273px-Zeichen_101_-_Gefahrstelle%2C_StVO_1970.svg.png",
-        _ => "https://upload.wikimedia.org/wikipedia/commons/thumb/8/8a/RWB-RWBA_Information.svg/240px-RWB-RWBA_Information.svg.png",
     }
 }
 
-fn item_to_embed(item: &Item) -> anyhow::Result<Value> {
+struct DisbahnClient {
-    const COLOUR: u32 = 0x008d4f;
+    webhook: Webhook,
-    const FOOTER_ICON_URL: &str = "https://www.zuginfo.nrw/img/customer/apple-touch-icon.png";
+    http: Http,
+    rss_url: String,
+    known_guids: RefreshDb,
+}
 
-    let categories = item.categories();
+impl DisbahnClient {
+    fn new(webhook: Webhook, http: Http, rss_url: String, known_guids: RefreshDb) -> Self {
+        Self {
+            webhook,
+            http,
+            rss_url,
+            known_guids,
+        }
+    }
 
-    let title = item.title().ok_or(anyhow!("Missing title"))?;
+    async fn get_rss_channel<T: IntoUrl>(url: T) -> anyhow::Result<rss::Channel> {
-    let link = item.link().ok_or(anyhow!("Missing link"))?;
+        let content = reqwest::get(url).await?.bytes().await?;
+        let channel = rss::Channel::read_from(&content[..])?;
+        Ok(channel)
+    }
 
-    let validity_begin = &categories
+    fn validity_time_to_timestamp(input: &str) -> anyhow::Result<i64> {
-        .iter()
+        let naive = NaiveDateTime::parse_from_str(input, "%Y-%m-%d %H:%M:%S")?;
-        .find(|c| c.domain() == Some("validityBegin"))
+        let timestamp = chrono_tz::Europe::Berlin
-        .ok_or(anyhow!("Missing validityBegin category"))?
+            .from_local_datetime(&naive)
-        .name;
+            .unwrap()
-    let validity_begin = validity_time_to_timestamp(validity_begin)?;
+            .timestamp();
+        Ok(timestamp)
+    }
 
-    let validity_end = &categories
+    fn html_to_discord_markdown(input: &str) -> String {
-        .iter()
+        let re_times = regex!(r#"^.*<br\s*/>\s*<br\s*/>"#i);
-        .find(|c| c.domain() == Some("validityEnd"))
+        let re_bold = regex!(r#"<b\s*>((.|\n)*?)</b\s*>"#i);
-        .ok_or(anyhow!("Missing validityEnd category"))?
+        let re_italic = regex!(r#"<i\s*>((.|\n)*?)</i\s*>"#i);
-        .name;
+        let re_strikethrough = regex!(r#"<s\s*>((.|\n)*?)</s\s*>"#i);
-    let validity_end = validity_time_to_timestamp(validity_end)?;
+        let re_newline = regex!(r#"<br\s*/?>"#i);
 
-    let icon = categories
+        let input = re_times.replace(input, "");
-        .iter()
+        let input = re_bold.replace_all(&input, "**$1**");
-        .find(|c| c.domain() == Some("icon"))
+        let input = re_italic.replace_all(&input, "*$1*");
-        .map(|c| c.name())
+        let input = re_strikethrough.replace_all(&input, "~~$1~~");
-        .unwrap_or("");
+        let input = re_newline.replace_all(&input, "\n");
-    let icon_url = icon_name_to_url(icon);
+        input.to_string()
+    }
 
-    let description =
+    fn icon_name_to_url(name: &str) -> &str {
-        html_to_discord_markdown(item.description().ok_or(anyhow!("Missing description"))?);
+        match name {
+            "HIM1" => "https://upload.wikimedia.org/wikipedia/commons/thumb/a/a9/Zeichen_123_-_Arbeitsstelle%2C_StVO_2013.svg/273px-Zeichen_123_-_Arbeitsstelle%2C_StVO_2013.svg.png",
+            "HIM2" => "https://upload.wikimedia.org/wikipedia/commons/thumb/0/02/Zeichen_101_-_Gefahrstelle%2C_StVO_1970.svg/273px-Zeichen_101_-_Gefahrstelle%2C_StVO_1970.svg.png",
+            _ => "https://upload.wikimedia.org/wikipedia/commons/thumb/5/56/Zeichen_365-61_-_Informationsstelle%2C_StVO_2013.svg/240px-Zeichen_365-61_-_Informationsstelle%2C_StVO_2013.svg.png",
+        }
+    }
 
-    let pub_date = item.pub_date().ok_or(anyhow!("Missing publication date"))?;
+    fn item_to_embed(item: &rss::Item, is_update: bool) -> anyhow::Result<Value> {
-    let pub_timestamp = DateTime::parse_from_rfc2822(pub_date)
+        const COLOUR: u32 = 0x008d4f;
-        .with_context(|| format!("Unable to parse publication date string {pub_date:?}"))?
+        const FOOTER_ICON_URL: &str = "https://www.zuginfo.nrw/img/customer/apple-touch-icon.png";
-        .naive_utc()
-        .and_utc()
-        .to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
 
-    let embed = Embed::fake(|e| {
+        let categories = item.categories();
-        e.title(title)
+
+        let title = item.title().ok_or(anyhow!("Missing title"))?;
+        let link = item.link().ok_or(anyhow!("Missing link"))?;
+
+        let validity_begin = &categories
+            .iter()
+            .find(|c| c.domain() == Some("validityBegin"))
+            .ok_or(anyhow!("Missing validityBegin category"))?
+            .name;
+        let validity_begin = Self::validity_time_to_timestamp(validity_begin)?;
+
+        let validity_end = &categories
+            .iter()
+            .find(|c| c.domain() == Some("validityEnd"))
+            .ok_or(anyhow!("Missing validityEnd category"))?
+            .name;
+        let validity_end = Self::validity_time_to_timestamp(validity_end)?;
+
+        let icon = categories
+            .iter()
+            .find(|c| c.domain() == Some("icon"))
+            .map(|c| c.name())
+            .unwrap_or("");
+        let icon_url = Self::icon_name_to_url(icon);
+
+        let description = Self::html_to_discord_markdown(
+            item.description().ok_or(anyhow!("Missing description"))?,
+        );
+
+        let pub_date_str = item.pub_date().ok_or(anyhow!("Missing publication date"))?;
+        let pub_datetime = DateTime::parse_from_rfc2822(pub_date_str)
+            .with_context(|| format!("Unable to parse publication date string {pub_date_str:?}"))?
+            .naive_utc()
+            .and_utc();
+
+        let pub_timestamp = pub_datetime.to_rfc3339_opts(chrono::SecondsFormat::Secs, true);
+
+        let embed = Embed::fake(|e| {
+            e.title(if is_update {
+                Cow::Owned(format!("UPDATE: {title}"))
+            } else {
+                Cow::Borrowed(title)
+            })
             .url(link)
             .thumbnail(icon_url)
             .colour(COLOUR)
@@ -108,38 +153,56 @@ fn item_to_embed(item: &Item) -> anyhow::Result<Value> {
                 f.text("Quelle: https://zuginfo.nrw/ \u{2013} Alle Angaben ohne Gewehr \u{1F52B}")
                     .icon_url(FOOTER_ICON_URL)
             })
-    });
+        });
 
-    Ok(embed)
+        Ok(embed)
-}
+    }
 
-async fn refresh_and_process_rss(
+    async fn refresh(&mut self) -> anyhow::Result<()> {
-    feed_url: &str,
+        debug!("Refreshing RSS feed ...");
-    http: &Http,
+        let channel = Self::get_rss_channel(&self.rss_url)
-    webhook: &Webhook,
+            .await
-    known_guids: &mut HashSet<String>,
+            .map_err(|e| anyhow!(e.to_string()))
-    known_guids_file: &mut File,
+            .with_context(|| "Failed to get RSS channel")?;
-) -> anyhow::Result<()> {
+
-    debug!("Refreshing RSS feed ...");
+        let items = channel.items();
-    let channel = get_rss_channel(feed_url)
+
-        .await
+        for item in items {
-        .map_err(|e| anyhow!(e.to_string()))
+            let guid = item.guid().ok_or(anyhow!("Missing GUID"))?.value();
-        .with_context(|| "Failed to get RSS channel")?;
+            let pub_date_str = item.pub_date().ok_or(anyhow!("Missing publication date"))?;
-    for item in channel.items() {
+            let pub_datetime = DateTime::parse_from_rfc2822(pub_date_str)
-        let guid = item.guid().ok_or(anyhow!("Missing GUID"))?.value();
+                .with_context(|| {
-        if !known_guids.contains(guid) {
+                    format!("Unable to parse publication date string {pub_date_str:?}")
-            info!("New item: {}", guid);
+                })?
-            known_guids.insert(guid.to_owned());
+                .naive_utc()
-            writeln!(known_guids_file, "{}", guid).with_context(|| "Failed to write known GUID")?;
+                .and_utc();
-            let embed = item_to_embed(item)?;
+
-            webhook
+            let is_update;
-                .execute(&http, false, |w| w.embeds(vec![embed]))
+            if let Some(last_pub_datetime) = self.known_guids.get(guid)? {
+                if last_pub_datetime < pub_datetime {
+                    is_update = true;
+                    info!("Updated item: {guid}");
+                } else {
+                    continue;
+                }
+            } else {
+                is_update = false;
+                info!("New item: {guid}");
+            }
+
+            let embed = Self::item_to_embed(item, is_update)?;
+            self.webhook
+                .execute(&self.http, false, |w| w.embeds(vec![embed]))
                 .await
                 .with_context(|| "Failed to execute webhook")?;
+            self.known_guids
+                .insert(guid.to_string(), pub_datetime)
+                .with_context(|| "Failed to write to database")?;
         }
+
+        debug!("Done.");
+        Ok(())
     }
-    debug!("Done.");
-    Ok(())
 }
 
 #[tokio::main]
@@ -152,36 +215,15 @@ async fn main() -> anyhow::Result<()> {
 
     let webhook_url = env::var("WEBHOOK_URL")?;
     let feed_url = env::var("FEED_URL")?;
-    let known_guids_file = env::var("KNOWN_GUIDS_FILE")?;
+    let known_guids_db = env::var("KNOWN_GUIDS_DB")?;
 
-    let mut known_guids_file = OpenOptions::new()
+    let known_guids_db =
-        .read(true)
+        RefreshDb(sled::open(known_guids_db).with_context(|| "unable to open database")?);
-        .write(true)
-        .create(true)
-        .open(known_guids_file)?;
 
     let http = Http::new("");
     let webhook = http.get_webhook_from_url(&webhook_url).await.unwrap();
 
-    let mut known_guids: HashSet<String> = HashSet::new();
+    let mut disbahn_client = DisbahnClient::new(webhook, http, feed_url, known_guids_db);
-    for line in BufReader::new(&known_guids_file).lines() {
-        let line = line?;
-        known_guids.insert(line);
-    }
 
-    loop {
+    disbahn_client.refresh().await
-        if let Err(err) = refresh_and_process_rss(
-            &feed_url,
-            &http,
-            &webhook,
-            &mut known_guids,
-            &mut known_guids_file,
-        )
-        .await
-        {
-            error!("Error: {}", err);
-        }
-
-        tokio::time::sleep(REFRESH_INTERVAL).await;
-    }
 }