emojis are now replaced by images ! (except for unicode emoji)

Cargo.toml

@@ -23,4 +23,7 @@ regex = "*"
 wordcloud-rs = "*"
 bimap = "*"
 tokio = { version = "*", features = ["macros", "rt-multi-thread"] }
-dashmap = "*"
+dashmap = "*"
+moka = "*"
+reqwest = "*"
+futures = "*"

src/discord_emojis.rs

@@ -0,0 +1,28 @@
+use image::DynamicImage;
+use moka::sync::Cache;
+use reqwest::get;
+use anyhow::Result;
+
+pub struct DiscordEmojis {
+    emojis: Cache<String, DynamicImage>
+}
+
+impl DiscordEmojis {
+    pub fn new(cap: usize) -> Self {
+        Self {
+            emojis: Cache::new(cap as u64)
+        }
+    }
+
+    pub async fn get(&self, id: &str) -> Result<DynamicImage> {
+        if let Some(img) = self.emojis.get(id) {
+            Ok(img.clone())
+        } else {
+            let url = format!("https://cdn.discordapp.com/emojis/{}.webp", id);
+            let img_bytes = get(url).await?.bytes().await?;
+            let image = image::load_from_memory(&img_bytes)?;
+            self.emojis.insert(id.to_string(), image.clone());
+            Ok(image)
+        }
+    }
+}

src/handler.rs

@@ -1,6 +1,7 @@
 use itertools::Itertools;
 use log::{warn, info, trace};
 use image::{write_buffer_with_format, ColorType, ImageOutputFormat};
+use regex::Regex;
 use std::{io::{Cursor, Seek, SeekFrom}, sync::Arc};
 use palette::rgb::Rgb;
 use dashmap::DashMap;
@@ -15,10 +16,16 @@ use serenity::{
     },
     prelude::*, utils::Color
 };
+use futures::future::join_all;
+use lazy_static::lazy_static;
 use wordcloud_rs::{Token, WordCloud, Colors};
-use crate::idiom::{Idioms, tokenize};
+use crate::{idiom::{Idioms, tokenize}, discord_emojis::DiscordEmojis};
 const READ_PAST: u64 = 10000;
 
+lazy_static! {
+    static ref RE_EMO: Regex = Regex::new(r"<a?:(\w*):(\d*)>").unwrap();
+}
+
 fn convert_color(color: Color) -> Rgb {
     Rgb::new(
         color.r() as f32/255., 
@@ -28,23 +35,36 @@ fn convert_color(color: Color) -> Rgb {
 }
 
 pub struct Handler {
-    idioms: Arc<DashMap<GuildId, Idioms<ChannelId, UserId>>>
+    idioms: Arc<DashMap<GuildId, Idioms<ChannelId, UserId>>>,
+    emojis: DiscordEmojis
 }
 
 impl Handler {
     pub fn new() -> Self {
         Self {
-            idioms: Arc::new(DashMap::new())
+            idioms: Arc::new(DashMap::new()),
+            emojis: DiscordEmojis::new(1000)
         }
     }
 
     pub fn message(&self, guild_id: GuildId, channel_id: ChannelId, member_id: UserId, message: String) {
         self.idioms.get_mut(&guild_id).unwrap().update(channel_id, member_id, tokenize(message));
     }
 
-    fn to_wc_tokens(&self, tokens: Vec<(String, f32)>) -> Vec<(Token, f32)> {
-        // TODO: also convert :emojis: to images 
-        tokens.into_iter().map(|(str, v)| (Token::Text(str), v)).collect_vec()
+    async fn to_wc_tokens(&self, tokens: Vec<(String, f32)>) -> Vec<(Token, f32)> {
+        join_all(tokens.into_iter().map(|(str, v)| async move {
+            if let Some(capts) = RE_EMO.captures(&str) {
+                let id = capts.get(2).unwrap().as_str();
+                if let Ok(img) = self.emojis.get(id).await {
+                    (Token::Img(img), v)
+                } else {
+                    let name = capts.get(1).unwrap().as_str();
+                    (Token::Text(name.to_string()), v)
+                }
+            } else {
+                (Token::Text(str), v)
+            }
+        }).collect_vec()).await
     }
 
     pub async fn cloud(&self, ctx: Context, command: ApplicationCommandInteraction) {
@@ -54,9 +74,9 @@ impl Handler {
                 let member_id = member.user.id;
                 let tokens = self.idioms.get(&guild_id).unwrap().idiom(member_id);
                 trace!(target: "Wordy", "/cloud: retrieved {} tokens for {}", tokens.len(), member.user.name);
-                let wc_tokens = self.to_wc_tokens(tokens);
+                let wc_tokens = self.to_wc_tokens(tokens).await;
                 let image = WordCloud::new()
-                .colors(Colors::DoubleSplitCompl(convert_color(color))).generate(wc_tokens);
+                    .colors(Colors::DoubleSplitCompl(convert_color(color))).generate(wc_tokens);
                 let mut img_file = Cursor::new(Vec::new());
                 write_buffer_with_format(
                     &mut img_file,
@@ -150,6 +170,5 @@ impl Handler {
         {
             println!("Couldn't register slash commmands: {}", why);
         };
-
     }
 }

src/idiom/idiom.rs

@@ -1,34 +1,15 @@
 use std::collections::HashMap;
 use std::hash::Hash;
 use itertools::Itertools;
-use lazy_static::lazy_static;
-use regex::Regex;
 use bimap::BiMap;
 use super::top_freqs::TopFreqs;
+use super::text_utils::counts;
 const PLACE_VOC_LEN: usize = 200;
 const PERSON_VOC_LEN: usize = 100;
 const UNIQUENESS: f32 = PERSON_VOC_LEN as f32;
 // computed so that (INV-1.)*UNIQUENESS = 1.
 const INV: f32 = 1.+1./UNIQUENESS;
 
-lazy_static! {
-    static ref RE_TOKEN: Regex = Regex::new(r"\w+").unwrap();
-}
-
-pub fn tokenize(text: String) -> Vec<String> {
-    RE_TOKEN.find_iter(&text)
-        .map(|token| token.as_str().to_string())
-        .collect_vec()
-}
-
-fn counts(tokens: Vec<String>) -> Vec<(String, f32)> {
-    let mut counts: HashMap<String, usize> = HashMap::new();
-    for token in tokens {
-        *counts.entry(token.as_str().to_string()).or_default() += 1;
-    }
-    counts.into_iter().map(|(k, v)| (k, v as f32)).collect()
-}
-
 pub struct Idioms<P: Hash+Eq, U: Hash+Eq> {
     places: HashMap<P, TopFreqs<PLACE_VOC_LEN>>,
     people: HashMap<U, TopFreqs<PERSON_VOC_LEN>>,

src/idiom/mod.rs

@@ -1,4 +1,5 @@
 mod idiom;
 mod top_freqs;
+mod text_utils;
 pub use idiom::Idioms;
-pub use idiom::tokenize;
+pub use text_utils::tokenize;

src/idiom/text_utils.rs

@@ -0,0 +1,41 @@
+use std::collections::HashMap;
+use itertools::Itertools;
+use lazy_static::lazy_static;
+use regex::Regex;
+const OPENING_PUNCT: &[char] = &['(', '[', '{', '\'', '"'];
+const CLOSING_PUNCT: &[char] = &[':', '.', '?', '!', '`', ';', ',', ')', ']', '}', '\'', '"'];
+
+lazy_static! {
+    static ref RE_TOKEN: Regex = Regex::new(r"\S+").unwrap();
+}
+
+fn is_capitalized(token: &str) -> bool {
+    !token.chars().skip(1).all(|c| c.is_lowercase())
+}
+
+fn smart_lower(token: &str) -> String {
+    if is_capitalized(token) {
+        token.to_lowercase()
+    } else {
+        token.to_string()
+    }
+}
+
+fn trim(token: &str) -> &str {
+    token.trim_start_matches(OPENING_PUNCT)
+        .trim_end_matches(CLOSING_PUNCT)
+}
+
+pub fn tokenize(text: String) -> Vec<String> {
+    RE_TOKEN.find_iter(&text)
+        .map(|token| smart_lower(trim(token.as_str())))
+        .collect_vec()
+}
+
+pub(crate) fn counts(tokens: Vec<String>) -> Vec<(String, f32)> {
+    let mut counts: HashMap<String, usize> = HashMap::new();
+    for token in tokens {
+        *counts.entry(token.as_str().to_string()).or_default() += 1;
+    }
+    counts.into_iter().map(|(k, v)| (k, v as f32)).collect()
+}

src/main.rs

@@ -1,4 +1,5 @@
 mod idiom;
+mod discord_emojis;
 mod handler;
 mod handler_util;
 mod handle_events;