From 832d9afd8c03646aab67b8d3661b3357574459b8 Mon Sep 17 00:00:00 2001 From: Byte Dice Date: Wed, 26 Feb 2025 22:13:50 +0100 Subject: [PATCH] handling old posts & /bk_week_top --- README.md | 4 +- TODO.md | 6 +- bk_week_help_dc.md | 20 ++--- src/bk_week_cmds.rs | 162 ++++++++++++++++++++++++++++++------- src/main.rs | 1 + src/messages.rs | 3 + src/python/data.py | 27 ++++++- src/python/posts.py | 20 ++++- src/python/py_websocket.py | 3 +- 9 files changed, 194 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index d9dc397..d4cd8f1 100644 --- a/README.md +++ b/README.md @@ -42,5 +42,5 @@ You can install Python modules by running `$ pip install {module}` or `$ python ### How to run: * Download the code (and extract if needed). * Open a terminal and CD to the downloaded folder. -* Run `$ cargo run`. There are more options when running. You can view a list of those using `$ cargo run -- --help`. - * If you want to only run the Python code, you can either run `$ cargo run -- --py`, or `$ python ./src/python/main.py`. The second option is recommended for better output. \ No newline at end of file +* Run `$ cargo run`. There are more options when running. You can view a list of those using `$ cargo run -- --help` or `$ cargo run -- -h`. + * If you want to only run the Python code, you can either run `$ cargo run -- --py`, or `$ python ./src/python/main.py`. The second option is recommended for better error output. \ No newline at end of file diff --git a/TODO.md b/TODO.md index 0b0072a..d01657d 100644 --- a/TODO.md +++ b/TODO.md @@ -7,7 +7,7 @@ - - [ ] `/bk_week_top [category] [amount]` to get the top N posts in a category (e.g upvotes) + - [ ] Allow updating the data autonomously and via manual commands. @@ -19,9 +19,7 @@ - - [ ] Automatically remove posts older than 7 days from JSON - - [x] ~~Function~~ - - [ ] Automate + // need python function to remove data about them - [ ] Automatically approve posts that don't get caught by reverse image search (ris) diff --git a/bk_week_help_dc.md b/bk_week_help_dc.md index 10f1135..46d9102 100644 --- a/bk_week_help_dc.md +++ b/bk_week_help_dc.md @@ -1,29 +1,29 @@ # Discord Commands +`[this means it's a required argument]` +`` + ## `/bk_week_get [url]` Shows info about a single post. ## `/bk_week_add [url] ` Adds a URL to the list of posts. This is done automatically by the bot for certain posts. -- **``**: (OPTIONAL) `true` or `false`, determines whether to approve the post when added. +- **``**: `true` or `false`, determines whether to automatically approve the post when added. ## `/bk_week_remove [url]` Removes an existing URL from the list of posts. ## `/bk_week_approve [url] ` Flags the post as **human_approved**, confirming that the artwork is original. -- **``**: (OPTIONAL) `true` or `false`, set to `true` if you want to undo an approval. +- **``**: set to `true` if you want to undo an approval. ## `/bk_week_vote [url] ` Adds a vote to a post. The intended use for votes is for a "moderator picks" and a "community picks" section of the weekly art. You can vote on as many posts as you want, but only once per post. -## `/bk_week_update` +## `/bk_week_update ` Updates all data in the bound Discord channel. It's recommended to only run this if absolutely needed to. +- **``**: If it's `true`, the bot will only add new posts, not remove or update them. +- **``**: The bot will remove any post older than `max_age` days. (0 means infinite.) +## `/bk_week_top [category] ` +Shows you the top posts within a category, such as upvotes. (max 10 posts, default is 3) ## `/bk_admin_bind` Binds the current channel as the channel where the bk_week data is sent and updated in. No binding means the only way to view the data is using `/bk_week_get`. ## `/bk_week_help [option]` Shows a help text. -- **`[option]`**: `Discord` or `Reddit`, determines what help text to send. -### **Examples** -``` -/bk_week_add https://reddit.com/post_url false -/bk_week_add https://reddit.com/post_url -/bk_week_approve https://reddit.com/post_url -``` # Things to note * This bot uses shortURLs when storing posts. If you want to access posts, you should use their shortURL. You can get a Reddit shortURL by running `/re_shorturl [url]`. \ No newline at end of file diff --git a/src/bk_week_cmds.rs b/src/bk_week_cmds.rs index c3c6f50..9475863 100644 --- a/src/bk_week_cmds.rs +++ b/src/bk_week_cmds.rs @@ -3,9 +3,12 @@ use crate::{cmds, rs_println, websocket, Context, Data, Error, BK_WEEK}; use crate::messages::*; use crate::data::{self, dc_bind_bk}; +use std::collections::HashMap; use std::fs; +use std::time::{SystemTime, UNIX_EPOCH}; use poise::serenity_prelude::{ChannelId, EditMessage, GetMessages, Http, Message, MessageId, UserId}; +use poise::ReplyHandle; use serde_json::{json, Map, Value}; @@ -14,6 +17,13 @@ enum HelpOptions { Discord, Reddit } +#[derive(poise::ChoiceParameter, PartialEq)] +enum TopCategory { + Upvotes, + ModVotes, + Oldest, + Newest +} fn is_bk_mod(mod_list: Value, uid: u64) -> bool { @@ -342,20 +352,21 @@ async fn send_server_not_in_data_msg(ctx: Context<'_>) { /// Updates all logs pub async fn bk_week_update( ctx: Context<'_>, - #[description = "Only adds new posts, leaves everything else unchanged."] only_add: Option + #[description = "Only adds new posts, leaves everything else unchanged."] only_add: Option, + #[description = "The max age of a post (in days). Any post older than this will be removed. (0 is infinite.)"] max_age: Option ) -> Result<(), Error> { let http = ctx.http(); - let executed = format!("(Executed `/bk_week_update`, author: `{}`)", ctx.author().name); - let mut p_text = executed.clone(); + let mut p_text = "`/bk_week_update`:".to_string(); - send_msg(ctx, MANDATORY_MSG.to_string(), true, true).await; + let progress = send_msg(ctx, p_text.clone(), true, true).await.unwrap(); + p_text = update_progress(ctx, progress.clone(), p_text, "\nFetching new posts & updating data file...".to_string()).await; - let progress = http_send_msg(http, ctx.channel_id(), p_text.clone()).await.unwrap(); - p_text = update_progress(ctx.http(), progress.clone(), p_text, "\nFetching new posts & updating data file...".to_string()).await; + let max_age_u = max_age.unwrap_or_else(|| 8); + let max_age_secs = max_age_u as u64 * (60 * 60 * 24); - send_cmd_json("add_new_posts", None).await; + send_cmd_json("add_new_posts", Some(json!([max_age_secs]))).await; data::update_re_data(ctx.data()).await; let r_data = get_reddit_data(ctx.data()).await.unwrap(); @@ -369,51 +380,56 @@ pub async fn bk_week_update( let c_id = c_id_u.unwrap(); // Reading messages - p_text = update_progress(http, progress.clone(), p_text.clone(), format!("✅\nReading messages in <#{}>...", c_id)).await; + p_text = update_progress(ctx, progress.clone(), p_text.clone(), format!("✅\nReading messages in <#{}>...", c_id)).await; let msgs = read_msgs(http, ctx.framework().bot_id, c_id).await; // Parsing messages to JSON - p_text = update_progress(http, progress.clone(), p_text.clone(), "✅\nParsing messages to JSON...".to_string()).await; - let msgs_json = msgs_to_json(msgs, &r_data).await; + p_text = update_progress(ctx, progress.clone(), p_text.clone(), "✅\nParsing messages to JSON...".to_string()).await; + let msgs_json = msgs_to_json(msgs, &r_data, max_age_secs).await; // Adding new posts - p_text = update_progress(http, progress.clone(), p_text.clone(), "✅\nAdding new posts...".to_string()).await; + p_text = update_progress(ctx, progress.clone(), p_text.clone(), "✅\nAdding new posts...".to_string()).await; let weekly_art = r_data[BK_WEEK].as_object().unwrap(); add_posts(http, c_id, weekly_art, &msgs_json).await; // Stop if only_add if only_add.unwrap_or_else(|| false) { send_msg(ctx, "`/bk_week_update`\n## Done!".to_string(), true, true).await; - update_progress(http, progress.clone(), String::new(), executed).await; + update_progress(ctx, progress.clone(), p_text, "✅\n## Done!".to_string()).await; return Ok(()); } // Editing updated posts - p_text = update_progress(http, progress.clone(), p_text.clone(), "✅\nEditing updated posts...".to_string()).await; + p_text = update_progress(ctx, progress.clone(), p_text.clone(), "✅\nEditing updated posts...".to_string()).await; edit_posts(http, c_id, weekly_art, &msgs_json).await; // Removing removed posts - p_text = update_progress(http, progress.clone(), p_text.clone(), "✅\nRemoving removed posts...".to_string()).await; + p_text = update_progress(ctx, progress.clone(), p_text.clone(), "✅\nRemoving removed posts...".to_string()).await; remove_posts(http, c_id, weekly_art, &msgs_json).await; + // Removing old posts + if max_age_u > 0 { + p_text = update_progress(ctx, progress.clone(), p_text.clone(), format!("✅\nRemoving old posts (threshold: {}d)...", max_age_u)).await; + remove_old(http, c_id, &msgs_json).await; + send_cmd_json("remove_old_posts", Some(json!([max_age_secs]))).await; + } + // Removing duplicate posts - update_progress(http, progress.clone(), p_text.clone(), "✅\nRemoving duplicate posts...".to_string()).await; + p_text = update_progress(ctx, progress.clone(), p_text.clone(), "✅\nRemoving duplicate posts...".to_string()).await; remove_dupes(http, c_id, &msgs_json).await; // Done + update_progress(ctx, progress.clone(), p_text, "✅\n## Done!".to_string()).await; send_msg(ctx, "`/bk_week_update`\n## Done!".to_string(), true, true).await; - update_progress(http, progress.clone(), String::new(), executed).await; return Ok(()); } -async fn update_progress(http: &Http, p: Message, t: String, added_t: String) -> String { +async fn update_progress(ctx: Context<'_>, p: ReplyHandle<'_>, t: String, added_t: String) -> String { let p_text = format!("{} {}", t, added_t); - let new_msg = EditMessage::new().content(&p_text); - - http_edit_msg(http, p, new_msg).await; + edit_reply(ctx, p, p_text.clone()).await; return p_text; } @@ -437,7 +453,7 @@ async fn get_c_id(ctx: Context<'_>) -> Option { } -pub async fn read_msgs(http: &Http, bot_id: UserId, c_id: ChannelId) -> Vec { +async fn read_msgs(http: &Http, bot_id: UserId, c_id: ChannelId) -> Vec { let b = GetMessages::new().limit(100); let mut msgs = c_id.messages(http, b).await.unwrap(); msgs = msgs.into_iter().filter(|item| item.author.id == bot_id).collect(); @@ -466,8 +482,12 @@ pub async fn read_msgs(http: &Http, bot_id: UserId, c_id: ChannelId) -> Vec(msgs: Vec, reddit_data: &'a Value) -> Value { - let mut msgs_json: Value = json!({"no_change": {}, "updated": {}, "removed": {}, "duplicates": {}}); +async fn msgs_to_json<'a>(msgs: Vec, reddit_data: &'a Value, max_age: u64) -> Value { + let mut msgs_json: Value = json!({"no_change": {}, "updated": {}, "removed": {}, "duplicates": {}, "old": {}}); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs() as u64; for msg in msgs { if msg.embeds.len() == 0 { continue; } @@ -475,7 +495,8 @@ pub async fn msgs_to_json<'a>(msgs: Vec, reddit_data: &'a Value) -> Val let url = msg.embeds[0].url.clone().unwrap(); - if ["no_change", "updated", "removed"] + // duplicates + if ["no_change", "updated", "removed", "old"] .iter() .any(|key| msgs_json[key].as_object().unwrap().contains_key(&url)) { @@ -500,20 +521,34 @@ pub async fn msgs_to_json<'a>(msgs: Vec, reddit_data: &'a Value) -> Val let mut u_json: Value = msg_json.unwrap(); let re_url = &reddit_data[BK_WEEK][&url]; + let post_date = re_url["post_data"]["date_unix"].as_u64().unwrap_or_else(|| 0); + + // old + if now - post_date > max_age { + if let Some(obj) = msgs_json["old"].as_object_mut() { + obj.insert(url.clone(), json!(msg.id.get())); + continue; + } + } + + // removed if re_url.get("removed").is_some() { if u_json.get("removed").is_some() { + // no change if let Some(obj) = msgs_json["no_change"].as_object_mut() { obj.insert(url.clone(), json!(msg.id.get())); continue; } } + // removed if let Some(obj) = msgs_json["removed"].as_object_mut() { obj.insert(url.clone(), json!(msg.id.get())); continue; } } + // updated if u_json["added"] != re_url["added"] || u_json["approved"] != re_url["approved"] || u_json["post_data"]["upvotes"] != re_url["post_data"]["upvotes"] @@ -527,6 +562,7 @@ pub async fn msgs_to_json<'a>(msgs: Vec, reddit_data: &'a Value) -> Val } } + // no change if let Some(obj) = msgs_json["no_change"].as_object_mut() { obj.insert(url.clone(), json!(msg.id.get())); } @@ -536,9 +572,9 @@ pub async fn msgs_to_json<'a>(msgs: Vec, reddit_data: &'a Value) -> Val } -pub async fn add_posts(http: &Http, c_id: ChannelId, r_data: &Map, msgs_json: &Value) { +async fn add_posts(http: &Http, c_id: ChannelId, r_data: &Map, msgs_json: &Value) { for url in r_data.keys() { - if ["no_change", "updated", "removed"] + if ["no_change", "updated", "removed", "old"] .iter() .any(|key| msgs_json[key].as_object().unwrap().contains_key(url)) { continue; } @@ -554,7 +590,7 @@ pub async fn add_posts(http: &Http, c_id: ChannelId, r_data: &Map } -pub async fn edit_posts(http: &Http, c_id: ChannelId, r_data: &Map, msgs_json: &Value) { +async fn edit_posts(http: &Http, c_id: ChannelId, r_data: &Map, msgs_json: &Value) { for (url, msg_id) in msgs_json["updated"].as_object().unwrap() { let mut msg = http.get_message(c_id, MessageId::new(msg_id.as_u64().unwrap())).await.unwrap(); let r = EditMessage::new() @@ -565,7 +601,7 @@ pub async fn edit_posts(http: &Http, c_id: ChannelId, r_data: &Map, msgs_json: &Value) { +async fn remove_posts(http: &Http, c_id: ChannelId, r_data: &Map, msgs_json: &Value) { for (url, msg_id) in msgs_json["removed"].as_object().unwrap() { let mut msg = http.get_message(c_id, MessageId::new(msg_id.as_u64().unwrap())).await.unwrap(); let r = EditMessage::new() @@ -576,7 +612,15 @@ pub async fn remove_posts(http: &Http, c_id: ChannelId, r_data: &Map, + #[description = "The sorting criteria, such as upvotes."] category: TopCategory, + #[description = "The amount of posts to show (max 10)."] amount: Option +) -> Result<(), Error> +{ + let mut all: HashMap<&str, i32> = HashMap::new(); + let posts = &get_reddit_data(ctx.data()).await.unwrap()[BK_WEEK]; + let posts_u = posts.as_object().unwrap(); + + for (url, dat) in posts_u { + if dat.get("removed").is_some() { continue; } + + let val: i32 = match category { + TopCategory::Upvotes => dat["post_data"]["upvotes"].as_i64().unwrap() as i32, + TopCategory::ModVotes => dat["votes"]["mod_voters"].as_array().unwrap().len() as i32, + TopCategory::Oldest + | TopCategory::Newest => dat["post_data"]["date_unix"].as_i64().unwrap() as i32, + }; + + all.insert(url, val); + } + + let amount_u = amount.unwrap_or_else(|| 3); + let amount_clamped = + if amount_u > 10 { 10 } + else if amount_u < 1 { 1 } + else { amount_u }; + + let top = + if category != TopCategory::Oldest + { largest_n (&all, amount_clamped as usize) } + else { smallest_n(&all, amount_clamped as usize) }; + + for post in top { + let url = post.0; + let _ = send_embed_for_post(ctx, posts_u[url].clone(), &url).await; + } + + return Ok(()); +} + + +fn largest_n<'a>(map: &'a HashMap<&'a str, i32>, n: usize) -> Vec<(&'a str, i32)> { + let mut vec: Vec<_> = map.iter().collect(); + vec.sort_unstable_by(|a, b| b.1.cmp(a.1)); + vec.into_iter().take(n).map(|(&k, &v)| (k, v)).collect() +} + + +fn smallest_n<'a>(map: &'a HashMap<&'a str, i32>, n: usize) -> Vec<(&'a str, i32)> { + let mut vec: Vec<_> = map.iter().collect(); + vec.sort_unstable_by(|a, b| a.1.cmp(b.1)); + vec.into_iter().take(n).map(|(&k, &v)| (k, v)).collect() +} + + + #[poise::command(slash_command, prefix_command, default_member_permissions = "ADMINISTRATOR")] /// Changes the subreddit(s) the bot patrols in. diff --git a/src/main.rs b/src/main.rs index 90a9349..34314c4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -197,6 +197,7 @@ async fn gen_bot(data: Data, args: Args) -> Client { bk_week_cmds::bk_week_approve(), bk_week_cmds::bk_week_update(), bk_week_cmds::bk_week_vote(), + bk_week_cmds::bk_week_top(), // bk_admin bk_week_cmds::bk_admin_bind(), // bk_cfg diff --git a/src/messages.rs b/src/messages.rs index 4737432..e7d2bfd 100644 --- a/src/messages.rs +++ b/src/messages.rs @@ -77,6 +77,7 @@ pub async fn send_msg( } +#[allow(dead_code)] pub async fn http_send_msg( http: &Http, c_id: ChannelId, @@ -167,6 +168,7 @@ pub async fn edit_reply( } +#[allow(dead_code)] pub async fn http_edit_msg( http: &Http, mut msg: Message, @@ -265,6 +267,7 @@ pub fn embed_post_removed(post_data: &Value, url: &str, empheral: bool) -> Embed ), col: Some(REMOVED_DC_COL), url: Some(url.to_string()), + ts: Some(Timestamp::from_unix_timestamp(post_data["post_data"]["date_unix"].as_i64().unwrap()).unwrap()), empheral, ..Default::default() }; diff --git a/src/python/data.py b/src/python/data.py index e882541..606ea0a 100644 --- a/src/python/data.py +++ b/src/python/data.py @@ -1,6 +1,7 @@ import os import json from typing import Final +import time import bot as botPy from macros import * @@ -136,13 +137,35 @@ def set_approve_post(bot: botPy.Bot, approved: bool, url: str) -> bool: def remove_post(bot: botPy.Bot, url: str, removed_by: str = "UNKNOWN", reason: str = "None") -> bool: - if url in bot.data[BK_WEEKLY]: - bot.data[BK_WEEKLY][url] = { "removed": True, "removed_by": removed_by, "remove_reason": reason } + weekly = bot.data[BK_WEEKLY] + + if url in weekly: + weekly[url] = { + "removed": True, + "removed_by": removed_by, + "remove_reason": reason, + "post_data": { "date_unix": weekly[url]["post_data"]["date_unix"] } + } return True else: return False +def remove_old_posts(bot: botPy.Bot, max_age: int) -> bool: + now = int(time.time()) + weekly = bot.data[BK_WEEKLY] + remove: list[str] = [] + + for url, post in weekly.items(): + if now - post["post_data"]["date_unix"] > max_age: + remove.append(url) + + for key in remove: + weekly.pop(key) + + return True + + def set_vote_post( bot: botPy.Bot, url: str, diff --git a/src/python/posts.py b/src/python/posts.py index 4c9607a..bbee15d 100644 --- a/src/python/posts.py +++ b/src/python/posts.py @@ -2,13 +2,14 @@ import emoji from asyncpraw import models import asyncprawcore as prawcore import asyncpraw.exceptions as exc +import time import data import bot as botPy from macros import * -async def add_new_posts(bot: botPy.Bot) -> bool: +async def add_new_posts(bot: botPy.Bot, max_age: int) -> bool: check_emoji = emoji.emojize(":check_mark_button:") cross_emoji = emoji.emojize(":cross_mark:") @@ -20,6 +21,7 @@ async def add_new_posts(bot: botPy.Bot) -> bool: added_posts = 0 without_media = 0 not_added = 0 + old_posts = 0 for post in posts: media = has_media(post) @@ -32,14 +34,23 @@ async def add_new_posts(bot: botPy.Bot) -> bool: f"\n {check_emoji if media[0] else cross_emoji} Media ({media[1]}) [{media[2]}]", f"\n {media_urls}\n" ) - + + details = get_post_details(post) + now = int(time.time()) + + if now - details.date_unix > max_age and max_age > 0: + old_posts += 1 + continue + if not media[0]: without_media += 1 continue + post_added = False + post_added = data.add_post_to_data( bot, - get_post_details(post) + details ) if post_added: added_posts += 1 @@ -48,7 +59,8 @@ async def add_new_posts(bot: botPy.Bot) -> bool: py_print(f"Successfully fetched {len(posts)} posts.\n" + f" Out of which were {added_posts} added.\n" + f" {without_media} had no media, " + - f"and {not_added} weren't added because they are removed or already existed") + f"{not_added} are removed or already existed, " + + f"and {old_posts} were older than the max age threshold.") return True diff --git a/src/python/py_websocket.py b/src/python/py_websocket.py index fbbafb5..01be77d 100644 --- a/src/python/py_websocket.py +++ b/src/python/py_websocket.py @@ -75,12 +75,13 @@ async def json_to_func(v: dict, bot: botPy.Bot) -> dict: match v["value"]: case "update_data_file": r = data .write_data (bot) - case "add_new_posts": r = await posts.add_new_posts (bot) case "respond_mentions": r = await cmds .respond_to_mention(bot) + case "add_new_posts": r = await posts.add_new_posts (bot, *v["args"]) case "add_post_url": r = await posts.add_post_url (bot, *v["args"]) case "remove_post_url": r = data .remove_post (bot, *v["args"]) case "set_approve_post": r = data .set_approve_post (bot, *v["args"]) case "set_vote_post": r = data .set_vote_post (bot, *v["args"]) + case "remove_old_posts": r = data .remove_old_posts (bot, *v["args"]) case "change_sr": r = await bot .change_sr (*v["args"]) case "stop_praw": r = await bot .stop () case _: value_supported = False