handling old posts & /bk_week_top

This commit is contained in:
2025-02-26 22:13:50 +01:00
parent 2423e62c06
commit 832d9afd8c
9 changed files with 194 additions and 52 deletions
+2 -2
View File
@@ -42,5 +42,5 @@ You can install Python modules by running `$ pip install {module}` or `$ python
### How to run:
* Download the code (and extract if needed).
* Open a terminal and CD to the downloaded folder.
* Run `$ cargo run`. There are more options when running. You can view a list of those using `$ cargo run -- --help`.
* If you want to only run the Python code, you can either run `$ cargo run -- --py`, or `$ python ./src/python/main.py`. The second option is recommended for better output.
* Run `$ cargo run`. There are more options when running. You can view a list of those using `$ cargo run -- --help` or `$ cargo run -- -h`.
* If you want to only run the Python code, you can either run `$ cargo run -- --py`, or `$ python ./src/python/main.py`. The second option is recommended for better error output.
+2 -4
View File
@@ -7,7 +7,7 @@
<!-- - [x] ~~Multithread so it can run both Discord and Reddit bot!!!~~ -->
<!-- - [x] Security that only allows bk mods to run these commands. -->
<!-- - [x] Some kind of voting system. -->
- [ ] `/bk_week_top [category] [amount]` to get the top N posts in a category (e.g upvotes)
<!-- - [x] ~~`/bk_week_top [category] [amount]` to get the top N posts in a category (e.g upvotes)~~ -->
<!-- - [x] ~~`/bk_cfg_sr [subreddit]` to change the target subreddit(s)~~ -->
- [ ] Allow updating the data autonomously and via manual commands.
<!-- - [ ] 10-minute schedule for updating Discord channel (IMPOSSIBLE / REALLY FUCKING HARD) -->
@@ -19,9 +19,7 @@
<!-- - [x] ~~Manually approve posts via `/bk_week_approve [url]`~~ -->
<!-- - [x] ~~Manually un-approve posts via `/bk_week_disapprove [url]`~~ -->
<!-- - [x] ~~Automatically add scraped posts to JSON~~ -->
- [ ] Automatically remove posts older than 7 days from JSON
- [x] ~~Function~~
- [ ] Automate
<!-- - [x] ~~Remove posts (from data) that are older than 7 days~~ --> // need python function to remove data about them
- [ ] Automatically approve posts that don't get caught by reverse image search (ris)
<!-- - [x] Log all posts in a Discord thread -->
<!-- - [x] ~~`/bk_week_bind` to bind a channel for bk_week logs~~ -->
+10 -10
View File
@@ -1,29 +1,29 @@
# Discord Commands
`[this means it's a required argument]`
`<this means it's an optional argument>`
## `/bk_week_get [url]`
Shows info about a single post.
## `/bk_week_add [url] <approve>`
Adds a URL to the list of posts. This is done automatically by the bot for certain posts.
- **`<approve>`**: (OPTIONAL) `true` or `false`, determines whether to approve the post when added.
- **`<approve>`**: `true` or `false`, determines whether to automatically approve the post when added.
## `/bk_week_remove [url]`
Removes an existing URL from the list of posts.
## `/bk_week_approve [url] <disapprove>`
Flags the post as **human_approved**, confirming that the artwork is original.
- **`<disapprove>`**: (OPTIONAL) `true` or `false`, set to `true` if you want to undo an approval.
- **`<disapprove>`**: set to `true` if you want to undo an approval.
## `/bk_week_vote [url] <remove_vote>`
Adds a vote to a post. The intended use for votes is for a "moderator picks" and a "community picks" section of the weekly art.
You can vote on as many posts as you want, but only once per post.
## `/bk_week_update`
## `/bk_week_update <only_add> <max_age>`
Updates all data in the bound Discord channel. It's recommended to only run this if absolutely needed to.
- **`<only_add>`**: If it's `true`, the bot will only add new posts, not remove or update them.
- **`<max_age>`**: The bot will remove any post older than `max_age` days. (0 means infinite.)
## `/bk_week_top [category] <amount>`
Shows you the top posts within a category, such as upvotes. (max 10 posts, default is 3)
## `/bk_admin_bind`
Binds the current channel as the channel where the bk_week data is sent and updated in. No binding means the only way to view the data is using `/bk_week_get`.
## `/bk_week_help [option]`
Shows a help text.
- **`[option]`**: `Discord` or `Reddit`, determines what help text to send.
### **Examples**
```
/bk_week_add https://reddit.com/post_url false
/bk_week_add https://reddit.com/post_url
/bk_week_approve https://reddit.com/post_url
```
# Things to note
* This bot uses shortURLs when storing posts. If you want to access posts, you should use their shortURL. You can get a Reddit shortURL by running `/re_shorturl [url]`.
+133 -29
View File
@@ -3,9 +3,12 @@ use crate::{cmds, rs_println, websocket, Context, Data, Error, BK_WEEK};
use crate::messages::*;
use crate::data::{self, dc_bind_bk};
use std::collections::HashMap;
use std::fs;
use std::time::{SystemTime, UNIX_EPOCH};
use poise::serenity_prelude::{ChannelId, EditMessage, GetMessages, Http, Message, MessageId, UserId};
use poise::ReplyHandle;
use serde_json::{json, Map, Value};
@@ -14,6 +17,13 @@ enum HelpOptions {
Discord,
Reddit
}
#[derive(poise::ChoiceParameter, PartialEq)]
enum TopCategory {
Upvotes,
ModVotes,
Oldest,
Newest
}
fn is_bk_mod(mod_list: Value, uid: u64) -> bool {
@@ -342,20 +352,21 @@ async fn send_server_not_in_data_msg(ctx: Context<'_>) {
/// Updates all logs
pub async fn bk_week_update(
ctx: Context<'_>,
#[description = "Only adds new posts, leaves everything else unchanged."] only_add: Option<bool>
#[description = "Only adds new posts, leaves everything else unchanged."] only_add: Option<bool>,
#[description = "The max age of a post (in days). Any post older than this will be removed. (0 is infinite.)"] max_age: Option<u16>
) -> Result<(), Error>
{
let http = ctx.http();
let executed = format!("(Executed `/bk_week_update`, author: `{}`)", ctx.author().name);
let mut p_text = executed.clone();
let mut p_text = "`/bk_week_update`:".to_string();
send_msg(ctx, MANDATORY_MSG.to_string(), true, true).await;
let progress = send_msg(ctx, p_text.clone(), true, true).await.unwrap();
p_text = update_progress(ctx, progress.clone(), p_text, "\nFetching new posts & updating data file...".to_string()).await;
let progress = http_send_msg(http, ctx.channel_id(), p_text.clone()).await.unwrap();
p_text = update_progress(ctx.http(), progress.clone(), p_text, "\nFetching new posts & updating data file...".to_string()).await;
let max_age_u = max_age.unwrap_or_else(|| 8);
let max_age_secs = max_age_u as u64 * (60 * 60 * 24);
send_cmd_json("add_new_posts", None).await;
send_cmd_json("add_new_posts", Some(json!([max_age_secs]))).await;
data::update_re_data(ctx.data()).await;
let r_data = get_reddit_data(ctx.data()).await.unwrap();
@@ -369,51 +380,56 @@ pub async fn bk_week_update(
let c_id = c_id_u.unwrap();
// Reading messages
p_text = update_progress(http, progress.clone(), p_text.clone(), format!("\nReading messages in <#{}>...", c_id)).await;
p_text = update_progress(ctx, progress.clone(), p_text.clone(), format!("\nReading messages in <#{}>...", c_id)).await;
let msgs = read_msgs(http, ctx.framework().bot_id, c_id).await;
// Parsing messages to JSON
p_text = update_progress(http, progress.clone(), p_text.clone(), "\nParsing messages to JSON...".to_string()).await;
let msgs_json = msgs_to_json(msgs, &r_data).await;
p_text = update_progress(ctx, progress.clone(), p_text.clone(), "\nParsing messages to JSON...".to_string()).await;
let msgs_json = msgs_to_json(msgs, &r_data, max_age_secs).await;
// Adding new posts
p_text = update_progress(http, progress.clone(), p_text.clone(), "\nAdding new posts...".to_string()).await;
p_text = update_progress(ctx, progress.clone(), p_text.clone(), "\nAdding new posts...".to_string()).await;
let weekly_art = r_data[BK_WEEK].as_object().unwrap();
add_posts(http, c_id, weekly_art, &msgs_json).await;
// Stop if only_add
if only_add.unwrap_or_else(|| false) {
send_msg(ctx, "`/bk_week_update`\n## Done!".to_string(), true, true).await;
update_progress(http, progress.clone(), String::new(), executed).await;
update_progress(ctx, progress.clone(), p_text, "\n## Done!".to_string()).await;
return Ok(());
}
// Editing updated posts
p_text = update_progress(http, progress.clone(), p_text.clone(), "\nEditing updated posts...".to_string()).await;
p_text = update_progress(ctx, progress.clone(), p_text.clone(), "\nEditing updated posts...".to_string()).await;
edit_posts(http, c_id, weekly_art, &msgs_json).await;
// Removing removed posts
p_text = update_progress(http, progress.clone(), p_text.clone(), "\nRemoving removed posts...".to_string()).await;
p_text = update_progress(ctx, progress.clone(), p_text.clone(), "\nRemoving removed posts...".to_string()).await;
remove_posts(http, c_id, weekly_art, &msgs_json).await;
// Removing old posts
if max_age_u > 0 {
p_text = update_progress(ctx, progress.clone(), p_text.clone(), format!("\nRemoving old posts (threshold: {}d)...", max_age_u)).await;
remove_old(http, c_id, &msgs_json).await;
send_cmd_json("remove_old_posts", Some(json!([max_age_secs]))).await;
}
// Removing duplicate posts
update_progress(http, progress.clone(), p_text.clone(), "\nRemoving duplicate posts...".to_string()).await;
p_text = update_progress(ctx, progress.clone(), p_text.clone(), "\nRemoving duplicate posts...".to_string()).await;
remove_dupes(http, c_id, &msgs_json).await;
// Done
update_progress(ctx, progress.clone(), p_text, "\n## Done!".to_string()).await;
send_msg(ctx, "`/bk_week_update`\n## Done!".to_string(), true, true).await;
update_progress(http, progress.clone(), String::new(), executed).await;
return Ok(());
}
async fn update_progress(http: &Http, p: Message, t: String, added_t: String) -> String {
async fn update_progress(ctx: Context<'_>, p: ReplyHandle<'_>, t: String, added_t: String) -> String {
let p_text = format!("{} {}", t, added_t);
let new_msg = EditMessage::new().content(&p_text);
http_edit_msg(http, p, new_msg).await;
edit_reply(ctx, p, p_text.clone()).await;
return p_text;
}
@@ -437,7 +453,7 @@ async fn get_c_id(ctx: Context<'_>) -> Option<ChannelId> {
}
pub async fn read_msgs(http: &Http, bot_id: UserId, c_id: ChannelId) -> Vec<Message> {
async fn read_msgs(http: &Http, bot_id: UserId, c_id: ChannelId) -> Vec<Message> {
let b = GetMessages::new().limit(100);
let mut msgs = c_id.messages(http, b).await.unwrap();
msgs = msgs.into_iter().filter(|item| item.author.id == bot_id).collect();
@@ -466,8 +482,12 @@ pub async fn read_msgs(http: &Http, bot_id: UserId, c_id: ChannelId) -> Vec<Mess
}
pub async fn msgs_to_json<'a>(msgs: Vec<Message>, reddit_data: &'a Value) -> Value {
let mut msgs_json: Value = json!({"no_change": {}, "updated": {}, "removed": {}, "duplicates": {}});
async fn msgs_to_json<'a>(msgs: Vec<Message>, reddit_data: &'a Value, max_age: u64) -> Value {
let mut msgs_json: Value = json!({"no_change": {}, "updated": {}, "removed": {}, "duplicates": {}, "old": {}});
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("Time went backwards")
.as_secs() as u64;
for msg in msgs {
if msg.embeds.len() == 0 { continue; }
@@ -475,7 +495,8 @@ pub async fn msgs_to_json<'a>(msgs: Vec<Message>, reddit_data: &'a Value) -> Val
let url = msg.embeds[0].url.clone().unwrap();
if ["no_change", "updated", "removed"]
// duplicates
if ["no_change", "updated", "removed", "old"]
.iter()
.any(|key| msgs_json[key].as_object().unwrap().contains_key(&url))
{
@@ -500,20 +521,34 @@ pub async fn msgs_to_json<'a>(msgs: Vec<Message>, reddit_data: &'a Value) -> Val
let mut u_json: Value = msg_json.unwrap();
let re_url = &reddit_data[BK_WEEK][&url];
let post_date = re_url["post_data"]["date_unix"].as_u64().unwrap_or_else(|| 0);
// old
if now - post_date > max_age {
if let Some(obj) = msgs_json["old"].as_object_mut() {
obj.insert(url.clone(), json!(msg.id.get()));
continue;
}
}
// removed
if re_url.get("removed").is_some() {
if u_json.get("removed").is_some() {
// no change
if let Some(obj) = msgs_json["no_change"].as_object_mut() {
obj.insert(url.clone(), json!(msg.id.get()));
continue;
}
}
// removed
if let Some(obj) = msgs_json["removed"].as_object_mut() {
obj.insert(url.clone(), json!(msg.id.get()));
continue;
}
}
// updated
if u_json["added"] != re_url["added"]
|| u_json["approved"] != re_url["approved"]
|| u_json["post_data"]["upvotes"] != re_url["post_data"]["upvotes"]
@@ -527,6 +562,7 @@ pub async fn msgs_to_json<'a>(msgs: Vec<Message>, reddit_data: &'a Value) -> Val
}
}
// no change
if let Some(obj) = msgs_json["no_change"].as_object_mut() {
obj.insert(url.clone(), json!(msg.id.get()));
}
@@ -536,9 +572,9 @@ pub async fn msgs_to_json<'a>(msgs: Vec<Message>, reddit_data: &'a Value) -> Val
}
pub async fn add_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value>, msgs_json: &Value) {
async fn add_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value>, msgs_json: &Value) {
for url in r_data.keys() {
if ["no_change", "updated", "removed"]
if ["no_change", "updated", "removed", "old"]
.iter()
.any(|key| msgs_json[key].as_object().unwrap().contains_key(url))
{ continue; }
@@ -554,7 +590,7 @@ pub async fn add_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value>
}
pub async fn edit_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value>, msgs_json: &Value) {
async fn edit_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value>, msgs_json: &Value) {
for (url, msg_id) in msgs_json["updated"].as_object().unwrap() {
let mut msg = http.get_message(c_id, MessageId::new(msg_id.as_u64().unwrap())).await.unwrap();
let r = EditMessage::new()
@@ -565,7 +601,7 @@ pub async fn edit_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value
}
pub async fn remove_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value>, msgs_json: &Value) {
async fn remove_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Value>, msgs_json: &Value) {
for (url, msg_id) in msgs_json["removed"].as_object().unwrap() {
let mut msg = http.get_message(c_id, MessageId::new(msg_id.as_u64().unwrap())).await.unwrap();
let r = EditMessage::new()
@@ -576,7 +612,15 @@ pub async fn remove_posts(http: &Http, c_id: ChannelId, r_data: &Map<String, Val
}
pub async fn remove_dupes(http: &Http, c_id: ChannelId, msgs_json: &Value) {
async fn remove_old(http: &Http, c_id: ChannelId, msgs_json: &Value) {
for (_url, msg_id) in msgs_json["old"].as_object().unwrap() {
let msg = http.get_message(c_id, MessageId::new(msg_id.as_u64().unwrap())).await.unwrap();
let _ = msg.delete(http).await;
}
}
async fn remove_dupes(http: &Http, c_id: ChannelId, msgs_json: &Value) {
for (_url, msgs) in msgs_json["duplicates"].as_object().unwrap() {
for msg_id in msgs.as_array().unwrap() {
let msg = http.get_message(c_id, MessageId::new(msg_id.as_u64().unwrap())).await.unwrap();
@@ -646,6 +690,66 @@ pub async fn bk_week_vote(
#[poise::command(slash_command, prefix_command)]
/// Gets the top N (up to 10) posts within a certain category, such as upvotes. (Sorted descending.)
pub async fn bk_week_top(
ctx: Context<'_>,
#[description = "The sorting criteria, such as upvotes."] category: TopCategory,
#[description = "The amount of posts to show (max 10)."] amount: Option<u8>
) -> Result<(), Error>
{
let mut all: HashMap<&str, i32> = HashMap::new();
let posts = &get_reddit_data(ctx.data()).await.unwrap()[BK_WEEK];
let posts_u = posts.as_object().unwrap();
for (url, dat) in posts_u {
if dat.get("removed").is_some() { continue; }
let val: i32 = match category {
TopCategory::Upvotes => dat["post_data"]["upvotes"].as_i64().unwrap() as i32,
TopCategory::ModVotes => dat["votes"]["mod_voters"].as_array().unwrap().len() as i32,
TopCategory::Oldest
| TopCategory::Newest => dat["post_data"]["date_unix"].as_i64().unwrap() as i32,
};
all.insert(url, val);
}
let amount_u = amount.unwrap_or_else(|| 3);
let amount_clamped =
if amount_u > 10 { 10 }
else if amount_u < 1 { 1 }
else { amount_u };
let top =
if category != TopCategory::Oldest
{ largest_n (&all, amount_clamped as usize) }
else { smallest_n(&all, amount_clamped as usize) };
for post in top {
let url = post.0;
let _ = send_embed_for_post(ctx, posts_u[url].clone(), &url).await;
}
return Ok(());
}
fn largest_n<'a>(map: &'a HashMap<&'a str, i32>, n: usize) -> Vec<(&'a str, i32)> {
let mut vec: Vec<_> = map.iter().collect();
vec.sort_unstable_by(|a, b| b.1.cmp(a.1));
vec.into_iter().take(n).map(|(&k, &v)| (k, v)).collect()
}
fn smallest_n<'a>(map: &'a HashMap<&'a str, i32>, n: usize) -> Vec<(&'a str, i32)> {
let mut vec: Vec<_> = map.iter().collect();
vec.sort_unstable_by(|a, b| a.1.cmp(b.1));
vec.into_iter().take(n).map(|(&k, &v)| (k, v)).collect()
}
#[poise::command(slash_command, prefix_command, default_member_permissions = "ADMINISTRATOR")]
/// Changes the subreddit(s) the bot patrols in.
+1
View File
@@ -197,6 +197,7 @@ async fn gen_bot(data: Data, args: Args) -> Client {
bk_week_cmds::bk_week_approve(),
bk_week_cmds::bk_week_update(),
bk_week_cmds::bk_week_vote(),
bk_week_cmds::bk_week_top(),
// bk_admin
bk_week_cmds::bk_admin_bind(),
// bk_cfg
+3
View File
@@ -77,6 +77,7 @@ pub async fn send_msg(
}
#[allow(dead_code)]
pub async fn http_send_msg(
http: &Http,
c_id: ChannelId,
@@ -167,6 +168,7 @@ pub async fn edit_reply(
}
#[allow(dead_code)]
pub async fn http_edit_msg(
http: &Http,
mut msg: Message,
@@ -265,6 +267,7 @@ pub fn embed_post_removed(post_data: &Value, url: &str, empheral: bool) -> Embed
),
col: Some(REMOVED_DC_COL),
url: Some(url.to_string()),
ts: Some(Timestamp::from_unix_timestamp(post_data["post_data"]["date_unix"].as_i64().unwrap()).unwrap()),
empheral,
..Default::default()
};
+25 -2
View File
@@ -1,6 +1,7 @@
import os
import json
from typing import Final
import time
import bot as botPy
from macros import *
@@ -136,13 +137,35 @@ def set_approve_post(bot: botPy.Bot, approved: bool, url: str) -> bool:
def remove_post(bot: botPy.Bot, url: str, removed_by: str = "UNKNOWN", reason: str = "None") -> bool:
if url in bot.data[BK_WEEKLY]:
bot.data[BK_WEEKLY][url] = { "removed": True, "removed_by": removed_by, "remove_reason": reason }
weekly = bot.data[BK_WEEKLY]
if url in weekly:
weekly[url] = {
"removed": True,
"removed_by": removed_by,
"remove_reason": reason,
"post_data": { "date_unix": weekly[url]["post_data"]["date_unix"] }
}
return True
else:
return False
def remove_old_posts(bot: botPy.Bot, max_age: int) -> bool:
now = int(time.time())
weekly = bot.data[BK_WEEKLY]
remove: list[str] = []
for url, post in weekly.items():
if now - post["post_data"]["date_unix"] > max_age:
remove.append(url)
for key in remove:
weekly.pop(key)
return True
def set_vote_post(
bot: botPy.Bot,
url: str,
+15 -3
View File
@@ -2,13 +2,14 @@ import emoji
from asyncpraw import models
import asyncprawcore as prawcore
import asyncpraw.exceptions as exc
import time
import data
import bot as botPy
from macros import *
async def add_new_posts(bot: botPy.Bot) -> bool:
async def add_new_posts(bot: botPy.Bot, max_age: int) -> bool:
check_emoji = emoji.emojize(":check_mark_button:")
cross_emoji = emoji.emojize(":cross_mark:")
@@ -20,6 +21,7 @@ async def add_new_posts(bot: botPy.Bot) -> bool:
added_posts = 0
without_media = 0
not_added = 0
old_posts = 0
for post in posts:
media = has_media(post)
@@ -33,13 +35,22 @@ async def add_new_posts(bot: botPy.Bot) -> bool:
f"\n {media_urls}\n"
)
details = get_post_details(post)
now = int(time.time())
if now - details.date_unix > max_age and max_age > 0:
old_posts += 1
continue
if not media[0]:
without_media += 1
continue
post_added = False
post_added = data.add_post_to_data(
bot,
get_post_details(post)
details
)
if post_added: added_posts += 1
@@ -48,7 +59,8 @@ async def add_new_posts(bot: botPy.Bot) -> bool:
py_print(f"Successfully fetched {len(posts)} posts.\n" +
f" Out of which were {added_posts} added.\n" +
f" {without_media} had no media, " +
f"and {not_added} weren't added because they are removed or already existed")
f"{not_added} are removed or already existed, " +
f"and {old_posts} were older than the max age threshold.")
return True
+2 -1
View File
@@ -75,12 +75,13 @@ async def json_to_func(v: dict, bot: botPy.Bot) -> dict:
match v["value"]:
case "update_data_file": r = data .write_data (bot)
case "add_new_posts": r = await posts.add_new_posts (bot)
case "respond_mentions": r = await cmds .respond_to_mention(bot)
case "add_new_posts": r = await posts.add_new_posts (bot, *v["args"])
case "add_post_url": r = await posts.add_post_url (bot, *v["args"])
case "remove_post_url": r = data .remove_post (bot, *v["args"])
case "set_approve_post": r = data .set_approve_post (bot, *v["args"])
case "set_vote_post": r = data .set_vote_post (bot, *v["args"])
case "remove_old_posts": r = data .remove_old_posts (bot, *v["args"])
case "change_sr": r = await bot .change_sr (*v["args"])
case "stop_praw": r = await bot .stop ()
case _: value_supported = False