Kaveen Kumarasinghe
2 years ago
parent
c338d2aea3
commit
369c836a72
@ -0,0 +1,113 @@
|
||||
import asyncio
|
||||
import os
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import discord
|
||||
|
||||
from models.openai_model import Model
|
||||
from models.usage_service_model import UsageService
|
||||
|
||||
usage_service = UsageService(Path(os.environ.get("DATA_DIR", os.getcwd())))
|
||||
model = Model(usage_service)
|
||||
|
||||
class Moderation:
|
||||
|
||||
def __init__(self, message, timestamp):
|
||||
self.message = message
|
||||
self.timestamp = timestamp
|
||||
|
||||
@staticmethod
|
||||
def build_moderation_embed():
|
||||
# Create a discord embed to send to the user when their message gets moderated
|
||||
embed = discord.Embed(
|
||||
title="Your message was moderated",
|
||||
description="Our automatic moderation systems detected that your message was inappropriate and has been deleted. Please review the rules.",
|
||||
colour=discord.Colour.red(),
|
||||
)
|
||||
# Set the embed thumbnail
|
||||
embed.set_thumbnail(
|
||||
url="https://i.imgur.com/2oL8JSp.png"
|
||||
)
|
||||
embed.set_footer(text="If you think this was a mistake, please contact the server admins.")
|
||||
return embed
|
||||
|
||||
@staticmethod
|
||||
def build_admin_moderated_message(moderated_message, response_message):
|
||||
|
||||
# Create a discord embed to send to the user when their message gets moderated
|
||||
embed = discord.Embed(
|
||||
title="A message was moderated in the "+moderated_message.message.guild.name+" server",
|
||||
description=f"Message from {moderated_message.message.author.mention} was moderated: {moderated_message.message.content}",
|
||||
colour=discord.Colour.yellow(),
|
||||
)
|
||||
# Get the link to the moderated message
|
||||
link = f"https://discord.com/channels/{response_message.guild.id}/{response_message.channel.id}/{response_message.id}"
|
||||
# set the link of the embed
|
||||
embed.add_field(name="Moderated message link", value=link, inline=False)
|
||||
return embed
|
||||
|
||||
|
||||
@staticmethod
|
||||
def determine_moderation_result(response):
|
||||
HATE_THRESHOLD = 0.005
|
||||
HATE_VIOLENCE_THRESHOLD = 0.05
|
||||
SELF_HARM_THRESHOLD = 0.05
|
||||
SEXUAL_THRESHOLD = 0.75
|
||||
SEXUAL_MINORS_THRESHOLD = 0.1
|
||||
VIOLENCE_THRESHOLD = 0.01
|
||||
VIOLENCE_GRAPHIC_THRESHOLD = 0.1
|
||||
|
||||
thresholds = [HATE_THRESHOLD, HATE_VIOLENCE_THRESHOLD, SELF_HARM_THRESHOLD, SEXUAL_THRESHOLD, SEXUAL_MINORS_THRESHOLD, VIOLENCE_THRESHOLD, VIOLENCE_GRAPHIC_THRESHOLD]
|
||||
threshold_iterator = ['hate','hate/threatening','self-harm','sexual','sexual/minors','violence','violence/graphic']
|
||||
|
||||
category_scores = response['results'][0]['category_scores']
|
||||
|
||||
flagged = response['results'][0]['flagged']
|
||||
|
||||
# Iterate the category scores using the threshold_iterator and compare the values to thresholds
|
||||
for category, threshold in zip(threshold_iterator, thresholds):
|
||||
if category_scores[category] > threshold:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# This function will be called by the bot to process the message queue
|
||||
@staticmethod
|
||||
async def process_moderation_queue(
|
||||
moderation_queue, PROCESS_WAIT_TIME, EMPTY_WAIT_TIME, moderations_alert_channel
|
||||
):
|
||||
while True:
|
||||
try:
|
||||
# If the queue is empty, sleep for a short time before checking again
|
||||
if moderation_queue.empty():
|
||||
await asyncio.sleep(EMPTY_WAIT_TIME)
|
||||
continue
|
||||
|
||||
# Get the next message from the queue
|
||||
to_moderate = await moderation_queue.get()
|
||||
|
||||
# Check if the current timestamp is greater than the deletion timestamp
|
||||
if datetime.now().timestamp() > to_moderate.timestamp:
|
||||
response = await model.send_moderations_request(to_moderate.message.content)
|
||||
moderation_result = Moderation.determine_moderation_result(response)
|
||||
|
||||
if moderation_result:
|
||||
# Take care of the flagged message
|
||||
response_message = await to_moderate.message.reply(embed=Moderation.build_moderation_embed())
|
||||
# Do the same response as above but use an ephemeral message
|
||||
await to_moderate.message.delete()
|
||||
|
||||
# Send to the moderation alert channel
|
||||
if moderations_alert_channel:
|
||||
await moderations_alert_channel.send(embed=Moderation.build_admin_moderated_message(to_moderate, response_message))
|
||||
|
||||
else:
|
||||
await moderation_queue.put(to_moderate)
|
||||
# Sleep for a short time before processing the next message
|
||||
# This will prevent the bot from spamming messages too quickly
|
||||
await asyncio.sleep(PROCESS_WAIT_TIME)
|
||||
except:
|
||||
traceback.print_exc()
|
||||
pass
|
||||
Loading…
Reference in new issue