Kaveen Kumarasinghe 2 years ago
commit 602742eb59

@ -484,10 +484,17 @@ class GPT3ComCon(discord.Cog, name="GPT3ComCon"):
async def on_message_edit(self, before, after): async def on_message_edit(self, before, after):
# Moderation # Moderation
if after.guild.id in self.moderation_queues and self.moderation_queues[after.guild.id] is not None: if (
after.guild.id in self.moderation_queues
and self.moderation_queues[after.guild.id] is not None
):
# Create a timestamp that is 0.5 seconds from now # Create a timestamp that is 0.5 seconds from now
timestamp = (datetime.datetime.now() + datetime.timedelta(seconds=0.5)).timestamp() timestamp = (
await self.moderation_queues[after.guild.id].put(Moderation(after, timestamp)) datetime.datetime.now() + datetime.timedelta(seconds=0.5)
).timestamp()
await self.moderation_queues[after.guild.id].put(
Moderation(after, timestamp)
)
if after.author.id in self.redo_users: if after.author.id in self.redo_users:
if after.id == original_message[after.author.id]: if after.id == original_message[after.author.id]:
@ -514,7 +521,10 @@ class GPT3ComCon(discord.Cog, name="GPT3ComCon"):
print("Doing the encapsulated send") print("Doing the encapsulated send")
await self.encapsulated_send( await self.encapsulated_send(
user_id=after.author.id, prompt=edited_content, ctx=ctx, response_message=response_message user_id=after.author.id,
prompt=edited_content,
ctx=ctx,
response_message=response_message,
) )
self.redo_users[after.author.id].prompt = after.content self.redo_users[after.author.id].prompt = after.content
@ -529,10 +539,17 @@ class GPT3ComCon(discord.Cog, name="GPT3ComCon"):
content = message.content.strip() content = message.content.strip()
# Moderations service # Moderations service
if message.guild.id in self.moderation_queues and self.moderation_queues[message.guild.id] is not None: if (
message.guild.id in self.moderation_queues
and self.moderation_queues[message.guild.id] is not None
):
# Create a timestamp that is 0.5 seconds from now # Create a timestamp that is 0.5 seconds from now
timestamp = (datetime.datetime.now() + datetime.timedelta(seconds=0.5)).timestamp() timestamp = (
await self.moderation_queues[message.guild.id].put(Moderation(message, timestamp)) datetime.datetime.now() + datetime.timedelta(seconds=0.5)
).timestamp()
await self.moderation_queues[message.guild.id].put(
Moderation(message, timestamp)
)
conversing = self.check_conversing( conversing = self.check_conversing(
message.author.id, message.channel.id, content message.author.id, message.channel.id, content
@ -983,8 +1000,8 @@ class GPT3ComCon(discord.Cog, name="GPT3ComCon"):
async def moderations_test(self, ctx: discord.ApplicationContext, prompt: str): async def moderations_test(self, ctx: discord.ApplicationContext, prompt: str):
await ctx.defer() await ctx.defer()
response = await self.model.send_moderations_request(prompt) response = await self.model.send_moderations_request(prompt)
await ctx.respond(response['results'][0]['category_scores']) await ctx.respond(response["results"][0]["category_scores"])
await ctx.send_followup(response['results'][0]['flagged']) await ctx.send_followup(response["results"][0]["flagged"])
@add_to_group("system") @add_to_group("system")
@discord.slash_command( @discord.slash_command(
@ -992,10 +1009,20 @@ class GPT3ComCon(discord.Cog, name="GPT3ComCon"):
description="The AI moderations service", description="The AI moderations service",
guild_ids=ALLOWED_GUILDS, guild_ids=ALLOWED_GUILDS,
) )
@discord.option(name="status", description="Enable or disable the moderations service for the current guild (on/off)", required = True) @discord.option(
@discord.option(name="alert_channel_id", description="The channel ID to send moderation alerts to", required=False) name="status",
description="Enable or disable the moderations service for the current guild (on/off)",
required=True,
)
@discord.option(
name="alert_channel_id",
description="The channel ID to send moderation alerts to",
required=False,
)
@discord.guild_only() @discord.guild_only()
async def moderations(self, ctx: discord.ApplicationContext, status: str, alert_channel_id: str): async def moderations(
self, ctx: discord.ApplicationContext, status: str, alert_channel_id: str
):
await ctx.defer() await ctx.defer()
status = status.lower().strip() status = status.lower().strip()
@ -1007,11 +1034,19 @@ class GPT3ComCon(discord.Cog, name="GPT3ComCon"):
# Create the moderations service. # Create the moderations service.
self.moderation_queues[ctx.guild_id] = asyncio.Queue() self.moderation_queues[ctx.guild_id] = asyncio.Queue()
if self.moderation_alerts_channel or alert_channel_id: if self.moderation_alerts_channel or alert_channel_id:
moderations_channel = await self.bot.fetch_channel(self.moderation_alerts_channel if not alert_channel_id else alert_channel_id) moderations_channel = await self.bot.fetch_channel(
self.moderation_alerts_channel
if not alert_channel_id
else alert_channel_id
)
else: else:
moderations_channel = self.moderation_alerts_channel # None moderations_channel = self.moderation_alerts_channel # None
self.moderation_tasks[ctx.guild_id] = asyncio.ensure_future(Moderation.process_moderation_queue(self.moderation_queues[ctx.guild_id], 1, 1, moderations_channel)) self.moderation_tasks[ctx.guild_id] = asyncio.ensure_future(
Moderation.process_moderation_queue(
self.moderation_queues[ctx.guild_id], 1, 1, moderations_channel
)
)
await ctx.respond("Moderations service enabled") await ctx.respond("Moderations service enabled")
elif status == "off": elif status == "off":

@ -12,8 +12,8 @@ from models.usage_service_model import UsageService
usage_service = UsageService(Path(os.environ.get("DATA_DIR", os.getcwd()))) usage_service = UsageService(Path(os.environ.get("DATA_DIR", os.getcwd())))
model = Model(usage_service) model = Model(usage_service)
class Moderation:
class Moderation:
def __init__(self, message, timestamp): def __init__(self, message, timestamp):
self.message = message self.message = message
self.timestamp = timestamp self.timestamp = timestamp
@ -27,10 +27,10 @@ class Moderation:
colour=discord.Colour.red(), colour=discord.Colour.red(),
) )
# Set the embed thumbnail # Set the embed thumbnail
embed.set_thumbnail( embed.set_thumbnail(url="https://i.imgur.com/2oL8JSp.png")
url="https://i.imgur.com/2oL8JSp.png" embed.set_footer(
text="If you think this was a mistake, please contact the server admins."
) )
embed.set_footer(text="If you think this was a mistake, please contact the server admins.")
return embed return embed
@staticmethod @staticmethod
@ -38,7 +38,9 @@ class Moderation:
# Create a discord embed to send to the user when their message gets moderated # Create a discord embed to send to the user when their message gets moderated
embed = discord.Embed( embed = discord.Embed(
title="A message was moderated in the "+moderated_message.message.guild.name+" server", title="A message was moderated in the "
+ moderated_message.message.guild.name
+ " server",
description=f"Message from {moderated_message.message.author.mention} was moderated: {moderated_message.message.content}", description=f"Message from {moderated_message.message.author.mention} was moderated: {moderated_message.message.content}",
colour=discord.Colour.yellow(), colour=discord.Colour.yellow(),
) )
@ -48,7 +50,6 @@ class Moderation:
embed.add_field(name="Moderated message link", value=link, inline=False) embed.add_field(name="Moderated message link", value=link, inline=False)
return embed return embed
@staticmethod @staticmethod
def determine_moderation_result(response): def determine_moderation_result(response):
HATE_THRESHOLD = 0.005 HATE_THRESHOLD = 0.005
@ -59,12 +60,28 @@ class Moderation:
VIOLENCE_THRESHOLD = 0.01 VIOLENCE_THRESHOLD = 0.01
VIOLENCE_GRAPHIC_THRESHOLD = 0.1 VIOLENCE_GRAPHIC_THRESHOLD = 0.1
thresholds = [HATE_THRESHOLD, HATE_VIOLENCE_THRESHOLD, SELF_HARM_THRESHOLD, SEXUAL_THRESHOLD, SEXUAL_MINORS_THRESHOLD, VIOLENCE_THRESHOLD, VIOLENCE_GRAPHIC_THRESHOLD] thresholds = [
threshold_iterator = ['hate','hate/threatening','self-harm','sexual','sexual/minors','violence','violence/graphic'] HATE_THRESHOLD,
HATE_VIOLENCE_THRESHOLD,
category_scores = response['results'][0]['category_scores'] SELF_HARM_THRESHOLD,
SEXUAL_THRESHOLD,
flagged = response['results'][0]['flagged'] SEXUAL_MINORS_THRESHOLD,
VIOLENCE_THRESHOLD,
VIOLENCE_GRAPHIC_THRESHOLD,
]
threshold_iterator = [
"hate",
"hate/threatening",
"self-harm",
"sexual",
"sexual/minors",
"violence",
"violence/graphic",
]
category_scores = response["results"][0]["category_scores"]
flagged = response["results"][0]["flagged"]
# Iterate the category scores using the threshold_iterator and compare the values to thresholds # Iterate the category scores using the threshold_iterator and compare the values to thresholds
for category, threshold in zip(threshold_iterator, thresholds): for category, threshold in zip(threshold_iterator, thresholds):
@ -90,18 +107,26 @@ class Moderation:
# Check if the current timestamp is greater than the deletion timestamp # Check if the current timestamp is greater than the deletion timestamp
if datetime.now().timestamp() > to_moderate.timestamp: if datetime.now().timestamp() > to_moderate.timestamp:
response = await model.send_moderations_request(to_moderate.message.content) response = await model.send_moderations_request(
to_moderate.message.content
)
moderation_result = Moderation.determine_moderation_result(response) moderation_result = Moderation.determine_moderation_result(response)
if moderation_result: if moderation_result:
# Take care of the flagged message # Take care of the flagged message
response_message = await to_moderate.message.reply(embed=Moderation.build_moderation_embed()) response_message = await to_moderate.message.reply(
embed=Moderation.build_moderation_embed()
)
# Do the same response as above but use an ephemeral message # Do the same response as above but use an ephemeral message
await to_moderate.message.delete() await to_moderate.message.delete()
# Send to the moderation alert channel # Send to the moderation alert channel
if moderations_alert_channel: if moderations_alert_channel:
await moderations_alert_channel.send(embed=Moderation.build_admin_moderated_message(to_moderate, response_message)) await moderations_alert_channel.send(
embed=Moderation.build_admin_moderated_message(
to_moderate, response_message
)
)
else: else:
await moderation_queue.put(to_moderate) await moderation_queue.put(to_moderate)

@ -320,7 +320,7 @@ class Model:
async def send_moderations_request(self, text): async def send_moderations_request(self, text):
# Use aiohttp to send the above request: # Use aiohttp to send the above request:
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
headers={ headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
"Authorization": f"Bearer {self.openai_key}", "Authorization": f"Bearer {self.openai_key}",
} }
@ -332,7 +332,6 @@ class Model:
) as response: ) as response:
return await response.json() return await response.json()
async def send_summary_request(self, prompt): async def send_summary_request(self, prompt):
""" """
Sends a summary request to the OpenAI API Sends a summary request to the OpenAI API

Loading…
Cancel
Save