diff --git a/.github/workflows/build-and-publish-docker.yml b/.github/workflows/build-and-publish-docker.yml index 8e0d832..4110b4b 100644 --- a/.github/workflows/build-and-publish-docker.yml +++ b/.github/workflows/build-and-publish-docker.yml @@ -91,7 +91,7 @@ jobs: with: context: . push: true - tags: ${{ steps.meta.outputs.tags }}-base + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max @@ -187,7 +187,7 @@ jobs: build-args: FULL=true context: . push: true - tags: ${{ steps.meta.outputs.tags }}-full + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max @@ -238,7 +238,7 @@ jobs: with: context: . push: false - tags: ${{ steps.meta.outputs.tags }}-base + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max @@ -251,7 +251,7 @@ jobs: build-args: FULL=true context: . push: false - tags: ${{ steps.meta.outputs.tags }}-full + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/docker-compose.yml b/docker-compose.yml index 2cd301d..364121f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,9 @@ services: volumes: #replace left side with the path to your .env if different, this uses the env in the folder the docker-compose.yml is in - ./.env:/opt/gpt3discord/etc/environment - # Replace the left side with a path on your hard drive where you want to store the data for both of these + # Replace the left side with a path on your hard drive where you want to store the data for both of these to keep persistence + # .env add DATA_DIR=/data - /containers/gpt3discord:/data + # .env add SHARE_DIR=/data/share - /containers/gpt3discord/share:/data/share restart: always diff --git a/models/index_model.py b/models/index_model.py index cbfd2ae..2cb8170 100644 --- a/models/index_model.py +++ b/models/index_model.py @@ -48,7 +48,7 @@ from llama_index.readers.web import DEFAULT_WEBSITE_EXTRACTOR from llama_index.composability import ComposableGraph from models.embed_statics_model import EmbedStatics -from services.environment_service import EnvService, app_root_path +from services.environment_service import EnvService SHORT_TO_LONG_CACHE = {} MAX_DEEP_COMPOSE_PRICE = EnvService.get_max_deep_compose_price() @@ -110,13 +110,20 @@ class IndexData: def has_indexes(self, user_id): try: - return len(os.listdir(f"{app_root_path()}/indexes/{user_id}")) > 0 + return ( + len(os.listdir(EnvService.find_shared_file(f"indexes/{user_id}"))) > 0 + ) except Exception: return False def has_search_indexes(self, user_id): try: - return len(os.listdir(f"{app_root_path()}/indexes/{user_id}_search")) > 0 + return ( + len( + os.listdir(EnvService.find_shared_file(f"indexes/{user_id}_search")) + ) + > 0 + ) except Exception: return False @@ -125,7 +132,9 @@ class IndexData: self.queryable_index = index # Create a folder called "indexes/{USER_ID}" if it doesn't exist already - Path(f"{app_root_path()}/indexes/{user_id}").mkdir(parents=True, exist_ok=True) + Path(f"{EnvService.save_path()}/indexes/{user_id}").mkdir( + parents=True, exist_ok=True + ) # Save the index to file under the user id file = f"{file_name}_{date.today().month}_{date.today().day}" # If file is > 93 in length, cut it off to 93 @@ -133,7 +142,7 @@ class IndexData: file = file[:93] index.save_to_disk( - app_root_path() / "indexes" / f"{str(user_id)}" / f"{file}.json" + EnvService.save_path() / "indexes" / f"{str(user_id)}" / f"{file}.json" ) def reset_indexes(self, user_id): @@ -143,10 +152,14 @@ class IndexData: # Delete the user indexes try: # First, clear all the files inside it - for file in os.listdir(f"{app_root_path()}/indexes/{user_id}"): - os.remove(f"{app_root_path()}/indexes/{user_id}/{file}") - for file in os.listdir(f"{app_root_path()}/indexes/{user_id}_search"): - os.remove(f"{app_root_path()}/indexes/{user_id}_search/{file}") + for file in os.listdir(EnvService.find_shared_file(f"indexes/{user_id}")): + os.remove(EnvService.find_shared_file(f"indexes/{user_id}/{file}")) + for file in os.listdir( + EnvService.find_shared_file(f"indexes/{user_id}_search") + ): + os.remove( + EnvService.find_shared_file(f"indexes/{user_id}_search/{file}") + ) except Exception: traceback.print_exc() @@ -792,7 +805,9 @@ class Index_handler: ) # Save the composed index - tree_index.save_to_disk(app_root_path() / "indexes" / str(user_id) / name) + tree_index.save_to_disk( + EnvService.save_path() / "indexes" / str(user_id) / name + ) self.index_storage[user_id].queryable_index = tree_index @@ -822,7 +837,9 @@ class Index_handler: name = f"composed_index_{date.today().month}_{date.today().day}.json" # Save the composed index - simple_index.save_to_disk(app_root_path() / "indexes" / str(user_id) / name) + simple_index.save_to_disk( + EnvService.save_path() / "indexes" / str(user_id) / name + ) self.index_storage[user_id].queryable_index = simple_index try: @@ -863,11 +880,11 @@ class Index_handler: except Exception: traceback.print_exc() price = "Unknown" - Path(app_root_path() / "indexes" / str(ctx.guild.id)).mkdir( + Path(EnvService.save_path() / "indexes" / str(ctx.guild.id)).mkdir( parents=True, exist_ok=True ) index.save_to_disk( - app_root_path() + EnvService.save_path() / "indexes" / str(ctx.guild.id) / f"{ctx.guild.name.replace(' ', '-')}_{date.today().month}_{date.today().day}.json" diff --git a/models/openai_model.py b/models/openai_model.py index 8b9dd4b..93ecd26 100644 --- a/models/openai_model.py +++ b/models/openai_model.py @@ -14,13 +14,18 @@ import discord # An enum of two modes, TOP_P or TEMPERATURE import requests +from services.environment_service import EnvService from PIL import Image from discord import File from sqlitedict import SqliteDict try: print("Attempting to retrieve the settings DB") - SETTINGS_DB = SqliteDict("main_db.sqlite", tablename="settings", autocommit=True) + SETTINGS_DB = SqliteDict( + f"{EnvService.save_path()}/main_db.sqlite", + tablename="settings", + autocommit=True, + ) print("Retrieved the settings DB") except Exception as e: print("Failed to retrieve the settings DB. The bot is terminating.") diff --git a/models/search_model.py b/models/search_model.py index 47facb6..d10fb68 100644 --- a/models/search_model.py +++ b/models/search_model.py @@ -30,7 +30,7 @@ from llama_index.prompts.prompt_type import PromptType from llama_index.readers.web import DEFAULT_WEBSITE_EXTRACTOR from langchain import OpenAI -from services.environment_service import EnvService, app_root_path +from services.environment_service import EnvService from services.usage_service import UsageService MAX_SEARCH_PRICE = EnvService.get_max_search_price() @@ -57,14 +57,17 @@ class Search: def add_search_index(self, index, user_id, query): # Create a folder called "indexes/{USER_ID}" if it doesn't exist already - Path(f"{app_root_path()}/indexes/{user_id}_search").mkdir( + Path(f"{EnvService.save_path()}/indexes/{user_id}_search").mkdir( parents=True, exist_ok=True ) # Save the index to file under the user id file = f"{query[:20]}_{date.today().month}_{date.today().day}" index.save_to_disk( - app_root_path() / "indexes" / f"{str(user_id)}_search" / f"{file}.json" + EnvService.save_path() + / "indexes" + / f"{str(user_id)}_search" + / f"{file}.json" ) def build_search_started_embed(self): diff --git a/services/environment_service.py b/services/environment_service.py index 6a1ee35..fdf7dba 100644 --- a/services/environment_service.py +++ b/services/environment_service.py @@ -43,6 +43,13 @@ class EnvService: return Path.cwd() + @staticmethod + def save_path(): + share_dir = os.getenv("SHARE_DIR") + if share_dir is not None: + return Path(share_dir) + return app_root_path() + @staticmethod def find_shared_file(file_name): share_file_paths = []