From a1af23179da5c61e02fbb41a52a40f8a63df364c Mon Sep 17 00:00:00 2001 From: Rene Teigen Date: Fri, 3 Mar 2023 10:19:17 +0000 Subject: [PATCH] Fix docker persistence Fix ghcr.io having double base and full to the image name Add some comments to the compose file --- .../workflows/build-and-publish-docker.yml | 8 +++--- docker-compose.yml | 4 ++- models/index_model.py | 26 +++++++++---------- models/openai_model.py | 3 ++- models/search_model.py | 6 ++--- services/environment_service.py | 7 +++++ 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-and-publish-docker.yml b/.github/workflows/build-and-publish-docker.yml index 8e0d832..4110b4b 100644 --- a/.github/workflows/build-and-publish-docker.yml +++ b/.github/workflows/build-and-publish-docker.yml @@ -91,7 +91,7 @@ jobs: with: context: . push: true - tags: ${{ steps.meta.outputs.tags }}-base + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max @@ -187,7 +187,7 @@ jobs: build-args: FULL=true context: . push: true - tags: ${{ steps.meta.outputs.tags }}-full + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max @@ -238,7 +238,7 @@ jobs: with: context: . push: false - tags: ${{ steps.meta.outputs.tags }}-base + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max @@ -251,7 +251,7 @@ jobs: build-args: FULL=true context: . push: false - tags: ${{ steps.meta.outputs.tags }}-full + tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/docker-compose.yml b/docker-compose.yml index 2cd301d..364121f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,9 @@ services: volumes: #replace left side with the path to your .env if different, this uses the env in the folder the docker-compose.yml is in - ./.env:/opt/gpt3discord/etc/environment - # Replace the left side with a path on your hard drive where you want to store the data for both of these + # Replace the left side with a path on your hard drive where you want to store the data for both of these to keep persistence + # .env add DATA_DIR=/data - /containers/gpt3discord:/data + # .env add SHARE_DIR=/data/share - /containers/gpt3discord/share:/data/share restart: always diff --git a/models/index_model.py b/models/index_model.py index cbfd2ae..48d85f2 100644 --- a/models/index_model.py +++ b/models/index_model.py @@ -48,7 +48,7 @@ from llama_index.readers.web import DEFAULT_WEBSITE_EXTRACTOR from llama_index.composability import ComposableGraph from models.embed_statics_model import EmbedStatics -from services.environment_service import EnvService, app_root_path +from services.environment_service import EnvService SHORT_TO_LONG_CACHE = {} MAX_DEEP_COMPOSE_PRICE = EnvService.get_max_deep_compose_price() @@ -110,13 +110,13 @@ class IndexData: def has_indexes(self, user_id): try: - return len(os.listdir(f"{app_root_path()}/indexes/{user_id}")) > 0 + return len(os.listdir(EnvService.find_shared_file(f"indexes/{user_id}"))) > 0 except Exception: return False def has_search_indexes(self, user_id): try: - return len(os.listdir(f"{app_root_path()}/indexes/{user_id}_search")) > 0 + return len(os.listdir(EnvService.find_shared_file(f"indexes/{user_id}_search"))) > 0 except Exception: return False @@ -125,7 +125,7 @@ class IndexData: self.queryable_index = index # Create a folder called "indexes/{USER_ID}" if it doesn't exist already - Path(f"{app_root_path()}/indexes/{user_id}").mkdir(parents=True, exist_ok=True) + Path(f"{EnvService.save_path()}/indexes/{user_id}").mkdir(parents=True, exist_ok=True) # Save the index to file under the user id file = f"{file_name}_{date.today().month}_{date.today().day}" # If file is > 93 in length, cut it off to 93 @@ -133,7 +133,7 @@ class IndexData: file = file[:93] index.save_to_disk( - app_root_path() / "indexes" / f"{str(user_id)}" / f"{file}.json" + EnvService.save_path() / "indexes" / f"{str(user_id)}" / f"{file}.json" ) def reset_indexes(self, user_id): @@ -143,10 +143,10 @@ class IndexData: # Delete the user indexes try: # First, clear all the files inside it - for file in os.listdir(f"{app_root_path()}/indexes/{user_id}"): - os.remove(f"{app_root_path()}/indexes/{user_id}/{file}") - for file in os.listdir(f"{app_root_path()}/indexes/{user_id}_search"): - os.remove(f"{app_root_path()}/indexes/{user_id}_search/{file}") + for file in os.listdir(EnvService.find_shared_file(f"indexes/{user_id}")): + os.remove(EnvService.find_shared_file(f"indexes/{user_id}/{file}")) + for file in os.listdir(EnvService.find_shared_file(f"indexes/{user_id}_search")): + os.remove(EnvService.find_shared_file(f"indexes/{user_id}_search/{file}")) except Exception: traceback.print_exc() @@ -792,7 +792,7 @@ class Index_handler: ) # Save the composed index - tree_index.save_to_disk(app_root_path() / "indexes" / str(user_id) / name) + tree_index.save_to_disk(EnvService.save_path() / "indexes" / str(user_id) / name) self.index_storage[user_id].queryable_index = tree_index @@ -822,7 +822,7 @@ class Index_handler: name = f"composed_index_{date.today().month}_{date.today().day}.json" # Save the composed index - simple_index.save_to_disk(app_root_path() / "indexes" / str(user_id) / name) + simple_index.save_to_disk(EnvService.save_path() / "indexes" / str(user_id) / name) self.index_storage[user_id].queryable_index = simple_index try: @@ -863,11 +863,11 @@ class Index_handler: except Exception: traceback.print_exc() price = "Unknown" - Path(app_root_path() / "indexes" / str(ctx.guild.id)).mkdir( + Path(EnvService.save_path() / "indexes" / str(ctx.guild.id)).mkdir( parents=True, exist_ok=True ) index.save_to_disk( - app_root_path() + EnvService.save_path() / "indexes" / str(ctx.guild.id) / f"{ctx.guild.name.replace(' ', '-')}_{date.today().month}_{date.today().day}.json" diff --git a/models/openai_model.py b/models/openai_model.py index 8b9dd4b..335609b 100644 --- a/models/openai_model.py +++ b/models/openai_model.py @@ -14,13 +14,14 @@ import discord # An enum of two modes, TOP_P or TEMPERATURE import requests +from services.environment_service import EnvService from PIL import Image from discord import File from sqlitedict import SqliteDict try: print("Attempting to retrieve the settings DB") - SETTINGS_DB = SqliteDict("main_db.sqlite", tablename="settings", autocommit=True) + SETTINGS_DB = SqliteDict(f"{EnvService.save_path()}/main_db.sqlite", tablename="settings", autocommit=True) print("Retrieved the settings DB") except Exception as e: print("Failed to retrieve the settings DB. The bot is terminating.") diff --git a/models/search_model.py b/models/search_model.py index 47facb6..19ca002 100644 --- a/models/search_model.py +++ b/models/search_model.py @@ -30,7 +30,7 @@ from llama_index.prompts.prompt_type import PromptType from llama_index.readers.web import DEFAULT_WEBSITE_EXTRACTOR from langchain import OpenAI -from services.environment_service import EnvService, app_root_path +from services.environment_service import EnvService from services.usage_service import UsageService MAX_SEARCH_PRICE = EnvService.get_max_search_price() @@ -57,14 +57,14 @@ class Search: def add_search_index(self, index, user_id, query): # Create a folder called "indexes/{USER_ID}" if it doesn't exist already - Path(f"{app_root_path()}/indexes/{user_id}_search").mkdir( + Path(f"{EnvService.save_path()}/indexes/{user_id}_search").mkdir( parents=True, exist_ok=True ) # Save the index to file under the user id file = f"{query[:20]}_{date.today().month}_{date.today().day}" index.save_to_disk( - app_root_path() / "indexes" / f"{str(user_id)}_search" / f"{file}.json" + EnvService.save_path() / "indexes" / f"{str(user_id)}_search" / f"{file}.json" ) def build_search_started_embed(self): diff --git a/services/environment_service.py b/services/environment_service.py index 6a1ee35..6b6b25b 100644 --- a/services/environment_service.py +++ b/services/environment_service.py @@ -42,6 +42,13 @@ class EnvService: return app_relative return Path.cwd() + + @staticmethod + def save_path(): + share_dir = os.getenv("SHARE_DIR") + if share_dir is not None: + return Path(share_dir) + return app_root_path() @staticmethod def find_shared_file(file_name):