Merge pull request #204 from Hikari-Haru/main

Docker persistence fixes and other fixes

.github/workflows/build-and-publish-docker.yml

@@ -91,7 +91,7 @@ jobs:
         with:
           context: .
           push: true
-          tags: ${{ steps.meta.outputs.tags }}-base
+          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
@@ -187,7 +187,7 @@ jobs:
           build-args: FULL=true
           context: .
           push: true
-          tags: ${{ steps.meta.outputs.tags }}-full
+          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
@@ -238,7 +238,7 @@ jobs:
         with:
           context: .
           push: false
-          tags: ${{ steps.meta.outputs.tags }}-base
+          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
@@ -251,7 +251,7 @@ jobs:
           build-args: FULL=true
           context: .
           push: false
-          tags: ${{ steps.meta.outputs.tags }}-full
+          tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max

docker-compose.yml

@@ -6,7 +6,9 @@ services:
     volumes:
       #replace left side with the path to your .env if different, this uses the env in the folder the docker-compose.yml is in
       - ./.env:/opt/gpt3discord/etc/environment
-      # Replace the left side with a path on your hard drive where you want to store the data for both of these
+      # Replace the left side with a path on your hard drive where you want to store the data for both of these to keep persistence
+      # .env add DATA_DIR=/data
       - /containers/gpt3discord:/data
+      # .env add SHARE_DIR=/data/share
       - /containers/gpt3discord/share:/data/share
     restart: always

models/index_model.py

@@ -48,7 +48,7 @@ from llama_index.readers.web import DEFAULT_WEBSITE_EXTRACTOR
 from llama_index.composability import ComposableGraph
 
 from models.embed_statics_model import EmbedStatics
-from services.environment_service import EnvService, app_root_path
+from services.environment_service import EnvService
 
 SHORT_TO_LONG_CACHE = {}
 MAX_DEEP_COMPOSE_PRICE = EnvService.get_max_deep_compose_price()
@@ -110,13 +110,20 @@ class IndexData:
 
     def has_indexes(self, user_id):
         try:
-            return len(os.listdir(f"{app_root_path()}/indexes/{user_id}")) > 0
+            return (
+                len(os.listdir(EnvService.find_shared_file(f"indexes/{user_id}"))) > 0
+            )
         except Exception:
             return False
 
     def has_search_indexes(self, user_id):
         try:
-            return len(os.listdir(f"{app_root_path()}/indexes/{user_id}_search")) > 0
+            return (
+                len(
+                    os.listdir(EnvService.find_shared_file(f"indexes/{user_id}_search"))
+                )
+                > 0
+            )
         except Exception:
             return False
 
@@ -125,7 +132,9 @@ class IndexData:
         self.queryable_index = index
 
         # Create a folder called "indexes/{USER_ID}" if it doesn't exist already
-        Path(f"{app_root_path()}/indexes/{user_id}").mkdir(parents=True, exist_ok=True)
+        Path(f"{EnvService.save_path()}/indexes/{user_id}").mkdir(
+            parents=True, exist_ok=True
+        )
         # Save the index to file under the user id
         file = f"{file_name}_{date.today().month}_{date.today().day}"
         # If file is > 93 in length, cut it off to 93
@@ -133,7 +142,7 @@ class IndexData:
             file = file[:93]
 
         index.save_to_disk(
-            app_root_path() / "indexes" / f"{str(user_id)}" / f"{file}.json"
+            EnvService.save_path() / "indexes" / f"{str(user_id)}" / f"{file}.json"
         )
 
     def reset_indexes(self, user_id):
@@ -143,10 +152,14 @@ class IndexData:
         # Delete the user indexes
         try:
             # First, clear all the files inside it
-            for file in os.listdir(f"{app_root_path()}/indexes/{user_id}"):
+            for file in os.listdir(EnvService.find_shared_file(f"indexes/{user_id}")):
-                os.remove(f"{app_root_path()}/indexes/{user_id}/{file}")
+                os.remove(EnvService.find_shared_file(f"indexes/{user_id}/{file}"))
-            for file in os.listdir(f"{app_root_path()}/indexes/{user_id}_search"):
+            for file in os.listdir(
-                os.remove(f"{app_root_path()}/indexes/{user_id}_search/{file}")
+                EnvService.find_shared_file(f"indexes/{user_id}_search")
+            ):
+                os.remove(
+                    EnvService.find_shared_file(f"indexes/{user_id}_search/{file}")
+                )
         except Exception:
             traceback.print_exc()
 
@@ -792,7 +805,9 @@ class Index_handler:
                 )
 
             # Save the composed index
-            tree_index.save_to_disk(app_root_path() / "indexes" / str(user_id) / name)
+            tree_index.save_to_disk(
+                EnvService.save_path() / "indexes" / str(user_id) / name
+            )
 
             self.index_storage[user_id].queryable_index = tree_index
 
@@ -822,7 +837,9 @@ class Index_handler:
                 name = f"composed_index_{date.today().month}_{date.today().day}.json"
 
             # Save the composed index
-            simple_index.save_to_disk(app_root_path() / "indexes" / str(user_id) / name)
+            simple_index.save_to_disk(
+                EnvService.save_path() / "indexes" / str(user_id) / name
+            )
             self.index_storage[user_id].queryable_index = simple_index
 
             try:
@@ -863,11 +880,11 @@ class Index_handler:
             except Exception:
                 traceback.print_exc()
                 price = "Unknown"
-            Path(app_root_path() / "indexes" / str(ctx.guild.id)).mkdir(
+            Path(EnvService.save_path() / "indexes" / str(ctx.guild.id)).mkdir(
                 parents=True, exist_ok=True
             )
             index.save_to_disk(
-                app_root_path()
+                EnvService.save_path()
                 / "indexes"
                 / str(ctx.guild.id)
                 / f"{ctx.guild.name.replace(' ', '-')}_{date.today().month}_{date.today().day}.json"

models/openai_model.py

@@ -14,13 +14,18 @@ import discord
 
 # An enum of two modes, TOP_P or TEMPERATURE
 import requests
+from services.environment_service import EnvService
 from PIL import Image
 from discord import File
 from sqlitedict import SqliteDict
 
 try:
     print("Attempting to retrieve the settings DB")
-    SETTINGS_DB = SqliteDict("main_db.sqlite", tablename="settings", autocommit=True)
+    SETTINGS_DB = SqliteDict(
+        f"{EnvService.save_path()}/main_db.sqlite",
+        tablename="settings",
+        autocommit=True,
+    )
     print("Retrieved the settings DB")
 except Exception as e:
     print("Failed to retrieve the settings DB. The bot is terminating.")

models/search_model.py

@@ -30,7 +30,7 @@ from llama_index.prompts.prompt_type import PromptType
 from llama_index.readers.web import DEFAULT_WEBSITE_EXTRACTOR
 from langchain import OpenAI
 
-from services.environment_service import EnvService, app_root_path
+from services.environment_service import EnvService
 from services.usage_service import UsageService
 
 MAX_SEARCH_PRICE = EnvService.get_max_search_price()
@@ -57,14 +57,17 @@ class Search:
 
     def add_search_index(self, index, user_id, query):
         # Create a folder called "indexes/{USER_ID}" if it doesn't exist already
-        Path(f"{app_root_path()}/indexes/{user_id}_search").mkdir(
+        Path(f"{EnvService.save_path()}/indexes/{user_id}_search").mkdir(
             parents=True, exist_ok=True
         )
         # Save the index to file under the user id
         file = f"{query[:20]}_{date.today().month}_{date.today().day}"
 
         index.save_to_disk(
-            app_root_path() / "indexes" / f"{str(user_id)}_search" / f"{file}.json"
+            EnvService.save_path()
+            / "indexes"
+            / f"{str(user_id)}_search"
+            / f"{file}.json"
         )
 
     def build_search_started_embed(self):

services/environment_service.py

@@ -43,6 +43,13 @@ class EnvService:
 
         return Path.cwd()
 
+    @staticmethod
+    def save_path():
+        share_dir = os.getenv("SHARE_DIR")
+        if share_dir is not None:
+            return Path(share_dir)
+        return app_root_path()
+
     @staticmethod
     def find_shared_file(file_name):
         share_file_paths = []