diff --git a/.github/workflows/build-and-publish-docker.yml b/.github/workflows/build-and-publish-docker.yml index ed3c8f4..6d6ff4e 100644 --- a/.github/workflows/build-and-publish-docker.yml +++ b/.github/workflows/build-and-publish-docker.yml @@ -6,26 +6,106 @@ name: Docker # documentation. on: - schedule: - - cron: '28 2 * * *' push: - branches: [ "main" ] - # Publish semver tags as releases. - tags: [ 'v*.*.*' ] + branches: + - main + release: + types: [published] pull_request: - branches: [ "main" ] + branches: + - main -env: - # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} +jobs: + docker-push-base: + if: ${{ github.event_name != 'pull_request' && github.repository == 'Kav-K/GPT3Discord' }} + runs-on: ubuntu-20.04 + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + steps: + #Add swap for building arm64 image + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 10 -jobs: - build: + - name: Checkout repository + uses: actions/checkout@v3 + + # https://github.com/sigstore/cosign-installer + - name: Install cosign + uses: sigstore/cosign-installer@v2 + with: + cosign-release: 'v1.13.1' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + + - name: Check if release + uses: haya14busa/action-cond@v1 + id: isRelease + with: + cond: ${{ github.event_name == 'release' }} + if_true: 'linux/amd64,linux/arm64' + if_false: 'linux/amd64' + + # Build and push Docker image with Buildx + - name: Build and push base Docker image + id: build-and-push-base + uses: docker/build-push-action@v4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }}-base + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: ${{ steps.isRelease.outputs.value }} + + + # Sign the resulting Docker image digest + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published base Docker image + env: + COSIGN_EXPERIMENTAL: "true" + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push-base.outputs.digest }} + - runs-on: ubuntu-latest + docker-push-full: + if: ${{ github.event_name != 'pull_request' && github.repository == 'Kav-K/GPT3Discord' }} + runs-on: ubuntu-20.04 permissions: contents: read packages: write @@ -34,63 +114,127 @@ jobs: id-token: write steps: + - name: Set Swap Space + uses: pierotofy/set-swap-space@master + with: + swap-size-gb: 10 + - name: Checkout repository uses: actions/checkout@v3 - # Install the cosign tool except on PR # https://github.com/sigstore/cosign-installer - name: Install cosign - if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0 + uses: sigstore/cosign-installer@v2 with: cosign-release: 'v1.13.1' + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 - # Workaround: https://github.com/docker/build-push-action/issues/461 - name: Setup Docker buildx - uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - # Login against a Docker registry except on PR - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - if: github.event_name != 'pull_request' - uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 with: - registry: ${{ env.REGISTRY }} + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - name: Extract Docker metadata id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@v4 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: ghcr.io/${{ github.repository }} - # Build and push Docker image with Buildx (don't push on PR) + - name: Check if release + uses: haya14busa/action-cond@v1 + id: isRelease + with: + cond: ${{ github.event_name == 'release' }} + if_true: 'linux/amd64,linux/arm64' + if_false: 'linux/amd64' + + # Build and push Docker image with Buildx # https://github.com/docker/build-push-action - - name: Build and push Docker image - id: build-and-push - uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a + - name: Build and push full Docker image + id: build-and-push-full + uses: docker/build-push-action@v4 with: + build-args: FULL=true context: . - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} + push: true + tags: ${{ steps.meta.outputs.tags }}-full labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + platforms: ${{ steps.isRelease.outputs.value }} - # Sign the resulting Docker image digest except on PRs. + # Sign the resulting Docker image digest # This will only write to the public Rekor transparency log when the Docker # repository is public to avoid leaking data. If you would like to publish # transparency data even for private images, pass --force to cosign below. # https://github.com/sigstore/cosign - - name: Sign the published Docker image - if: ${{ github.event_name != 'pull_request' }} + - name: Sign the published full Docker image env: COSIGN_EXPERIMENTAL: "true" # This step uses the identity token to provision an ephemeral certificate # against the sigstore community Fulcio instance. - run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }} + run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push-full.outputs.digest }} + + + docker-pr: + # Only builds amd64 due to arm64 being slow + if: ${{ github.event_name == 'pull_request' && github.repository == 'Kav-K/GPT3Discord' }} + runs-on: ubuntu-20.04 + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + + - name: Build and push base Docker image + id: build-and-push-base + uses: docker/build-push-action@v4 + with: + context: . + push: false + tags: ${{ steps.meta.outputs.tags }}-base + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 + + - name: Build and push full Docker image + id: build-and-push-full + uses: docker/build-push-action@v4 + with: + build-args: FULL=true + context: . + push: false + tags: ${{ steps.meta.outputs.tags }}-full + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64 \ No newline at end of file diff --git a/.github/workflows/docker_upload.yml b/.github/workflows/docker_upload.yml deleted file mode 100644 index 86c3552..0000000 --- a/.github/workflows/docker_upload.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: docker - -on: - push: - branches: - - "main" - release: - types: [published] - -permissions: - contents: read - -jobs: - docker: - if: github.repository == 'Kav-K/GPT3Discord' - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Check + set version tag - run: - echo "GIT_TAG=$(git describe --candidates=0 --tags 2> /dev/null || echo - latest_non_release)" >> $GITHUB_ENV - - - name: Build and push - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: kaveenk/gpt3discord:latest,kaveenk/gpt3discord:${{ env.GIT_TAG }} - - - name: Build and push latest_release tag - if: ${{ github.event_name == 'release' && github.event.action == 'published' }} - uses: docker/build-push-action@v3 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: kaveenk/gpt3discord:latest_release - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/Dockerfile b/Dockerfile index 6a67690..c74b575 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,11 @@ ARG PY_VERSION=3.9 - - # Build container FROM python:${PY_VERSION} as base FROM base as builder ARG PY_VERSION ARG TARGETPLATFORM +ARG FULL COPY . . @@ -14,29 +13,36 @@ COPY . . RUN apt-get update RUN apt-get install -y \ build-essential \ + gcc \ curl -RUN apt-get update RUN curl https://sh.rustup.rs -sSf | bash -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" - RUN mkdir /install /src WORKDIR /install -RUN pip install --target="/install" --upgrade pip setuptools wheel -RUN pip install --target="/install" --upgrade setuptools_rust -# if empty run as usual, if amd64 do the same, if arm64 load an arm version of torch -RUN if [ -z "{$TARGETPLATFORM}" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi -RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi -RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then pip install --target="/install" --upgrade torch==1.9.0 torchvision==0.10.0 -f https://torch.kmtea.eu/whl/stable.html -f https://ext.kmtea.eu/whl/stable.html ; fi -RUN pip install --target="/install" --upgrade git+https://github.com/openai/whisper.git -COPY requirements.txt /install -RUN pip install --target="/install" -r requirements.txt + +RUN pip install --target="/install" --upgrade pip setuptools wheel setuptools_rust + +COPY requirements_base.txt /install +COPY requirements_full.txt /install +RUN pip install --target="/install" --upgrade -r requirements_base.txt +RUN if [ "${FULL}" = "true" ]; then \ + if [ -z "{$TARGETPLATFORM}" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi \ + ; if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi \ + ; if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then pip install --target="/install" --upgrade torch==1.9.0 torchvision==0.10.0 -f https://torch.kmtea.eu/whl/stable.html -f https://ext.kmtea.eu/whl/stable.html ; fi \ + ; pip install --target="/install" --upgrade \ + -r requirements_full.txt \ + ; pip install --target="/install" --upgrade \ + --no-deps --no-build-isolation openai-whisper sentence-transformers==2.2.2 \ + ; fi + COPY README.md /src COPY cogs /src/cogs COPY models /src/models COPY services /src/services COPY gpt3discord.py /src COPY pyproject.toml /src + # For debugging + seeing that the modiles file layouts look correct ... RUN find /src RUN pip install --target="/install" /src @@ -44,25 +50,17 @@ RUN pip install --target="/install" /src # Copy minimal to main image (to keep as small as possible) FROM python:${PY_VERSION}-slim -ENV OPENAI_TOKEN="" -ENV DISCORD_TOKEN="" -ENV PINECONE_TOKEN="" -ENV DEBUG_GUILD="" -ENV DEBUG_CHANNEL="" -ENV ALLOWED_GUILDS="" -ENV ADMIN_ROLES="" -ENV DALLE_ROLES="" -ENV GPT_ROLES="" -ENV WELCOME_MESSAGE="" -ENV USER_INPUT_API_KEYS="" -ENV MODERATIONS_ALERT_CHANNEL="" -ENV USER_KEY_DB_PATH="" - ARG PY_VERSION COPY . . COPY --from=builder /install /usr/local/lib/python${PY_VERSION}/site-packages +#Install ffmpeg and clean +RUN apt-get -y update +RUN apt-get -y install --no-install-recommends ffmpeg +RUN apt-get clean +RUN rm -rf /var/lib/apt/lists/* + RUN mkdir -p /opt/gpt3discord/etc COPY gpt3discord.py /opt/gpt3discord/bin/ -COPY image_optimizer_pretext.txt conversation_starter_pretext.txt conversation_starter_pretext_minimal.txt /opt/gpt3discord/share/ +COPY image_optimizer_pretext.txt language_detection_pretext.txt conversation_starter_pretext.txt conversation_starter_pretext_minimal.txt /opt/gpt3discord/share/ COPY openers /opt/gpt3discord/share/openers CMD ["python3", "/opt/gpt3discord/bin/gpt3discord.py"] diff --git a/detailed_guides/INSTALLATION.md b/detailed_guides/INSTALLATION.md index a62f67c..0044656 100644 --- a/detailed_guides/INSTALLATION.md +++ b/detailed_guides/INSTALLATION.md @@ -143,7 +143,7 @@ If you wish to build your own image then do the following commands instead ```shell # build the image -docker build -t gpt3discord . +docker build --build-arg FULL=true -t gpt3discord . # run it docker run -d --name gpt3discord -v env_file:/opt/gpt3discord/etc/environment -v /containers/gpt3discord:/data -v /containers/gpt3discord/share:/data/share gpt3discord ``` @@ -162,8 +162,8 @@ To use Docker Compose, you need to have Docker and Docker Compose installed on y To start the gpt3discord container with Docker Compose, follow these steps: -1. Open a terminal or command prompt and navigate to the directory that contains the docker-compose.yml file. -2. Open the docker-compose.yml file and replace the environment variable values with your actual tokens and IDs. +1. Rename the `sample.env` file to `.env` and fill it out +2. Open a terminal or command prompt and navigate to the directory that contains the docker-compose.yml file. 3. In the docker-compose.yml replace the volumes with a path on your machine if you don't use the ones listed, the path to replace is the one on the left side of the colon. 4. Run the following command to start the container in detached mode: diff --git a/docker-compose.yml b/docker-compose.yml index 6647db0..2cd301d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,31 +1,12 @@ version: '3.8' services: gpt3discord: - image: ghcr.io/kav-k/gpt3discord:main + image: ghcr.io/kav-k/gpt3discord:main-full container_name: gpt3discord - environment: - OPENAI_TOKEN: "" - DISCORD_TOKEN: "" - PINECONE_TOKEN: "" - PINECONE_REGION: "" - DEBUG_GUILD: "974519864045756446" - DEBUG_CHANNEL: "977697652147892304" - ALLOWED_GUILDS: "971268468148166697,971268468148166697" - GOOGLE_SEARCH_API_KEY: "" - GOOGLE_SEARCH_ENGINE_ID: "" - DEEPL_TOKEN: "" - ADMIN_ROLES: "Admin,Owner" - DALLE_ROLES: "Admin,Openai,Dalle,gpt" - GPT_ROLES: "openai,gpt" - TRANSLATOR_ROLES: "Admin,Owner" - INDEX_ROLES: "Admin,Owner" - SEARCH_ROLES: "Admin,Owner" - CUSTOM_BOT_NAME: "GPT3Discord" - WELCOME_MESSAGE: "Hi There! Welcome to our Discord server. We hope you'll enjoy our server and we look forward to engaging with you!" - USER_INPUT_API_KEYS: "False" - MODERATIONS_ALERT_CHANNEL: "977697652147892304" - USER_KEY_DB_PATH: "user_key_db.sqlite" volumes: + #replace left side with the path to your .env if different, this uses the env in the folder the docker-compose.yml is in + - ./.env:/opt/gpt3discord/etc/environment + # Replace the left side with a path on your hard drive where you want to store the data for both of these - /containers/gpt3discord:/data - /containers/gpt3discord/share:/data/share restart: always diff --git a/pyproject.toml b/pyproject.toml index 04bc610..08156d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,20 +34,38 @@ dependencies = [ "gpt-index", "PyPDF2", "youtube_transcript_api", - "sentence-transformers", "sentencepiece", "protobuf", "python-pptx", ] dynamic = ["version"] + + [project.scripts] gpt3discord = "gpt3discord:init" + [project.urls] Documentation = "https://github.com/Kav-K/GPT3Discord/#readme" Issues = "https://github.com/Kav-K/GPT3Discord/issues" Source = "https://github.com/Kav-K/GPT3Discord" +[project.optional-dependencies] +full = [ + "torch==1.9.1", + "torchvision==1.10.1", + "tokenizers==0.10.3", + "tqdm==4.64.1", + "numpy==1.24.2", + "scipy==1.10.1", + "nltk==3.8.1", + "huggingface-hub==0.12.1", + "more-itertools==9.0.0", + "ffmpeg-python==0.2.0", + "sentence-transformers==2.2.2", + "openai-whisper", +] + [tool.hatch.version] path = "gpt3discord.py" diff --git a/requirements.txt b/requirements.txt index 5a8ef98..2b8fa78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,7 @@ gpt-index==0.4.7 PyPDF2==3.0.1 youtube_transcript_api==0.5.0 sentencepiece==0.1.97 +protobuf==3.20.1 +python-pptx==0.6.21 sentence-transformers==2.2.2 -protobuf==3.20.2 -python-pptx==0.6.21 \ No newline at end of file +openai-whisper \ No newline at end of file diff --git a/requirements_base.txt b/requirements_base.txt new file mode 100644 index 0000000..b9a5846 --- /dev/null +++ b/requirements_base.txt @@ -0,0 +1,18 @@ +Pillow==9.3.0 +py-cord==2.3.2 +python-dotenv==0.21.0 +requests==2.28.1 +transformers==4.25.1 +pycord-multicog==1.0.2 +aiofiles==22.1.0 +pinecone-client==2.1.0 +sqlitedict==2.1.0 +backoff==2.2.1 +flask==2.2.2 +beautifulsoup4==4.11.1 +gpt-index==0.4.7 +PyPDF2==3.0.1 +youtube_transcript_api==0.5.0 +sentencepiece==0.1.97 +protobuf==3.20.1 +python-pptx==0.6.21 \ No newline at end of file diff --git a/requirements_full.txt b/requirements_full.txt new file mode 100644 index 0000000..c6dfb55 --- /dev/null +++ b/requirements_full.txt @@ -0,0 +1,8 @@ +tokenizers==0.13.2 +tqdm==4.64.1 +numpy==1.24.2 +scipy==1.10.1 +nltk==3.8.1 +huggingface-hub==0.12.1 +more-itertools==9.0.0 +ffmpeg-python==0.2.0 \ No newline at end of file diff --git a/sample.env b/sample.env index 236dffd..97a208e 100644 --- a/sample.env +++ b/sample.env @@ -1,6 +1,13 @@ +# DONT PUSH THIS TO GITHUB IF EDITED + OPENAI_TOKEN = "" DISCORD_TOKEN = "" #PINECONE_TOKEN = "" # pinecone token if you have it enabled. See readme +#PINECONE_REGION = "" # add your region here if it's not us-west1-gcp +#GOOGLE_SEARCH_API_KEY: "" +#GOOGLE_SEARCH_ENGINE_ID: "" +#DEEPL_TOKEN: "" + DEBUG_GUILD = "974519864045756446" # discord_server_id DEBUG_CHANNEL = "977697652147892304" # discord_chanel_id ALLOWED_GUILDS = "971268468148166697,971268468148166697" @@ -14,9 +21,15 @@ DALLE_ROLES = "Admin,Openai,Dalle,gpt" # People with the roles in GPT_ROLES can use commands like /gpt ask or /gpt converse GPT_ROLES = "openai,gpt" -WELCOME_MESSAGE = "Hi There! Welcome to our Discord server. We hope you'll enjoy our server and we look forward to engaging with you!" # This is a fallback message if gpt3 fails to generate a welcome message. +TRANSLATOR_ROLES: "Admin,Owner" + +SEARCH_ROLES: "Admin,Owner" -USER_INPUT_API_KEYS="False" # If True, users must use their own API keys for OpenAI. If False, the bot will use the API key in the .env file. +# Change the name of the bot in conversations +CUSTOM_BOT_NAME: "GPT3Discord" + +# If True, users must use their own API keys for OpenAI. If False, the bot will use the API key in the .env file. +USER_INPUT_API_KEYS="False" # Moderations Service alert channel, this is where moderation alerts will be sent as a default if enabled MODERATIONS_ALERT_CHANNEL = "977697652147892304" @@ -28,4 +41,7 @@ USER_KEY_DB_PATH = "user_key_db.sqlite" PRE_MODERATE = "False" # Force only english to be spoken in the server -FORCE_ENGLISH = "False" \ No newline at end of file +FORCE_ENGLISH = "False" + +# The welcome message to send it the welcome setting is set to true +WELCOME_MESSAGE = "Hi There! Welcome to our Discord server. We hope you'll enjoy our server and we look forward to engaging with you!" # This is a fallback message if gpt3 fails to generate a welcome message. \ No newline at end of file