Remove extra envs from dockerfile

Add missing ffmpeg to docker image
Change compose to use the .env file
Remove old docker workflow
Set arm64 to only be added to release docker images
Push to main and pr's make amd64 docker images only
Update INSTALLATION.md
Add more variables to sample.env
Update requirements.txt to include whisper as a pypi package
Update pyproject.toml with optional dependencies as [full]
Make two docker images, base(865MB) and full(1.69GB)
Add two requirements files as base and full for dependencies
Base doesn't have image, video and audio indexing capability
Rene Teigen 1 year ago
parent b034309ec0
commit f5bef32166

@ -6,26 +6,106 @@ name: Docker
# documentation.
on:
schedule:
- cron: '28 2 * * *'
push:
branches: [ "main" ]
# Publish semver tags as releases.
tags: [ 'v*.*.*' ]
branches:
- main
release:
types: [published]
pull_request:
branches: [ "main" ]
branches:
- main
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
jobs:
docker-push-base:
if: ${{ github.event_name != 'pull_request' && github.repository == 'Kav-K/GPT3Discord' }}
runs-on: ubuntu-20.04
permissions:
contents: read
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio when running outside of PRs.
id-token: write
steps:
#Add swap for building arm64 image
- name: Set Swap Space
uses: pierotofy/set-swap-space@master
with:
swap-size-gb: 10
jobs:
build:
- name: Checkout repository
uses: actions/checkout@v3
# https://github.com/sigstore/cosign-installer
- name: Install cosign
uses: sigstore/cosign-installer@v2
with:
cosign-release: 'v1.13.1'
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Setup Docker buildx
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Extract metadata (tags, labels) for Docker
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v4
with:
images: ghcr.io/${{ github.repository }}
- name: Check if release
uses: haya14busa/action-cond@v1
id: isRelease
with:
cond: ${{ github.event_name == 'release' }}
if_true: 'linux/amd64,linux/arm64'
if_false: 'linux/amd64'
# Build and push Docker image with Buildx
- name: Build and push base Docker image
id: build-and-push-base
uses: docker/build-push-action@v4
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}-base
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
platforms: ${{ steps.isRelease.outputs.value }}
# Sign the resulting Docker image digest
# This will only write to the public Rekor transparency log when the Docker
# repository is public to avoid leaking data. If you would like to publish
# transparency data even for private images, pass --force to cosign below.
# https://github.com/sigstore/cosign
- name: Sign the published base Docker image
env:
COSIGN_EXPERIMENTAL: "true"
# This step uses the identity token to provision an ephemeral certificate
# against the sigstore community Fulcio instance.
run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push-base.outputs.digest }}
runs-on: ubuntu-latest
docker-push-full:
if: ${{ github.event_name != 'pull_request' && github.repository == 'Kav-K/GPT3Discord' }}
runs-on: ubuntu-20.04
permissions:
contents: read
packages: write
@ -34,63 +114,127 @@ jobs:
id-token: write
steps:
- name: Set Swap Space
uses: pierotofy/set-swap-space@master
with:
swap-size-gb: 10
- name: Checkout repository
uses: actions/checkout@v3
# Install the cosign tool except on PR
# https://github.com/sigstore/cosign-installer
- name: Install cosign
if: github.event_name != 'pull_request'
uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0
uses: sigstore/cosign-installer@v2
with:
cosign-release: 'v1.13.1'
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
# Workaround: https://github.com/docker/build-push-action/issues/461
- name: Setup Docker buildx
uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
uses: docker/setup-buildx-action@v2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
if: github.event_name != 'pull_request'
uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
uses: docker/metadata-action@v4
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
images: ghcr.io/${{ github.repository }}
# Build and push Docker image with Buildx (don't push on PR)
- name: Check if release
uses: haya14busa/action-cond@v1
id: isRelease
with:
cond: ${{ github.event_name == 'release' }}
if_true: 'linux/amd64,linux/arm64'
if_false: 'linux/amd64'
# Build and push Docker image with Buildx
# https://github.com/docker/build-push-action
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
- name: Build and push full Docker image
id: build-and-push-full
uses: docker/build-push-action@v4
with:
build-args: FULL=true
context: .
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
push: true
tags: ${{ steps.meta.outputs.tags }}-full
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
platforms: ${{ steps.isRelease.outputs.value }}
# Sign the resulting Docker image digest except on PRs.
# Sign the resulting Docker image digest
# This will only write to the public Rekor transparency log when the Docker
# repository is public to avoid leaking data. If you would like to publish
# transparency data even for private images, pass --force to cosign below.
# https://github.com/sigstore/cosign
- name: Sign the published Docker image
if: ${{ github.event_name != 'pull_request' }}
- name: Sign the published full Docker image
env:
COSIGN_EXPERIMENTAL: "true"
# This step uses the identity token to provision an ephemeral certificate
# against the sigstore community Fulcio instance.
run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
run: echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push-full.outputs.digest }}
docker-pr:
# Only builds amd64 due to arm64 being slow
if: ${{ github.event_name == 'pull_request' && github.repository == 'Kav-K/GPT3Discord' }}
runs-on: ubuntu-20.04
permissions:
contents: read
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio when running outside of PRs.
id-token: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Setup Docker buildx
uses: docker/setup-buildx-action@v2
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v4
with:
images: ghcr.io/${{ github.repository }}
- name: Build and push base Docker image
id: build-and-push-base
uses: docker/build-push-action@v4
with:
context: .
push: false
tags: ${{ steps.meta.outputs.tags }}-base
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
platforms: linux/amd64
- name: Build and push full Docker image
id: build-and-push-full
uses: docker/build-push-action@v4
with:
build-args: FULL=true
context: .
push: false
tags: ${{ steps.meta.outputs.tags }}-full
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
platforms: linux/amd64

@ -1,56 +0,0 @@
name: docker
on:
push:
branches:
- "main"
release:
types: [published]
permissions:
contents: read
jobs:
docker:
if: github.repository == 'Kav-K/GPT3Discord'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Check + set version tag
run:
echo "GIT_TAG=$(git describe --candidates=0 --tags 2> /dev/null || echo
latest_non_release)" >> $GITHUB_ENV
- name: Build and push
uses: docker/build-push-action@v3
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: kaveenk/gpt3discord:latest,kaveenk/gpt3discord:${{ env.GIT_TAG }}
- name: Build and push latest_release tag
if: ${{ github.event_name == 'release' && github.event.action == 'published' }}
uses: docker/build-push-action@v3
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: kaveenk/gpt3discord:latest_release
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}

@ -1,12 +1,11 @@
ARG PY_VERSION=3.9
# Build container
FROM python:${PY_VERSION} as base
FROM base as builder
ARG PY_VERSION
ARG TARGETPLATFORM
ARG FULL
COPY . .
@ -14,29 +13,36 @@ COPY . .
RUN apt-get update
RUN apt-get install -y \
build-essential \
gcc \
curl
RUN apt-get update
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN mkdir /install /src
WORKDIR /install
RUN pip install --target="/install" --upgrade pip setuptools wheel
RUN pip install --target="/install" --upgrade setuptools_rust
# if empty run as usual, if amd64 do the same, if arm64 load an arm version of torch
RUN if [ -z "{$TARGETPLATFORM}" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi
RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi
RUN if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then pip install --target="/install" --upgrade torch==1.9.0 torchvision==0.10.0 -f https://torch.kmtea.eu/whl/stable.html -f https://ext.kmtea.eu/whl/stable.html ; fi
RUN pip install --target="/install" --upgrade git+https://github.com/openai/whisper.git
COPY requirements.txt /install
RUN pip install --target="/install" -r requirements.txt
RUN pip install --target="/install" --upgrade pip setuptools wheel setuptools_rust
COPY requirements_base.txt /install
COPY requirements_full.txt /install
RUN pip install --target="/install" --upgrade -r requirements_base.txt
RUN if [ "${FULL}" = "true" ]; then \
if [ -z "{$TARGETPLATFORM}" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi \
; if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then pip install --target="/install" --upgrade torch==1.9.1+cpu torchvision==0.10.1+cpu -f https://download.pytorch.org/whl/torch_stable.html ; fi \
; if [ "${TARGETPLATFORM}" = "linux/arm64" ]; then pip install --target="/install" --upgrade torch==1.9.0 torchvision==0.10.0 -f https://torch.kmtea.eu/whl/stable.html -f https://ext.kmtea.eu/whl/stable.html ; fi \
; pip install --target="/install" --upgrade \
-r requirements_full.txt \
; pip install --target="/install" --upgrade \
--no-deps --no-build-isolation openai-whisper sentence-transformers==2.2.2 \
; fi
COPY README.md /src
COPY cogs /src/cogs
COPY models /src/models
COPY services /src/services
COPY gpt3discord.py /src
COPY pyproject.toml /src
# For debugging + seeing that the modiles file layouts look correct ...
RUN find /src
RUN pip install --target="/install" /src
@ -44,25 +50,17 @@ RUN pip install --target="/install" /src
# Copy minimal to main image (to keep as small as possible)
FROM python:${PY_VERSION}-slim
ENV OPENAI_TOKEN=""
ENV DISCORD_TOKEN=""
ENV PINECONE_TOKEN=""
ENV DEBUG_GUILD=""
ENV DEBUG_CHANNEL=""
ENV ALLOWED_GUILDS=""
ENV ADMIN_ROLES=""
ENV DALLE_ROLES=""
ENV GPT_ROLES=""
ENV WELCOME_MESSAGE=""
ENV USER_INPUT_API_KEYS=""
ENV MODERATIONS_ALERT_CHANNEL=""
ENV USER_KEY_DB_PATH=""
ARG PY_VERSION
COPY . .
COPY --from=builder /install /usr/local/lib/python${PY_VERSION}/site-packages
#Install ffmpeg and clean
RUN apt-get -y update
RUN apt-get -y install --no-install-recommends ffmpeg
RUN apt-get clean
RUN rm -rf /var/lib/apt/lists/*
RUN mkdir -p /opt/gpt3discord/etc
COPY gpt3discord.py /opt/gpt3discord/bin/
COPY image_optimizer_pretext.txt conversation_starter_pretext.txt conversation_starter_pretext_minimal.txt /opt/gpt3discord/share/
COPY image_optimizer_pretext.txt language_detection_pretext.txt conversation_starter_pretext.txt conversation_starter_pretext_minimal.txt /opt/gpt3discord/share/
COPY openers /opt/gpt3discord/share/openers
CMD ["python3", "/opt/gpt3discord/bin/gpt3discord.py"]

@ -143,7 +143,7 @@ If you wish to build your own image then do the following commands instead
```shell
# build the image
docker build -t gpt3discord .
docker build --build-arg FULL=true -t gpt3discord .
# run it
docker run -d --name gpt3discord -v env_file:/opt/gpt3discord/etc/environment -v /containers/gpt3discord:/data -v /containers/gpt3discord/share:/data/share gpt3discord
```
@ -162,8 +162,8 @@ To use Docker Compose, you need to have Docker and Docker Compose installed on y
To start the gpt3discord container with Docker Compose, follow these steps:
1. Open a terminal or command prompt and navigate to the directory that contains the docker-compose.yml file.
2. Open the docker-compose.yml file and replace the environment variable values with your actual tokens and IDs.
1. Rename the `sample.env` file to `.env` and fill it out
2. Open a terminal or command prompt and navigate to the directory that contains the docker-compose.yml file.
3. In the docker-compose.yml replace the volumes with a path on your machine if you don't use the ones listed, the path to replace is the one on the left side of the colon.
4. Run the following command to start the container in detached mode:

@ -1,31 +1,12 @@
version: '3.8'
services:
gpt3discord:
image: ghcr.io/kav-k/gpt3discord:main
image: ghcr.io/kav-k/gpt3discord:main-full
container_name: gpt3discord
environment:
OPENAI_TOKEN: "<openai_api_token>"
DISCORD_TOKEN: "<discord_bot_token>"
PINECONE_TOKEN: "<pinecone_api_token>"
PINECONE_REGION: "<pinecone_region>"
DEBUG_GUILD: "974519864045756446"
DEBUG_CHANNEL: "977697652147892304"
ALLOWED_GUILDS: "971268468148166697,971268468148166697"
GOOGLE_SEARCH_API_KEY: "<google_api_key>"
GOOGLE_SEARCH_ENGINE_ID: "<google_engine_id>"
DEEPL_TOKEN: "<deepl_token>"
ADMIN_ROLES: "Admin,Owner"
DALLE_ROLES: "Admin,Openai,Dalle,gpt"
GPT_ROLES: "openai,gpt"
TRANSLATOR_ROLES: "Admin,Owner"
INDEX_ROLES: "Admin,Owner"
SEARCH_ROLES: "Admin,Owner"
CUSTOM_BOT_NAME: "GPT3Discord"
WELCOME_MESSAGE: "Hi There! Welcome to our Discord server. We hope you'll enjoy our server and we look forward to engaging with you!"
USER_INPUT_API_KEYS: "False"
MODERATIONS_ALERT_CHANNEL: "977697652147892304"
USER_KEY_DB_PATH: "user_key_db.sqlite"
volumes:
#replace left side with the path to your .env if different, this uses the env in the folder the docker-compose.yml is in
- ./.env:/opt/gpt3discord/etc/environment
# Replace the left side with a path on your hard drive where you want to store the data for both of these
- /containers/gpt3discord:/data
- /containers/gpt3discord/share:/data/share
restart: always

@ -34,20 +34,38 @@ dependencies = [
"gpt-index",
"PyPDF2",
"youtube_transcript_api",
"sentence-transformers",
"sentencepiece",
"protobuf",
"python-pptx",
]
dynamic = ["version"]
[project.scripts]
gpt3discord = "gpt3discord:init"
[project.urls]
Documentation = "https://github.com/Kav-K/GPT3Discord/#readme"
Issues = "https://github.com/Kav-K/GPT3Discord/issues"
Source = "https://github.com/Kav-K/GPT3Discord"
[project.optional-dependencies]
full = [
"torch==1.9.1",
"torchvision==1.10.1",
"tokenizers==0.10.3",
"tqdm==4.64.1",
"numpy==1.24.2",
"scipy==1.10.1",
"nltk==3.8.1",
"huggingface-hub==0.12.1",
"more-itertools==9.0.0",
"ffmpeg-python==0.2.0",
"sentence-transformers==2.2.2",
"openai-whisper",
]
[tool.hatch.version]
path = "gpt3discord.py"

@ -14,6 +14,7 @@ gpt-index==0.4.7
PyPDF2==3.0.1
youtube_transcript_api==0.5.0
sentencepiece==0.1.97
protobuf==3.20.1
python-pptx==0.6.21
sentence-transformers==2.2.2
protobuf==3.20.2
python-pptx==0.6.21
openai-whisper

@ -0,0 +1,18 @@
Pillow==9.3.0
py-cord==2.3.2
python-dotenv==0.21.0
requests==2.28.1
transformers==4.25.1
pycord-multicog==1.0.2
aiofiles==22.1.0
pinecone-client==2.1.0
sqlitedict==2.1.0
backoff==2.2.1
flask==2.2.2
beautifulsoup4==4.11.1
gpt-index==0.4.7
PyPDF2==3.0.1
youtube_transcript_api==0.5.0
sentencepiece==0.1.97
protobuf==3.20.1
python-pptx==0.6.21

@ -0,0 +1,8 @@
tokenizers==0.13.2
tqdm==4.64.1
numpy==1.24.2
scipy==1.10.1
nltk==3.8.1
huggingface-hub==0.12.1
more-itertools==9.0.0
ffmpeg-python==0.2.0

@ -1,6 +1,13 @@
# DONT PUSH THIS TO GITHUB IF EDITED
OPENAI_TOKEN = "<openai_api_token>"
DISCORD_TOKEN = "<discord_bot_token>"
#PINECONE_TOKEN = "<pinecone_token>" # pinecone token if you have it enabled. See readme
#PINECONE_REGION = "<pinecone_region>" # add your region here if it's not us-west1-gcp
#GOOGLE_SEARCH_API_KEY: "<google_api_key>"
#GOOGLE_SEARCH_ENGINE_ID: "<google_engine_id>"
#DEEPL_TOKEN: "<deepl_token>"
DEBUG_GUILD = "974519864045756446" # discord_server_id
DEBUG_CHANNEL = "977697652147892304" # discord_chanel_id
ALLOWED_GUILDS = "971268468148166697,971268468148166697"
@ -14,9 +21,15 @@ DALLE_ROLES = "Admin,Openai,Dalle,gpt"
# People with the roles in GPT_ROLES can use commands like /gpt ask or /gpt converse
GPT_ROLES = "openai,gpt"
WELCOME_MESSAGE = "Hi There! Welcome to our Discord server. We hope you'll enjoy our server and we look forward to engaging with you!" # This is a fallback message if gpt3 fails to generate a welcome message.
TRANSLATOR_ROLES: "Admin,Owner"
SEARCH_ROLES: "Admin,Owner"
USER_INPUT_API_KEYS="False" # If True, users must use their own API keys for OpenAI. If False, the bot will use the API key in the .env file.
# Change the name of the bot in conversations
CUSTOM_BOT_NAME: "GPT3Discord"
# If True, users must use their own API keys for OpenAI. If False, the bot will use the API key in the .env file.
USER_INPUT_API_KEYS="False"
# Moderations Service alert channel, this is where moderation alerts will be sent as a default if enabled
MODERATIONS_ALERT_CHANNEL = "977697652147892304"
@ -28,4 +41,7 @@ USER_KEY_DB_PATH = "user_key_db.sqlite"
PRE_MODERATE = "False"
# Force only english to be spoken in the server
FORCE_ENGLISH = "False"
FORCE_ENGLISH = "False"
# The welcome message to send it the welcome setting is set to true
WELCOME_MESSAGE = "Hi There! Welcome to our Discord server. We hope you'll enjoy our server and we look forward to engaging with you!" # This is a fallback message if gpt3 fails to generate a welcome message.
Loading…
Cancel
Save