From d34bfe8679f2c84a4e789d324d9382723fd870fc Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 19:20:25 +0000 Subject: [PATCH 01/17] add Dockerfile --- Dockerfile | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..30eacd3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM python:alpine AS build +LABEL git="https://github.com/Aeriqu/CroBot" +ARG KYTEA_VERSION=0.4.7 + +WORKDIR /app + +RUN apk add --no-cache build-base libffi-dev && \ +wget http://www.phontron.com/kytea/download/kytea-${KYTEA_VERSION}.tar.gz && \ +tar xf kytea-${KYTEA_VERSION}.tar.gz && \ +cd kytea-${KYTEA_VERSION} && ./configure && make && \ +mkdir /app/kytea && make install prefix=/app/kytea + +RUN pip install virtualenv && python -m virtualenv venv +COPY /requirements.txt requirements.txt +RUN ./venv/bin/pip install \ +--global-option=build_ext \ +--global-option="-L/app/kytea/lib/" \ +--global-option="-I/app/kytea/include/" \ +-r requirements.txt + + +FROM python:alpine AS runtime +RUN apk add --no-cache libstdc++ +WORKDIR /app +COPY --from=build /app/venv venv +# we copy kytea/{bin,include,lib,share} +COPY --from=build /app/kytea/ /usr/local/ +ENV PATH="/app/venv/bin:$PATH" +RUN ln -s /usr/local/share/kytea/model.bin + +COPY . . +RUN ./db_init.py +CMD ./run.py From 8e10effff92cdb6a051cefa9cd249ac329bb8706 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 19:20:34 +0000 Subject: [PATCH 02/17] remove database update from db_init --- db_init.py | 2 -- 1 file changed, 2 deletions(-) mode change 100644 => 100755 db_init.py diff --git a/db_init.py b/db_init.py old mode 100644 new mode 100755 index 77ca66a..e426130 --- a/db_init.py +++ b/db_init.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 import asyncio -import CroBot.features.sdvxin.sdvx import CroBot.features.sdvxin.database async def upd(): await CroBot.features.sdvxin.database.recreate_db() - await CroBot.features.sdvxin.sdvx.update() def main(): loop = asyncio.new_event_loop() From 5ed638c399f09d6b4086f2a7be4f8745597ca9cb Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 19:20:50 +0000 Subject: [PATCH 03/17] add shebang and make run.py executable --- run.py | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 run.py diff --git a/run.py b/run.py old mode 100644 new mode 100755 index f191fe8..e7a8ba2 --- a/run.py +++ b/run.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import configparser import discord from discord.utils import get From 26d6487560e23b4080101f3e383442bf612b2424 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 19:22:05 +0000 Subject: [PATCH 04/17] update readme for db_init changes --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 48f65e0..1162b92 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,9 @@ source](http://www.phontron.com/kytea/#download). sdvxin: 1. Obtain a kytea model, name it ``model.bin``, and place it in the same directory as ``run.py`` -2. Run ``db_init.py`` to initialize the database and download the metadata from sdvx.in -3. Edit the configuration file for an api key for azure's cognitive translation features +2. Run ``db_init.py`` to initialize the database +3. Execute ``!sdvxin update`` in a channel with the bot to download the metadata from sdvx.in (this will take a while) +4. Edit the configuration file for an api key for azure's cognitive translation features ## License From 1aaa82424e5c1d5b1d0769d9b48a05a731b3e301 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 7 May 2020 23:16:39 +0100 Subject: [PATCH 05/17] Create dockerimage.yml --- .github/workflows/dockerimage.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/dockerimage.yml diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml new file mode 100644 index 0000000..a3e46ea --- /dev/null +++ b/.github/workflows/dockerimage.yml @@ -0,0 +1,15 @@ +name: Docker Image CI + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build the Docker image + run: docker build -t aeriqu/crobot:latest . From 10fb4ae3fc9265e33999956bd6fc7e305c9d4225 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 7 May 2020 23:19:50 +0100 Subject: [PATCH 06/17] Update and rename dockerimage.yml to docker.yml --- .github/workflows/docker.yml | 71 +++++++++++++++++++++++++++++++ .github/workflows/dockerimage.yml | 15 ------- 2 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/dockerimage.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..4bae9cc --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,71 @@ +name: Docker + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: aeriqu/crobot + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: | + if [ -f docker-compose.test.yml ]; then + docker-compose --file docker-compose.test.yml build + docker-compose --file docker-compose.test.yml run sut + else + docker build . --file Dockerfile + fi + + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: docker build . --file Dockerfile --tag $IMAGE_NAME + + - name: Log into registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin + + - name: Push image + run: | + IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag image $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml deleted file mode 100644 index a3e46ea..0000000 --- a/.github/workflows/dockerimage.yml +++ /dev/null @@ -1,15 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build the Docker image - run: docker build -t aeriqu/crobot:latest . From 2d9f41d3ea81aeb43dc73da9a474b53de08df48d Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 7 May 2020 23:22:15 +0100 Subject: [PATCH 07/17] remove test for now it's literally the same as just building the image --- .github/workflows/docker.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4bae9cc..164a49a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,26 +15,20 @@ env: jobs: # Run tests. # See also https://docs.docker.com/docker-hub/builds/automated-testing/ - test: - runs-on: ubuntu-latest + #test: + # runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 + # steps: + # - uses: actions/checkout@v2 - - name: Run tests - run: | - if [ -f docker-compose.test.yml ]; then - docker-compose --file docker-compose.test.yml build - docker-compose --file docker-compose.test.yml run sut - else - docker build . --file Dockerfile - fi + # - name: Run tests + # run: docker build . # Push image to GitHub Packages. # See also https://docs.docker.com/docker-hub/builds/ push: # Ensure test job passes before pushing image. - needs: test + #needs: test runs-on: ubuntu-latest if: github.event_name == 'push' From a243a9c4a916d057c1345c77198f050300a3dcb0 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 7 May 2020 23:23:14 +0100 Subject: [PATCH 08/17] use Dockerfile in build implicitly --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 164a49a..2d53b04 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@v2 - name: Build image - run: docker build . --file Dockerfile --tag $IMAGE_NAME + run: docker build -t $IMAGE_NAME . - name: Log into registry run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin From 642aaa352ee1e7304e4ada432aa3f42280a222b7 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 22:28:47 +0000 Subject: [PATCH 09/17] run on any branch --- .github/workflows/docker.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2d53b04..437bcef 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,13 +1,6 @@ name: Docker -on: - push: - # Publish `master` as Docker `latest` image. - branches: - - master - - # Run tests for any PRs. - pull_request: +on: [push, pull_request] env: IMAGE_NAME: aeriqu/crobot From 3f78d5f85d1c2c7ecf5f95370c06ce810b842f18 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 22:42:43 +0000 Subject: [PATCH 10/17] tag based on $IMAGE_NAME --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 437bcef..ac6af15 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,7 @@ name: Docker on: [push, pull_request] env: - IMAGE_NAME: aeriqu/crobot + IMAGE_NAME: crobot jobs: # Run tests. @@ -38,7 +38,7 @@ jobs: - name: Push image run: | IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME - + # Change all uppercase to lowercase IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') @@ -54,5 +54,5 @@ jobs: echo IMAGE_ID=$IMAGE_ID echo VERSION=$VERSION - docker tag image $IMAGE_ID:$VERSION + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION docker push $IMAGE_ID:$VERSION From f4d9df7c8761feece6984785030c7d78f2686ae1 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 23:05:28 +0000 Subject: [PATCH 11/17] switch to docker hub from github packages what in the fuck is the point of a package registry if you can't pull public images without authentication, github? --- .github/workflows/docker.yml | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ac6af15..d8fef7c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,44 +3,30 @@ name: Docker on: [push, pull_request] env: - IMAGE_NAME: crobot + IMAGE_NAME: aeriqu/crobot jobs: - # Run tests. - # See also https://docs.docker.com/docker-hub/builds/automated-testing/ - #test: - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v2 - - # - name: Run tests - # run: docker build . - - # Push image to GitHub Packages. - # See also https://docs.docker.com/docker-hub/builds/ push: - # Ensure test job passes before pushing image. - #needs: test - runs-on: ubuntu-latest if: github.event_name == 'push' steps: - uses: actions/checkout@v2 + - name: Docker Login + env: + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + run: | + echo "${DOCKER_PASSWORD}" | docker login -u ${DOCKER_USERNAME} --password-stdin + - name: Build image run: docker build -t $IMAGE_NAME . - - name: Log into registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin - - name: Push image run: | - IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME - # Change all uppercase to lowercase - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + IMAGE_ID=$(echo $IMAGE_NAME | tr '[A-Z]' '[a-z]') # Strip git ref prefix from version VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') From 0aff79ddbf34626ad9bb982928e5cdd34226c79d Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 23:11:07 +0000 Subject: [PATCH 12/17] use image id based on user and repo --- .github/workflows/docker.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d8fef7c..73f85b1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,7 @@ name: Docker on: [push, pull_request] env: - IMAGE_NAME: aeriqu/crobot + IMAGE_NAME: crobot jobs: push: @@ -25,8 +25,9 @@ jobs: - name: Push image run: | + IMAGE_ID=${{ github.repository }} # Change all uppercase to lowercase - IMAGE_ID=$(echo $IMAGE_NAME | tr '[A-Z]' '[a-z]') + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') # Strip git ref prefix from version VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') From 96b479f828a9f9d0d8f8e629ee3b3fe7f7786a02 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 23:18:30 +0000 Subject: [PATCH 13/17] add docker readme instructions --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1162b92..785ec6d 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,15 @@ To properly use kytea, a model file is required. You can obtain one from the [ky source](http://www.phontron.com/kytea/#download). -### Set up +## Set up + +### Docker setup + +1. Install [Docker](https://docs.docker.com/get-docker/) +2. Set up a settings.ini file using [the one in the repo](https://github.com/Aeriqu/CroBot/blob/master/settings.ini) as a template. +3. Run ``docker run -d -v crobot_db:/app/sdvxCharts.db -v /path/to/your/settings.ini:/app/settings.ini aeriqu/crobot`` + +### Traditional setup #### General setup From ecd8138542451c2adeb0e90411f1c6639edeb7da Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 23:26:50 +0000 Subject: [PATCH 14/17] push and cache build layer --- .github/workflows/docker.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 73f85b1..0290e45 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,15 @@ jobs: echo "${DOCKER_PASSWORD}" | docker login -u ${DOCKER_USERNAME} --password-stdin - name: Build image - run: docker build -t $IMAGE_NAME . + run: | + docker pull ${{ github.repository }}:build || true + docker pull ${{ github.repository }}:latest || true + docker build --target build \ + --cache-from=${{ github.repository }}:build \ + --tag $IMAGE_NAME:build . + docker build --target runtime \ + --cache-from=${{ github.repository }}:latest \ + --tag $IMAGE_NAME . - name: Push image run: | @@ -42,4 +50,6 @@ jobs: echo VERSION=$VERSION docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker tag $IMAGE_NAME:build $IMAGE_ID:build docker push $IMAGE_ID:$VERSION + docker push $IMAGE_ID:build From feeed43f39d92c9bb6c5c64d80e7641056f9285b Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 23:29:26 +0000 Subject: [PATCH 15/17] lowercase image id in build --- .github/workflows/docker.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0290e45..eb81fea 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -22,13 +22,17 @@ jobs: - name: Build image run: | - docker pull ${{ github.repository }}:build || true - docker pull ${{ github.repository }}:latest || true + IMAGE_ID=${{ github.repository }} + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker pull $IMAGE_ID:build || true + docker pull $IMAGE_ID:latest || true docker build --target build \ - --cache-from=${{ github.repository }}:build \ + --cache-from=$IMAGE_ID:build \ --tag $IMAGE_NAME:build . docker build --target runtime \ - --cache-from=${{ github.repository }}:latest \ + --cache-from=$IMAGE_ID:latest \ --tag $IMAGE_NAME . - name: Push image From 3d5c6a14a34b7b03c76830435d0f77f5e2466d99 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 23:36:46 +0000 Subject: [PATCH 16/17] use build cache on second build also --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eb81fea..8712ca5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -32,6 +32,7 @@ jobs: --cache-from=$IMAGE_ID:build \ --tag $IMAGE_NAME:build . docker build --target runtime \ + --cache-from=$IMAGE_ID:build \ --cache-from=$IMAGE_ID:latest \ --tag $IMAGE_NAME . From 672c50a165bf64b851c9a87ffc21ed5c5d8ad769 Mon Sep 17 00:00:00 2001 From: Alex W Date: Thu, 7 May 2020 23:48:31 +0000 Subject: [PATCH 17/17] use an explicit path instead of a docker volume we can't map just a file in the container to a volume, it has to be a directory --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 785ec6d..6f75587 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ source](http://www.phontron.com/kytea/#download). 1. Install [Docker](https://docs.docker.com/get-docker/) 2. Set up a settings.ini file using [the one in the repo](https://github.com/Aeriqu/CroBot/blob/master/settings.ini) as a template. -3. Run ``docker run -d -v crobot_db:/app/sdvxCharts.db -v /path/to/your/settings.ini:/app/settings.ini aeriqu/crobot`` +3. Run ``docker run -d -v /path/to/anywhere/sdvxCharts.db:/app/sdvxCharts.db -v /path/to/your/settings.ini:/app/settings.ini aeriqu/crobot`` ### Traditional setup