From aab2997e10da3a25cd8fbb2ba958f98c36d24228 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?F=C3=A9lix=20Voituret?= <fvoituret@deezer.com>
Date: Fri, 12 Feb 2021 11:58:12 +0100
Subject: [PATCH] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20split=20jobs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .github/workflows/docker.yml | 156 ++++++++++++++++++++++-------------
 1 file changed, 98 insertions(+), 58 deletions(-)

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index daeccee..b8613ac 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -7,78 +7,118 @@ on:
         required: true
         default: "2.1.2"
 jobs:
-  build-test-push:
+  cuda-base:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        distribution: [3.6, 3.7, 3.8]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build CUDA base image
+        run: |
+          docker build \
+            --build-arg BASE=python:${{ matrix.distribution }} \
+            -t deezer/python-cuda-10-1:${{ matrix.distribution }} \
+            -f docker/cuda-10-1.dockerfile .
+      - name: Docker login
+        run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
+      - name: Push deezer/python-cuda-10-1:${{ matrix.distribution }} image
+        run: docker push deezer/python-cuda-10-1:${{ matrix.distribution }}
+  pip-images:
+    needs: cuda-base
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [cpu, gpu]
+        distribution: [3.6, 3.7, 3.8]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - if: ${{ matrix.platform == 'cpu' }}
+        run: |
+          echo "base=deezer/spleeter:${{ matrix.distribution }}" >> $GITHUB_ENV
+          echo "image=spleeter" >> $GITHUB_ENV
+      - if: ${{ matrix.platform == 'gpu' }}
+        run: |
+          echo "base=deezer/python-cuda-10-1:${{ matrix.distribution }}" >> $GITHUB_ENV
+          echo "image=spleeter-gpu" >> $GITHUB_ENV
+      - name: Build deezer/${{ env.image }}:${{ matrix.distribution }} image
+        run: |
+          docker build \
+            --build-arg BASE=${{ env.base }} \
+            --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \
+            -t deezer/${{ env.image }}:${{ matrix.distribution }} \
+            -f docker/spleeter.dockerfile .
+      - name: Test deezer/${{ env.image }}:${{ matrix.distribution }} image
+        run: |
+          docker run \
+            -v $(pwd):/runtime \
+            deezer/${{ env.image }}:${{ matrix.distribution }} \
+            separate -i /runtime/audio_example.mp3  -o /tmp
+      - name: Docker login
+        run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
+      - name: Push deezer/${{ env.image }}:${{ matrix.distribution }} image
+        run: docker push deezer/${{ env.image }}:${{ matrix.distribution }}
+  conda-images:
+    needs: cuda-base
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform: [cpu, gpu]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - if: ${{ matrix.platform == 'cpu' }}
+      - name: Build Conda base image
+        run: |
+          docker build -t conda:cpu -f docker/conda.dockerfile .
+          echo "image=spleeter" >> $GITHUB_ENV
+      - if: ${{ matrix.platform == 'gpu' }}
+      - name: Build Conda base image
+        run: |
+          docker build --build-arg BASE=deezer/python-cuda-10-1:3.8 -t conda:gpu -f docker/conda.dockerfile .
+          echo "image=spleeter-gpu" >> $GITHUB_ENV
+      - name: Build deezer/${{ env.image }}:${{ env.tag }} image
+        run: |
+          docker build \
+            --build-arg BASE=conda:${{ matrix.platform }} \
+            --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \
+            -t deezer/${{ env.image }}:conda \
+            -f docker/spleeter-conda.dockerfile .
+      - name: Docker login
+        run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
+      - name: Push deezer/${{ env.image }}:conda image
+        run: docker push deezer/${{ env.image }}:conda
+  images-with-model:
+    needs: [pip-images, conda-images]
     runs-on: ubuntu-latest
     strategy:
       matrix:
         platform: [cpu, gpu]
         distribution: [3.6, 3.7, 3.8, conda]
-        model: [modelless, 2stems, 4stems, 5stems]
+        model: [2stems, 4stems, 5stems]
       fail-fast: true
     steps:
       - uses: actions/checkout@v2
-      # ----------------------------------------------------------------------
-      # Note: base image building and env setup.
-      - name: Setup Python distribution
-        run: |
-          echo "base=python:${{ matrix.distribution }}" >> $GITHUB_ENV
-          echo "tag=${{ matrix.distribution }}" >> $GITHUB_ENV
-          echo "file=spleeter" >> $GITHUB_ENV
-      - if: ${{ matrix.distribution == 'conda' }}
-        name: Build Conda base image
-        run: |
-          docker build -t python:conda -f docker/conda.dockerfile .
-          echo "file=spleeter-conda" >> $GITHUB_ENV
+      - if: ${{ matrix.platform == 'cpu' }}
+        run: echo "image=spleeter" >> $GITHUB_ENV
       - if: ${{ matrix.platform == 'gpu' }}
-        name: Build CUDA base image
+        run: echo "image=spleeter-gpu" >> $GITHUB_ENV
+      - name: Build deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image
         run: |
           docker build \
-            --build-arg BASE=python:${{ matrix.distribution }} \
-            -t cuda:${{ matrix.distribution }} \
-            -f docker/cuda-10-1.dockerfile .
-          echo "base=${{ matrix.distribution }}" >> $GITHUB_ENV
-          echo "tag=${{ matrix.distribution }}-gpu" >> $GITHUB_ENV
-      # ----------------------------------------------------------------------
-      # Note: image building.
-      - name: Build deezer/spleeter:${{ env.tag }} image
-        run: |
-          docker build \
-            --build-arg BASE=${{ env.base }} \
-            --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \
-            -t deezer/spleeter:${{ env.tag }} \
-            -f docker/${{ env.file }}.dockerfile .
-          echo "modelargs=" >> $GITHUB_ENV
-      - if: ${{ matrix.model != 'modelless' }}
-        name: Build deezer/spleeter:${{ env.tag }}-${{ matrix.model }} image
-        run: |
-          docker build \
-            --build-arg BASE=deezer/spleeter:${{ env.tag }} \
+            --build-arg BASE=deezer/${{ env.image }}:${{ matrix.distribution }} \
             --build-arg MODEL=${{ matrix.model }} \
-            -t deezer/spleeter:${{ env.tag }}-${{ matrix.model }} \
-            -f docker/spleeter-model.dockerfile .
-          echo "tag=${{ env.tag }}-${{ matrix.model }}" >> $GITHUB_ENV
-          echo "modelarg=-p spleeter:${{ matrix.model }}" >> $GITHUB_ENV
-      # ----------------------------------------------------------------------
-      # Note: image testing.
-      - name: Test deezer/spleeter:${{ env.tag }} image
+            -t deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} \
+            -f docker/spleeter-model.dockerfile .dockerfile
+      - name: Test deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image
         run: |
           docker run \
             -v $(pwd):/runtime \
-            deezer/spleeter:${{ env.tag }} \
-            separate -i /runtime/audio_example.mp3  -o /tmp \${{ env.modelarg }}
-      # ----------------------------------------------------------------------
-      # Note: image deploy.
+            deezer/${{ env.image }}:${{ matrix.distribution }} \
+            separate -i /runtime/audio_example.mp3 -o /tmp -p spleeter:${{ matrix.model }}
       - name: Docker login
         run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
-      - name: Push deezer/spleeter:${{ env.tag }} image
-        run: docker push deezer/spleeter:${{ env.tag }}
-      - if: ${{ env.tag == 'spleeter:3.8' }}
-        name: Push deezer/spleeter:latest image
-        run: |
-          docker tag deezer/spleeter:3.8 deezer/spleeter:latest
-          docker push deezer/spleeter:latest
-      - if: ${{ env.tag == 'spleeter:3.8-gpu' }}
-        name: Push deezer/spleeter:gpu image
-        run: |
-          docker tag deezer/spleeter:3.8-gpu deezer/spleeter:gpu
-          docker push deezer/spleeter:gpu
+      - name: Push deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image
+        run: docker push deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }}