From 8994dde6e81f3e64f1c96d523a3d84a05b407271 Mon Sep 17 00:00:00 2001 From: Moussallam Date: Fri, 29 Jan 2021 18:20:48 +0100 Subject: [PATCH 01/15] Update README.md adding the necessary conda add channel command in the quickstart --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c77362d..aa4f478 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ Ready to dig into it ? In a few lines you can install **Spleeter** using [Conda] ```bash # install using conda -conda install -c conda-forge spleeter +conda config --add channels conda-forge # only needed if you don't already have this channel set +conda install -c deezer-research spleeter # download an example audio file (if you don't have wget, use another tool for downloading) wget https://github.com/deezer/spleeter/raw/master/audio_example.mp3 # separate the example audio into two components From 69588a7be669a9c3db528a4baff60a5d7a1d255e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 10:40:41 +0100 Subject: [PATCH 02/15] =?UTF-8?q?=E2=9C=A8=20=20fix=20docker=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index afb7d6d..a434740 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,13 +1,18 @@ name: docker on: - - workflow_dispatch + workflow_dispatch: + inputs: + version: + description: "Spleeter version to build image for" + required: true + default: "2.1.2" jobs: build-test-push: runs-on: ubuntu-latest strategy: matrix: platform: [cpu, gpu] - distribution: [3.6, 3.7, 3.8, conda] + distribution: [3.7, 3.8, conda] model: [modelless, 2stems, 4stems, 5stems] fail-fast: true steps: @@ -19,7 +24,6 @@ jobs: echo "::set-env name=base::python:${{ matrix.distribution }}" echo "::set-env name=tag::${{ matrix.distribution }}" echo "::set-env name=file::spleeter" - echo "::set-env name=package::spleeter" - if: ${{ matrix.distribution == 'conda' }} name: Build Conda base image run: | @@ -31,17 +35,16 @@ jobs: docker build \ --build-arg BASE=python:${{ matrix.distribution }} \ -t cuda:${{ matrix.distribution }} \ - -f docker/cuda-10-0.dockerfile . + -f docker/cuda-10-1.dockerfile . echo "::set-env name=base::cuda:${{ matrix.distribution }}" echo "::set-env name=tag::${{ matrix.distribution }}-gpu" - echo "::set-env name=package::spleeter-gpu" # ---------------------------------------------------------------------- # Note: image building. - name: Build deezer/spleeter:${{ env.tag }} image run: | docker build \ --build-arg BASE=${{ env.base }} \ - --build-arg SPLEETER_PACKAGE=${{ env.package }} \ + --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \ -t deezer/spleeter:${{ env.tag }} \ -f docker/${{ env.file }}.dockerfile . echo "::set-env name=modelargs::" @@ -54,7 +57,7 @@ jobs: -t deezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ -f docker/spleeter-model.dockerfile . echo "::set-env name=tag::${{ env.tag }}-${{ matrix.model }}" - echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}" + echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}" # ---------------------------------------------------------------------- # Note: image testing. - name: Test deezer/spleeter:${{ env.tag }} image From ddc5debed7118c58cf697ede6dbcbac7b27dc40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 10:40:57 +0100 Subject: [PATCH 03/15] =?UTF-8?q?=E2=9C=A8=20fix=20docker=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/spleeter-conda.dockerfile | 3 +-- docker/spleeter.dockerfile | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/spleeter-conda.dockerfile b/docker/spleeter-conda.dockerfile index 2a21cd8..9eb2883 100644 --- a/docker/spleeter-conda.dockerfile +++ b/docker/spleeter-conda.dockerfile @@ -2,12 +2,11 @@ ARG BASE=conda FROM ${BASE} -ARG SPLEETER_PACKAGE=spleeter ARG SPLEETER_VERSION=1.5.3 ENV MODEL_PATH /model RUN mkdir -p /model RUN conda install -y -c conda-forge musdb -RUN conda install -y -c conda-forge ${SPLEETER_PACKAGE}==${SPLEETER_VERSION} +RUN conda install -y -c conda-forge spleeter==${SPLEETER_VERSION} COPY docker/conda-entrypoint.sh spleeter-entrypoint.sh ENTRYPOINT ["/bin/bash", "spleeter-entrypoint.sh"] \ No newline at end of file diff --git a/docker/spleeter.dockerfile b/docker/spleeter.dockerfile index 1161c3a..745ceae 100644 --- a/docker/spleeter.dockerfile +++ b/docker/spleeter.dockerfile @@ -2,13 +2,12 @@ ARG BASE=python:3.6 FROM ${BASE} -ARG SPLEETER_PACKAGE=spleeter ARG SPLEETER_VERSION=1.5.3 ENV MODEL_PATH /model RUN mkdir -p /model RUN apt-get update && apt-get install -y ffmpeg libsndfile1 RUN pip install musdb museval -RUN pip install ${SPLEETER_PACKAGE}==${SPLEETER_VERSION} +RUN pip install spleeter==${SPLEETER_VERSION} ENTRYPOINT ["spleeter"] From 5ac9b01d0cf19e0963c091fa25a48c159cbab140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 10:44:36 +0100 Subject: [PATCH 04/15] =?UTF-8?q?=F0=9F=90=9B=20=20Fix=20conda=20installat?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 2 +- docker/spleeter-conda.dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a434740..05c396b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: platform: [cpu, gpu] - distribution: [3.7, 3.8, conda] + distribution: [3.6, 3.7, 3.8, conda] model: [modelless, 2stems, 4stems, 5stems] fail-fast: true steps: diff --git a/docker/spleeter-conda.dockerfile b/docker/spleeter-conda.dockerfile index 9eb2883..4a8f704 100644 --- a/docker/spleeter-conda.dockerfile +++ b/docker/spleeter-conda.dockerfile @@ -6,7 +6,8 @@ ARG SPLEETER_VERSION=1.5.3 ENV MODEL_PATH /model RUN mkdir -p /model +RUN conda config --add channels conda-forge RUN conda install -y -c conda-forge musdb -RUN conda install -y -c conda-forge spleeter==${SPLEETER_VERSION} +RUN conda install -y -c deezer-research spleeter COPY docker/conda-entrypoint.sh spleeter-entrypoint.sh ENTRYPOINT ["/bin/bash", "spleeter-entrypoint.sh"] \ No newline at end of file From 0c710b118da1681982789b9c7c1cb335656043f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 11:18:07 +0100 Subject: [PATCH 05/15] =?UTF-8?q?=F0=9F=92=9A=20=20fix=20env=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 05c396b..daeccee 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,14 +21,14 @@ jobs: # Note: base image building and env setup. - name: Setup Python distribution run: | - echo "::set-env name=base::python:${{ matrix.distribution }}" - echo "::set-env name=tag::${{ matrix.distribution }}" - echo "::set-env name=file::spleeter" + echo "base=python:${{ matrix.distribution }}" >> $GITHUB_ENV + echo "tag=${{ matrix.distribution }}" >> $GITHUB_ENV + echo "file=spleeter" >> $GITHUB_ENV - if: ${{ matrix.distribution == 'conda' }} name: Build Conda base image run: | docker build -t python:conda -f docker/conda.dockerfile . - echo "::set-env name=file::spleeter-conda" + echo "file=spleeter-conda" >> $GITHUB_ENV - if: ${{ matrix.platform == 'gpu' }} name: Build CUDA base image run: | @@ -36,8 +36,8 @@ jobs: --build-arg BASE=python:${{ matrix.distribution }} \ -t cuda:${{ matrix.distribution }} \ -f docker/cuda-10-1.dockerfile . - echo "::set-env name=base::cuda:${{ matrix.distribution }}" - echo "::set-env name=tag::${{ matrix.distribution }}-gpu" + echo "base=${{ matrix.distribution }}" >> $GITHUB_ENV + echo "tag=${{ matrix.distribution }}-gpu" >> $GITHUB_ENV # ---------------------------------------------------------------------- # Note: image building. - name: Build deezer/spleeter:${{ env.tag }} image @@ -47,7 +47,7 @@ jobs: --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \ -t deezer/spleeter:${{ env.tag }} \ -f docker/${{ env.file }}.dockerfile . - echo "::set-env name=modelargs::" + echo "modelargs=" >> $GITHUB_ENV - if: ${{ matrix.model != 'modelless' }} name: Build deezer/spleeter:${{ env.tag }}-${{ matrix.model }} image run: | @@ -56,8 +56,8 @@ jobs: --build-arg MODEL=${{ matrix.model }} \ -t deezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ -f docker/spleeter-model.dockerfile . - echo "::set-env name=tag::${{ env.tag }}-${{ matrix.model }}" - echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}" + echo "tag=${{ env.tag }}-${{ matrix.model }}" >> $GITHUB_ENV + echo "modelarg=-p spleeter:${{ matrix.model }}" >> $GITHUB_ENV # ---------------------------------------------------------------------- # Note: image testing. - name: Test deezer/spleeter:${{ env.tag }} image From aab2997e10da3a25cd8fbb2ba958f98c36d24228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 11:58:12 +0100 Subject: [PATCH 06/15] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20split=20jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 156 ++++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 58 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index daeccee..b8613ac 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,78 +7,118 @@ on: required: true default: "2.1.2" jobs: - build-test-push: + cuda-base: + runs-on: ubuntu-latest + strategy: + matrix: + distribution: [3.6, 3.7, 3.8] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - name: Build CUDA base image + run: | + docker build \ + --build-arg BASE=python:${{ matrix.distribution }} \ + -t deezer/python-cuda-10-1:${{ matrix.distribution }} \ + -f docker/cuda-10-1.dockerfile . + - name: Docker login + run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + - name: Push deezer/python-cuda-10-1:${{ matrix.distribution }} image + run: docker push deezer/python-cuda-10-1:${{ matrix.distribution }} + pip-images: + needs: cuda-base + runs-on: ubuntu-latest + strategy: + matrix: + platform: [cpu, gpu] + distribution: [3.6, 3.7, 3.8] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - if: ${{ matrix.platform == 'cpu' }} + run: | + echo "base=deezer/spleeter:${{ matrix.distribution }}" >> $GITHUB_ENV + echo "image=spleeter" >> $GITHUB_ENV + - if: ${{ matrix.platform == 'gpu' }} + run: | + echo "base=deezer/python-cuda-10-1:${{ matrix.distribution }}" >> $GITHUB_ENV + echo "image=spleeter-gpu" >> $GITHUB_ENV + - name: Build deezer/${{ env.image }}:${{ matrix.distribution }} image + run: | + docker build \ + --build-arg BASE=${{ env.base }} \ + --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \ + -t deezer/${{ env.image }}:${{ matrix.distribution }} \ + -f docker/spleeter.dockerfile . + - name: Test deezer/${{ env.image }}:${{ matrix.distribution }} image + run: | + docker run \ + -v $(pwd):/runtime \ + deezer/${{ env.image }}:${{ matrix.distribution }} \ + separate -i /runtime/audio_example.mp3 -o /tmp + - name: Docker login + run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + - name: Push deezer/${{ env.image }}:${{ matrix.distribution }} image + run: docker push deezer/${{ env.image }}:${{ matrix.distribution }} + conda-images: + needs: cuda-base + runs-on: ubuntu-latest + strategy: + matrix: + platform: [cpu, gpu] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - if: ${{ matrix.platform == 'cpu' }} + - name: Build Conda base image + run: | + docker build -t conda:cpu -f docker/conda.dockerfile . + echo "image=spleeter" >> $GITHUB_ENV + - if: ${{ matrix.platform == 'gpu' }} + - name: Build Conda base image + run: | + docker build --build-arg BASE=deezer/python-cuda-10-1:3.8 -t conda:gpu -f docker/conda.dockerfile . + echo "image=spleeter-gpu" >> $GITHUB_ENV + - name: Build deezer/${{ env.image }}:${{ env.tag }} image + run: | + docker build \ + --build-arg BASE=conda:${{ matrix.platform }} \ + --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \ + -t deezer/${{ env.image }}:conda \ + -f docker/spleeter-conda.dockerfile . + - name: Docker login + run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + - name: Push deezer/${{ env.image }}:conda image + run: docker push deezer/${{ env.image }}:conda + images-with-model: + needs: [pip-images, conda-images] runs-on: ubuntu-latest strategy: matrix: platform: [cpu, gpu] distribution: [3.6, 3.7, 3.8, conda] - model: [modelless, 2stems, 4stems, 5stems] + model: [2stems, 4stems, 5stems] fail-fast: true steps: - uses: actions/checkout@v2 - # ---------------------------------------------------------------------- - # Note: base image building and env setup. - - name: Setup Python distribution - run: | - echo "base=python:${{ matrix.distribution }}" >> $GITHUB_ENV - echo "tag=${{ matrix.distribution }}" >> $GITHUB_ENV - echo "file=spleeter" >> $GITHUB_ENV - - if: ${{ matrix.distribution == 'conda' }} - name: Build Conda base image - run: | - docker build -t python:conda -f docker/conda.dockerfile . - echo "file=spleeter-conda" >> $GITHUB_ENV + - if: ${{ matrix.platform == 'cpu' }} + run: echo "image=spleeter" >> $GITHUB_ENV - if: ${{ matrix.platform == 'gpu' }} - name: Build CUDA base image + run: echo "image=spleeter-gpu" >> $GITHUB_ENV + - name: Build deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image run: | docker build \ - --build-arg BASE=python:${{ matrix.distribution }} \ - -t cuda:${{ matrix.distribution }} \ - -f docker/cuda-10-1.dockerfile . - echo "base=${{ matrix.distribution }}" >> $GITHUB_ENV - echo "tag=${{ matrix.distribution }}-gpu" >> $GITHUB_ENV - # ---------------------------------------------------------------------- - # Note: image building. - - name: Build deezer/spleeter:${{ env.tag }} image - run: | - docker build \ - --build-arg BASE=${{ env.base }} \ - --build-arg SPLEETER_VERSION=${{ github.event.inputs.version }} \ - -t deezer/spleeter:${{ env.tag }} \ - -f docker/${{ env.file }}.dockerfile . - echo "modelargs=" >> $GITHUB_ENV - - if: ${{ matrix.model != 'modelless' }} - name: Build deezer/spleeter:${{ env.tag }}-${{ matrix.model }} image - run: | - docker build \ - --build-arg BASE=deezer/spleeter:${{ env.tag }} \ + --build-arg BASE=deezer/${{ env.image }}:${{ matrix.distribution }} \ --build-arg MODEL=${{ matrix.model }} \ - -t deezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ - -f docker/spleeter-model.dockerfile . - echo "tag=${{ env.tag }}-${{ matrix.model }}" >> $GITHUB_ENV - echo "modelarg=-p spleeter:${{ matrix.model }}" >> $GITHUB_ENV - # ---------------------------------------------------------------------- - # Note: image testing. - - name: Test deezer/spleeter:${{ env.tag }} image + -t deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} \ + -f docker/spleeter-model.dockerfile .dockerfile + - name: Test deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image run: | docker run \ -v $(pwd):/runtime \ - deezer/spleeter:${{ env.tag }} \ - separate -i /runtime/audio_example.mp3 -o /tmp \${{ env.modelarg }} - # ---------------------------------------------------------------------- - # Note: image deploy. + deezer/${{ env.image }}:${{ matrix.distribution }} \ + separate -i /runtime/audio_example.mp3 -o /tmp -p spleeter:${{ matrix.model }} - name: Docker login run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - - name: Push deezer/spleeter:${{ env.tag }} image - run: docker push deezer/spleeter:${{ env.tag }} - - if: ${{ env.tag == 'spleeter:3.8' }} - name: Push deezer/spleeter:latest image - run: | - docker tag deezer/spleeter:3.8 deezer/spleeter:latest - docker push deezer/spleeter:latest - - if: ${{ env.tag == 'spleeter:3.8-gpu' }} - name: Push deezer/spleeter:gpu image - run: | - docker tag deezer/spleeter:3.8-gpu deezer/spleeter:gpu - docker push deezer/spleeter:gpu + - name: Push deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image + run: docker push deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} From 96fcf43348ba239a1e79e5e1513cdd3be13e3dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 12:00:09 +0100 Subject: [PATCH 07/15] =?UTF-8?q?=F0=9F=90=9B=20=20fix=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b8613ac..e6736ef 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -70,12 +70,12 @@ jobs: steps: - uses: actions/checkout@v2 - if: ${{ matrix.platform == 'cpu' }} - - name: Build Conda base image + name: Build Conda base image run: | docker build -t conda:cpu -f docker/conda.dockerfile . echo "image=spleeter" >> $GITHUB_ENV - if: ${{ matrix.platform == 'gpu' }} - - name: Build Conda base image + name: Build Conda base image run: | docker build --build-arg BASE=deezer/python-cuda-10-1:3.8 -t conda:gpu -f docker/conda.dockerfile . echo "image=spleeter-gpu" >> $GITHUB_ENV From a0daeff5e07655700548b508d125ef2628428047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 12:08:42 +0100 Subject: [PATCH 08/15] =?UTF-8?q?=F0=9F=90=9B=20=20with=20pip=20images=20b?= =?UTF-8?q?ase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e6736ef..d53fb2a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@v2 - if: ${{ matrix.platform == 'cpu' }} run: | - echo "base=deezer/spleeter:${{ matrix.distribution }}" >> $GITHUB_ENV + echo "base=python:${{ matrix.distribution }}" >> $GITHUB_ENV echo "image=spleeter" >> $GITHUB_ENV - if: ${{ matrix.platform == 'gpu' }} run: | From 4212ac0945d9d532b48836963d39d2590825d153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 12:19:09 +0100 Subject: [PATCH 09/15] =?UTF-8?q?=E2=9C=A8=20=20fix=20image=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d53fb2a..94281fa 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -55,7 +55,7 @@ jobs: docker run \ -v $(pwd):/runtime \ deezer/${{ env.image }}:${{ matrix.distribution }} \ - separate -i /runtime/audio_example.mp3 -o /tmp + separate -o /tmp /runtime/audio_example.mp3 - name: Docker login run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - name: Push deezer/${{ env.image }}:${{ matrix.distribution }} image @@ -117,7 +117,7 @@ jobs: docker run \ -v $(pwd):/runtime \ deezer/${{ env.image }}:${{ matrix.distribution }} \ - separate -i /runtime/audio_example.mp3 -o /tmp -p spleeter:${{ matrix.model }} + separate -o /tmp -p spleeter:${{ matrix.model }} /runtime/audio_example.mp3 - name: Docker login run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - name: Push deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image From bd4dedf51398220647c162bb081cd2564162475a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 12:45:33 +0100 Subject: [PATCH 10/15] =?UTF-8?q?=F0=9F=90=9B=20=20fix=20model=20image=20c?= =?UTF-8?q?ontext?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 94281fa..e16e370 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -111,7 +111,7 @@ jobs: --build-arg BASE=deezer/${{ env.image }}:${{ matrix.distribution }} \ --build-arg MODEL=${{ matrix.model }} \ -t deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} \ - -f docker/spleeter-model.dockerfile .dockerfile + -f docker/spleeter-model.dockerfile . - name: Test deezer/${{ env.image }}:${{ matrix.distribution }}-${{ matrix.model }} image run: | docker run \ From 6a39ef84196c8b4b08c75c0cac14c0b886247368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 12:46:29 +0100 Subject: [PATCH 11/15] =?UTF-8?q?=F0=9F=90=9B=20=20remove=20automatic=20tr?= =?UTF-8?q?igger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index d87f573..0c206d2 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,8 +1,6 @@ name: pypi on: - push: - branches: - - master + - workflow_dispatch env: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} jobs: @@ -22,4 +20,4 @@ jobs: - name: Deploy to pypi run: | poetry build - poetry publish \ No newline at end of file + poetry publish From 24744e2cacd9231ee3be85c13ce8839031c6f7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 12 Feb 2021 16:19:05 +0100 Subject: [PATCH 12/15] =?UTF-8?q?=F0=9F=93=8C=20=20force=20numpy=201.19?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conda/spleeter/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/spleeter/meta.yaml b/conda/spleeter/meta.yaml index e2a5cb0..d93f8eb 100644 --- a/conda/spleeter/meta.yaml +++ b/conda/spleeter/meta.yaml @@ -25,6 +25,7 @@ requirements: - python {{ python }} - tensorflow ==2.3.0 # [linux] - tensorflow ==2.3.0 # [win] + - numpy <1.20.0 - pandas - ffmpeg-python - norbert From 4ef593dc94d820fb019236cd799d03c858acd7fc Mon Sep 17 00:00:00 2001 From: Romain Hennequin Date: Fri, 12 Feb 2021 21:10:09 +0100 Subject: [PATCH 13/15] Update README.md Added warning for mac users regarding conda install --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index aa4f478..a23524e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -[![Github actions](https://github.com/deezer/spleeter/workflows/pytest/badge.svg)](https://github.com/deezer/spleeter/actions) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) +[![Github actions](https://github.com/deezer/spleeter/workflows/pytest/badge.svg)](https://github.com/deezer/spleeter/actions) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/deezer-research/spleeter)](https://anaconda.org/deezer-research/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/deezer/spleeter)](https://hub.docker.com/deezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) > :warning: [Spleeter 2.1.0](https://pypi.org/project/spleeter/) release introduces some breaking changes, including new CLI option naming for input, and the drop > of dedicated GPU package. Please read [CHANGELOG](CHANGELOG.md) for more details. @@ -52,10 +52,11 @@ wget https://github.com/deezer/spleeter/raw/master/audio_example.mp3 # separate the example audio into two components spleeter separate -p spleeter:2stems -o output audio_example.mp3 ``` +> :warning: for Mac Users, this will work but will install an old version of spleeter. To get the latest version, you need to install **Spleeter** using `pip`. Check the [wiki](https://github.com/deezer/spleeter/wiki/1.-Installation) for details. You should get two separated audio files (`vocals.wav` and `accompaniment.wav`) in the `output/audio_example` folder. -For a detailed documentation, please check the [repository wiki](https://github.com/deezer/spleeter/wiki) +For a detailed documentation, please check the [repository wiki](https://github.com/deezer/spleeter/wiki/1.-Installation) ## Development and Testing From 75583e0222a7e3435695e02f640dd010f0bc902c Mon Sep 17 00:00:00 2001 From: romi1502 Date: Thu, 25 Feb 2021 17:57:40 +0100 Subject: [PATCH 14/15] Replaced default value from None to empty string to avoid tensorflow error --- spleeter/separator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index 0c90ece..dc01a5f 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -307,7 +307,7 @@ class Separator(object): return prediction def separate( - self, waveform: np.ndarray, audio_descriptor: Optional[str] = None + self, waveform: np.ndarray, audio_descriptor: Optional[str] = "" ) -> None: """ Performs separation on a waveform. From af4ea322fe423e2d9e7e6539e2ab63bc43993430 Mon Sep 17 00:00:00 2001 From: romi1502 Date: Fri, 26 Feb 2021 10:26:26 +0100 Subject: [PATCH 15/15] Fixed channels setting in dataset + added dimension check --- spleeter/dataset.py | 24 ++++++++++++++-- tests/test_train.py | 68 ++++++++++++++++++++++++++++----------------- 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/spleeter/dataset.py b/spleeter/dataset.py index 4ee096d..b73e414 100644 --- a/spleeter/dataset.py +++ b/spleeter/dataset.py @@ -241,7 +241,8 @@ class InstrumentDatasetBuilder(object): def filter_shape(self, sample): """ Filter badly shaped sample. """ return check_tensor_shape( - sample[self._spectrogram_key], (self._parent._T, self._parent._F, 2) + sample[self._spectrogram_key], + (self._parent._T, self._parent._F, self._parent._n_channels), ) def reshape_spectrogram(self, sample): @@ -250,7 +251,8 @@ class InstrumentDatasetBuilder(object): sample, **{ self._spectrogram_key: set_tensor_shape( - sample[self._spectrogram_key], (self._parent._T, self._parent._F, 2) + sample[self._spectrogram_key], + (self._parent._T, self._parent._F, self._parent._n_channels), ) }, ) @@ -299,6 +301,7 @@ class DatasetBuilder(object): self._frame_length = audio_params["frame_length"] self._frame_step = audio_params["frame_step"] self._mix_name = audio_params["mix_name"] + self._n_channels = audio_params["n_channels"] self._instruments = [self._mix_name] + audio_params["instrument_list"] self._instrument_builders = None self._chunk_duration = chunk_duration @@ -307,6 +310,21 @@ class DatasetBuilder(object): self._audio_path = audio_path self._random_seed = random_seed + self.check_parameters_compatibility() + + def check_parameters_compatibility(self): + if self._frame_length / 2 + 1 < self._F: + raise ValueError( + "F is too large and must be set to at most frame_length/2+1. Decrease F or increase frame_length to fix." + ) + + if ( + self._chunk_duration * self._sample_rate - self._frame_length + ) / self._frame_step < self._T: + raise ValueError( + "T is too large considering STFT parameters and chunk duratoin. Make sure spectrogram time dimension of chunks is larger than T (for instance reducing T or frame_step or increasing chunk duration)." + ) + def expand_path(self, sample): """ Expands audio paths for the given sample. """ return dict( @@ -368,7 +386,7 @@ class DatasetBuilder(object): }, lambda x: tf.image.random_crop( x, - (self._T, len(self._instruments) * self._F, 2), + (self._T, len(self._instruments) * self._F, self._n_channels), seed=self._random_seed, ), ), diff --git a/tests/test_train.py b/tests/test_train.py index 47ce747..4a5280d 100644 --- a/tests/test_train.py +++ b/tests/test_train.py @@ -52,17 +52,19 @@ TRAIN_CONFIG = { } -def generate_fake_training_dataset(path, instrument_list=['vocals', 'other']): +def generate_fake_training_dataset(path, + instrument_list=['vocals', 'other'], + n_channels=2, + n_songs = 2, + fs = 44100, + duration = 6, + ): """ generates a fake training dataset in path: - generates audio files - generates a csv file describing the dataset """ aa = AudioAdapter.default() - n_songs = 2 - fs = 44100 - duration = 6 - n_channels = 2 rng = np.random.RandomState(seed=0) dataset_df = pd.DataFrame( columns=['mix_path'] + [ @@ -83,26 +85,40 @@ def generate_fake_training_dataset(path, instrument_list=['vocals', 'other']): def test_train(): + with TemporaryDirectory() as path: # generate training dataset - generate_fake_training_dataset(path) - # set training command aruments - runner = CliRunner() - TRAIN_CONFIG['train_csv'] = join(path, 'train', 'train.csv') - TRAIN_CONFIG['validation_csv'] = join(path, 'train', 'train.csv') - TRAIN_CONFIG['model_dir'] = join(path, 'model') - TRAIN_CONFIG['training_cache'] = join(path, 'cache', 'training') - TRAIN_CONFIG['validation_cache'] = join(path, 'cache', 'validation') - with open('useless_config.json', 'w') as stream: - json.dump(TRAIN_CONFIG, stream) - # execute training - result = runner.invoke(spleeter, [ - 'train', - '-p', 'useless_config.json', - '-d', path - ]) - # assert that model checkpoint was created. - assert os.path.exists(join(path, 'model', 'model.ckpt-10.index')) - assert os.path.exists(join(path, 'model', 'checkpoint')) - assert os.path.exists(join(path, 'model', 'model.ckpt-0.meta')) - assert result.exit_code == 0 + for n_channels in [1,2]: + TRAIN_CONFIG["n_channels"] = n_channels + generate_fake_training_dataset(path, + n_channels=n_channels, + fs=TRAIN_CONFIG["sample_rate"] + ) + # set training command arguments + runner = CliRunner() + + model_dir = join(path, f'model_{n_channels}') + train_dir = join(path, f'train') + cache_dir = join(path, f'cache_{n_channels}') + + TRAIN_CONFIG['train_csv'] = join(train_dir, 'train.csv') + TRAIN_CONFIG['validation_csv'] = join(train_dir, 'train.csv') + TRAIN_CONFIG['model_dir'] = model_dir + TRAIN_CONFIG['training_cache'] = join(cache_dir, 'training') + TRAIN_CONFIG['validation_cache'] = join(cache_dir, 'validation') + with open('useless_config.json', 'w') as stream: + json.dump(TRAIN_CONFIG, stream) + + # execute training + result = runner.invoke(spleeter, [ + 'train', + '-p', 'useless_config.json', + '-d', path, + "--verbose" + ]) + + # assert that model checkpoint was created. + assert os.path.exists(join(model_dir, 'model.ckpt-10.index')) + assert os.path.exists(join(model_dir, 'checkpoint')) + assert os.path.exists(join(model_dir, 'model.ckpt-0.meta')) + assert result.exit_code == 0