From afce3b95f1e6b2d7dd5680155c759d48f7bcde81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 14:59:50 +0200 Subject: [PATCH 01/16] =?UTF-8?q?=F0=9F=91=B7=20setup=20Github=20actions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 78 ++++++++++++++++++++++++++++++++++++ .github/workflows/pypi.yml | 27 +++++++++++++ .github/workflows/pytest.yml | 23 +++++++++++ Makefile | 2 - 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/pypi.yml create mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..16cd8ab --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,78 @@ +name: docker +on: + - workflow_dispatch +jobs: + build-test-push: + runs-on: ubuntu-latest + strategy: + matrix: + platform: [cpu, gpu] + distribution: [3.6, 3.7, conda] + model: [modelless, 2stems, 4stems, 5stems] + fail-fast: true + steps: + - uses: actions/checkout@v2 + # ---------------------------------------------------------------------- + # Note: base image building and env setup. + - name: Setup Python distribution + run: | + echo "::set-env name=base::python:${{ matrix.distribution }}" + echo "::set-env name=tag::${{ matrix.distribution }}" + echo "::set-env name=file::spleeter" + - if: ${{ matrix.distribution }} == 'conda' + name: Build Conda base image + run: | + docker build -t python:conda -f docker/conda.dockerfile . + echo "::set-env name=file::spleeter-conda" + - if: ${{ matrix.platform }} == 'gpu' + name: Build CUDA base image + run: | + docker build \ + --build-arg BASE=python:${{ matrix.distribution }} + -t cuda:${{ matrix.distribution }} \ + -f docker/cuda-10-0.dockerfile . + echo "::set-env name=base::cuda:${{ matrix.distribution }}" + echo "::set-env name=tag::${{ matrix.distribution }}-gpu" + # ---------------------------------------------------------------------- + # Note: image building. + - name: Build researchdeezer/spleeter:${{ env.tag }} image + run: | + docker build \ + --build-arg BASE=${{ env.base }} \ + -t researchdeezer/spleeter:${{ env.tag }} \ + -f docker/${{ env.file }}.dockerfile . + echo "::set-env name=modelargs::" + - if: ${{ matrix.model }} != 'modelless' + name: Build researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} image + run: | + docker build \ + --build-arg BASE=researchdeezer/spleeter:${{ env.tag }} \ + --build-arg MODEL=${{ matrix.model }} \ + -t researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ + -f docker/spleeter-model.dockerfile . + echo "::set-env name=tag::${{ env.tag }}-${{ matrix.model }}" + echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}" + # ---------------------------------------------------------------------- + # Note: image testing. + - name: Test researchdeezer/spleeter:${{ env.tag }} image + run: | + docker run \ + -v $(pwd):/runtime \ + researchdeezer/spleeter:${{ env.tag }} \ + separate -i /runtime/audio_example.mp3 -o /tmp \${{ env.modelarg }} + # ---------------------------------------------------------------------- + # Note: image deploy. + - name: Docker login + run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + - name: Push researchdeezer/spleeter:${{ env.tag }} image + run: docker push researchdeezer/spleeter:${{ env.tag }} + - if: ${{ env.tag }} == 'spleeter:3.7' + name: Push researchdeezer/spleeter:latest image + run: | + docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest + docker push researchdeezer/spleeter:latest + - if: ${{ env.tag }} == 'spleeter:3.7-gpu' + name: Push researchdeezer/spleeter:gpu image + run: | + docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu + docker push researchdeezer/spleeter:gpu diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..8f3e457 --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,27 @@ +name: pypi +on: + pull_request: + types: + - closed + branches: + - master +env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} +jobs: + deploy-cpu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Package sdist + run: make build + - name: Deploy to pypi + run: make deploy + deploy-gpu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Package sdist + run: make build-gpu + - name: Deploy to pypi + run: make deploy \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..41299e7 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,23 @@ +name: pytest +on: + - push + - pull_request +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + apt-get update && apt-get install -y ffmpeg + pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval + python setup.py install + - name: Test with pytest + run: make test \ No newline at end of file diff --git a/Makefile b/Makefile index 740d6cd..a67a64d 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,6 @@ test: $(PYTEST_CMD) tests/test_ffmpeg_adapter.py $(PYTEST_CMD) tests/test_github_model_provider.py $(PYTEST_CMD) --boxed tests/test_separator.py - - deploy: pip install twine From 75ebb62dd722a892a98aca26608f4c22786da5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:02:40 +0200 Subject: [PATCH 02/16] =?UTF-8?q?=F0=9F=92=9A=20switch=20to=20sudo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 41299e7..52f07f0 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -16,7 +16,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - apt-get update && apt-get install -y ffmpeg + sudo apt-get update && sudo apt-get install -y ffmpeg pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval python setup.py install - name: Test with pytest From 6d12077b451d90ce0d6b3388966389a08ced9b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:24:26 +0200 Subject: [PATCH 03/16] =?UTF-8?q?=F0=9F=91=B7=20add=20pip=20upgrade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 52f07f0..a6e5b42 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,7 +17,8 @@ jobs: - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -y ffmpeg + pip install --upgrade pip setuptools pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - python setup.py install + pip setup.py install --user - name: Test with pytest run: make test \ No newline at end of file From 5182795d3610afbf264c7f402907db031cc6a7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:26:14 +0200 Subject: [PATCH 04/16] =?UTF-8?q?=F0=9F=90=9B=20syntax=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index a6e5b42..49acfee 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,6 +19,6 @@ jobs: sudo apt-get update && sudo apt-get install -y ffmpeg pip install --upgrade pip setuptools pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - pip setup.py install --user + python setup.py install - name: Test with pytest run: make test \ No newline at end of file From 1fdcff31f8ac9b3a0b4d18028000d1bdb3137584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:50:12 +0200 Subject: [PATCH 05/16] =?UTF-8?q?=F0=9F=91=B7=20add=20model=20and=20packag?= =?UTF-8?q?e=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 28 +++++++++++++++++++--------- .github/workflows/pytest.yml | 10 ++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 8f3e457..26a98c8 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -9,19 +9,29 @@ env: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} jobs: - deploy-cpu: + package-and-deploy: + strategy: + matrix: + platform: [cpu, gpu] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Package sdist + - name: Cache sdist packages + uses: actions/cache@v2 + env: + cache-name: cache-sdist-packages + with: + path: dist + key: sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + restore-keys: | + sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + sdist-${{ matrix.platform }} + sdist- + - if: ${{ matrix.platform }} == 'cpu' + name: Package CPU distribution run: make build - - name: Deploy to pypi - run: make deploy - deploy-gpu: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Package sdist + - if: ${{ matrix.platform }} == 'gpu' + name: Package GPU distribution run: make build-gpu - name: Deploy to pypi run: make deploy \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 49acfee..2b9019b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -14,6 +14,16 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Cache spleeter models + uses: actions/cache@v2 + env: + cache-name: cache-spleeter-model + with: + path: pretrained_models + key: models-${{ hashFiles('spleeter/model/__init__.py') }} + restore-keys: | + models-${{ hashFiles('spleeter/model/__init__.py') }} + models- - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -y ffmpeg From 040a9fdcfbda892e4454b3148a585a8eb69a43e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:54:46 +0200 Subject: [PATCH 06/16] =?UTF-8?q?=E2=9C=85=20restore=20old=20test=20suite?= =?UTF-8?q?=20fashion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 285 ------------------------------------------- Makefile | 9 +- 2 files changed, 2 insertions(+), 292 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 30c5cfd..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,285 +0,0 @@ -version: 2 -jobs: - # ======================================================================================= - # Python 3.6 testing. - # ======================================================================================= - test-3.6: - docker: - - image: python:3.6 - working_directory: ~/spleeter - steps: - - checkout - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - - run: make test - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" - # ======================================================================================= - # Python 3.7 testing. - # ======================================================================================= - test-3.7: - docker: - - image: python:3.7 - working_directory: ~/spleeter - steps: - - checkout - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - - run: make test - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" - # ======================================================================================= - # Source distribution packaging. - # ======================================================================================= - sdist: - docker: - - image: python:3 - steps: - - checkout - - run: make build - - save_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - dist - sdist-gpu: - docker: - - image: python:3 - steps: - - checkout - - run: make build-gpu - - save_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - dist - # ======================================================================================= - # PyPi deployment. - # ======================================================================================= - pypi-deploy: - docker: - - image: python:3 - steps: - - checkout - - restore_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - - run: - name: upload to PyPi - # TODO: Infer destination regarding of branch. - # - master => production PyPi - # - other => testing PyPi - command: make build deploy - pypi-deploy-gpu: - docker: - - image: python:3 - steps: - - checkout - - restore_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - - run: - name: upload to PyPi - # TODO: Infer destination regarding of branch. - # - master => production PyPi - # - other => testing PyPi - command: make build-gpu deploy - # ======================================================================================= - # Docker build. - # ======================================================================================= - - docker-conda-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t conda -f docker/conda.dockerfile . - - run: docker build --build-arg BASE=conda -t researchdeezer/spleeter:conda -f docker/spleeter-conda.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:conda - - run: docker push researchdeezer/spleeter:conda-2stems - - run: docker push researchdeezer/spleeter:conda-4stems - - run: docker push researchdeezer/spleeter:conda-5stems - docker-conda-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t conda -f docker/conda.dockerfile . - - run: docker build --build-arg BASE=conda -t conda-gpu -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=conda-gpu --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:conda-gpu -f docker/spleeter-conda.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:conda-gpu - - run: docker push researchdeezer/spleeter:conda-gpu-2stems - - run: docker push researchdeezer/spleeter:conda-gpu-4stems - - run: docker push researchdeezer/spleeter:conda-gpu-5stems - docker-3.6-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.6 -t researchdeezer/spleeter:3.6 -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6 separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.6 - - run: docker push researchdeezer/spleeter:3.6-2stems - - run: docker push researchdeezer/spleeter:3.6-4stems - - run: docker push researchdeezer/spleeter:3.6-5stems - docker-3.6-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.6 -t cuda:3.6 -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=cuda:3.6 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.6-gpu -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.6-gpu - - run: docker push researchdeezer/spleeter:3.6-gpu-2stems - - run: docker push researchdeezer/spleeter:3.6-gpu-4stems - - run: docker push researchdeezer/spleeter:3.6-gpu-5stems - docker-3.7-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.7 -t researchdeezer/spleeter:3.7 -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7 separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest - - run: docker push researchdeezer/spleeter:latest - - run: docker push researchdeezer/spleeter:3.7 - - run: docker push researchdeezer/spleeter:3.7-2stems - - run: docker push researchdeezer/spleeter:3.7-4stems - - run: docker push researchdeezer/spleeter:3.7-5stems - docker-3.7-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t cuda:3.7 -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=cuda:3.7 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.7-gpu -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.7-gpu - - run: docker push researchdeezer/spleeter:3.7-gpu-2stems - - run: docker push researchdeezer/spleeter:3.7-gpu-4stems - - run: docker push researchdeezer/spleeter:3.7-gpu-5stems - - run: docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu - - run: docker push researchdeezer/spleeter:gpu - -workflows: - version: 2 - spleeter-release-pipeline: - jobs: - - test-3.6 - - test-3.7 - - sdist: - requires: - - test-3.6 - - test-3.7 - - sdist-gpu: - requires: - - test-3.6 - - test-3.7 - - pypi-deploy: - filters: - branches: - only: - - master - requires: - - sdist - - pypi-deploy-gpu: - filters: - branches: - only: - - master - requires: - - sdist-gpu - - conda-forge-validation: - type: approval - requires: - - pypi-deploy - - pypi-deploy-gpu - filters: - branches: - only: - - master - - docker-conda-cpu: - requires: - - conda-forge-validation - filters: - branches: - only: - - master - - docker-conda-gpu: - requires: - - conda-forge-validation - filters: - branches: - only: - - master - - docker-3.6-cpu: - requires: - - pypi-deploy - filters: - branches: - only: - - master - - docker-3.6-gpu: - requires: - - pypi-deploy-gpu - filters: - branches: - only: - - master - - docker-3.7-cpu: - requires: - - pypi-deploy - filters: - branches: - only: - - master - - docker-3.7-gpu: - requires: - - pypi-deploy-gpu - filters: - branches: - only: - - master diff --git a/Makefile b/Makefile index a67a64d..4db4e36 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ FEEDSTOCK = spleeter-feedstock FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK) FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml -PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv +PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked all: clean build test deploy @@ -27,12 +27,7 @@ build-gpu: clean python3 setup.py sdist test: - #$(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) - # deactivate this until we figure out why it fails on the CI container - #$(PYTEST_CMD) tests/test_eval.py - $(PYTEST_CMD) tests/test_ffmpeg_adapter.py - $(PYTEST_CMD) tests/test_github_model_provider.py - $(PYTEST_CMD) --boxed tests/test_separator.py + $(PYTEST_CMD) tests/ deploy: pip install twine From 531f0db15e8e5361d5a1bf65dff2b5dfe3bdc9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:00:20 +0200 Subject: [PATCH 07/16] =?UTF-8?q?=F0=9F=91=B7=20add=20pip=20cache=20and=20?= =?UTF-8?q?clean=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 9 +++------ .github/workflows/pytest.yml | 15 +++++++++------ requirements.txt | 9 --------- 3 files changed, 12 insertions(+), 21 deletions(-) delete mode 100644 requirements.txt diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 26a98c8..ad9996f 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -16,15 +16,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache sdist packages - uses: actions/cache@v2 - env: - cache-name: cache-sdist-packages + - uses: actions/cache@v2 with: path: dist - key: sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + key: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} restore-keys: | - sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} sdist-${{ matrix.platform }} sdist- - if: ${{ matrix.platform }} == 'cpu' diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2b9019b..4fd7c19 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -14,15 +14,18 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Cache spleeter models - uses: actions/cache@v2 - env: - cache-name: cache-spleeter-model + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - uses: actions/cache@v2 with: path: pretrained_models - key: models-${{ hashFiles('spleeter/model/__init__.py') }} + key: models-${{ hashFiles('**/setup.py') }} restore-keys: | - models-${{ hashFiles('spleeter/model/__init__.py') }} + models-${{ hashFiles('**/setup.py') }} models- - name: Install dependencies run: | diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 28f7286..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -importlib_resources; python_version<'3.7' -requests -setuptools>=41.0.0 -pandas==0.25.1 -tensorflow==1.15.2 -ffmpeg-python -norbert==0.2.1 -librosa==0.7.2 -numba==0.48.0 \ No newline at end of file From 99ef2b30fbf5fbd32ce8ba997abb4adb610a9a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:06:15 +0200 Subject: [PATCH 08/16] =?UTF-8?q?=F0=9F=93=9D=20update=20CI=20badge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2249c7f..702e0bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) +[![Github actions](https://github.com/deezer/spleeter/workflows/pytest/badge.svg)](https://github.com/deezer/spleeter/actions) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) ## About From d15b417ef5d2d0fcd67fba22fee14aa06e5954a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:30:45 +0200 Subject: [PATCH 09/16] =?UTF-8?q?=F0=9F=91=B7=20update=20pytest=20events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4fd7c19..fe7dff6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,7 +1,5 @@ name: pytest -on: - - push - - pull_request +on: [push, pull_request] jobs: tests: runs-on: ubuntu-latest From bd1d8d4f54c8e06790e5b2366a7a47f09afaa6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:41:51 +0200 Subject: [PATCH 10/16] =?UTF-8?q?=F0=9F=91=B7=20add=20branch=20filter=20fo?= =?UTF-8?q?r=20pytest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index fe7dff6..7895ab6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,5 +1,12 @@ name: pytest -on: [push, pull_request] +on: + push: + branches: + - '**' + - '!master' + pull_request: + branches: + - 'master' jobs: tests: runs-on: ubuntu-latest From 21a742aa1db5398c5bec6eca0c19ca47352b6b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:44:51 +0200 Subject: [PATCH 11/16] =?UTF-8?q?=F0=9F=90=9B=20branch=20ignore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7895ab6..27b175b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,9 +1,8 @@ name: pytest on: push: - branches: - - '**' - - '!master' + branches-ignore: + - 'master' pull_request: branches: - 'master' From d8c9e66e01bcd2ac94d89b60cce799a11811e320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:48:29 +0200 Subject: [PATCH 12/16] =?UTF-8?q?=F0=9F=90=9B=20remove=20pypi=20event?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 4 +--- .github/workflows/pytest.yml | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index ad9996f..015c13f 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,8 +1,6 @@ name: pypi on: - pull_request: - types: - - closed + push: branches: - master env: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 27b175b..1bf1576 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,11 +1,11 @@ name: pytest on: - push: - branches-ignore: - - 'master' pull_request: branches: - - 'master' + - master + push: + branches-ignore: + - master jobs: tests: runs-on: ubuntu-latest From 223e68747cd1e6dc3e448493c980bd45ad7ccd70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:49:42 +0200 Subject: [PATCH 13/16] =?UTF-8?q?=F0=9F=91=B7=20remove=20CI=20on=20push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1bf1576..66ef563 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -3,9 +3,6 @@ on: pull_request: branches: - master - push: - branches-ignore: - - master jobs: tests: runs-on: ubuntu-latest From 6e3b6ef78332a131ab2d3b794f65dd77186145bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:55:40 +0200 Subject: [PATCH 14/16] =?UTF-8?q?=F0=9F=90=9B=20=20add=20setuptools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 015c13f..bd25f52 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -14,6 +14,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - uses: actions/cache@v2 with: path: dist @@ -22,6 +28,8 @@ jobs: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} sdist-${{ matrix.platform }} sdist- + - name: Install dependencies + run: pip install --upgrade pip setuptools twine - if: ${{ matrix.platform }} == 'cpu' name: Package CPU distribution run: make build From 02fab48cfb7978b2dd76842ab25906b50cf3ce08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:13:37 +0200 Subject: [PATCH 15/16] =?UTF-8?q?=E2=9C=A8=20add=202=20CPU=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4db4e36..26129d3 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ build-gpu: clean python3 setup.py sdist test: - $(PYTEST_CMD) tests/ + $(PYTEST_CMD) -n 2 tests/ deploy: pip install twine From 9b9b3f3a252fef1290c2789b5c12801766cda0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:19:10 +0200 Subject: [PATCH 16/16] =?UTF-8?q?=F0=9F=90=9B=20caching=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 2 +- .github/workflows/pytest.yml | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index bd25f52..473f74c 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -22,7 +22,7 @@ jobs: ${{ runner.os }}-pip- - uses: actions/cache@v2 with: - path: dist + path: ${{ env.GITHUB_WORKSPACE }}/dist key: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} restore-keys: | sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 66ef563..d4b2965 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -16,18 +16,21 @@ jobs: with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 + id: spleeter-pip-cache with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- - uses: actions/cache@v2 + env: + model-release: 1 + id: spleeter-model-cache with: - path: pretrained_models - key: models-${{ hashFiles('**/setup.py') }} + path: ${{ env.GITHUB_WORKSPACE }}/pretrained_models + key: models-${{ env.model-release }} restore-keys: | - models-${{ hashFiles('**/setup.py') }} - models- + models-${{ env.model-release }} - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -y ffmpeg