diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 30c5cfd..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,285 +0,0 @@ -version: 2 -jobs: - # ======================================================================================= - # Python 3.6 testing. - # ======================================================================================= - test-3.6: - docker: - - image: python:3.6 - working_directory: ~/spleeter - steps: - - checkout - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - - run: make test - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" - # ======================================================================================= - # Python 3.7 testing. - # ======================================================================================= - test-3.7: - docker: - - image: python:3.7 - working_directory: ~/spleeter - steps: - - checkout - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - - run: make test - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" - # ======================================================================================= - # Source distribution packaging. - # ======================================================================================= - sdist: - docker: - - image: python:3 - steps: - - checkout - - run: make build - - save_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - dist - sdist-gpu: - docker: - - image: python:3 - steps: - - checkout - - run: make build-gpu - - save_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - dist - # ======================================================================================= - # PyPi deployment. - # ======================================================================================= - pypi-deploy: - docker: - - image: python:3 - steps: - - checkout - - restore_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - - run: - name: upload to PyPi - # TODO: Infer destination regarding of branch. - # - master => production PyPi - # - other => testing PyPi - command: make build deploy - pypi-deploy-gpu: - docker: - - image: python:3 - steps: - - checkout - - restore_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - - run: - name: upload to PyPi - # TODO: Infer destination regarding of branch. - # - master => production PyPi - # - other => testing PyPi - command: make build-gpu deploy - # ======================================================================================= - # Docker build. - # ======================================================================================= - - docker-conda-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t conda -f docker/conda.dockerfile . - - run: docker build --build-arg BASE=conda -t researchdeezer/spleeter:conda -f docker/spleeter-conda.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:conda - - run: docker push researchdeezer/spleeter:conda-2stems - - run: docker push researchdeezer/spleeter:conda-4stems - - run: docker push researchdeezer/spleeter:conda-5stems - docker-conda-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t conda -f docker/conda.dockerfile . - - run: docker build --build-arg BASE=conda -t conda-gpu -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=conda-gpu --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:conda-gpu -f docker/spleeter-conda.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:conda-gpu - - run: docker push researchdeezer/spleeter:conda-gpu-2stems - - run: docker push researchdeezer/spleeter:conda-gpu-4stems - - run: docker push researchdeezer/spleeter:conda-gpu-5stems - docker-3.6-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.6 -t researchdeezer/spleeter:3.6 -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6 separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.6 - - run: docker push researchdeezer/spleeter:3.6-2stems - - run: docker push researchdeezer/spleeter:3.6-4stems - - run: docker push researchdeezer/spleeter:3.6-5stems - docker-3.6-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.6 -t cuda:3.6 -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=cuda:3.6 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.6-gpu -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.6-gpu - - run: docker push researchdeezer/spleeter:3.6-gpu-2stems - - run: docker push researchdeezer/spleeter:3.6-gpu-4stems - - run: docker push researchdeezer/spleeter:3.6-gpu-5stems - docker-3.7-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.7 -t researchdeezer/spleeter:3.7 -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7 separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest - - run: docker push researchdeezer/spleeter:latest - - run: docker push researchdeezer/spleeter:3.7 - - run: docker push researchdeezer/spleeter:3.7-2stems - - run: docker push researchdeezer/spleeter:3.7-4stems - - run: docker push researchdeezer/spleeter:3.7-5stems - docker-3.7-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t cuda:3.7 -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=cuda:3.7 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.7-gpu -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.7-gpu - - run: docker push researchdeezer/spleeter:3.7-gpu-2stems - - run: docker push researchdeezer/spleeter:3.7-gpu-4stems - - run: docker push researchdeezer/spleeter:3.7-gpu-5stems - - run: docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu - - run: docker push researchdeezer/spleeter:gpu - -workflows: - version: 2 - spleeter-release-pipeline: - jobs: - - test-3.6 - - test-3.7 - - sdist: - requires: - - test-3.6 - - test-3.7 - - sdist-gpu: - requires: - - test-3.6 - - test-3.7 - - pypi-deploy: - filters: - branches: - only: - - master - requires: - - sdist - - pypi-deploy-gpu: - filters: - branches: - only: - - master - requires: - - sdist-gpu - - conda-forge-validation: - type: approval - requires: - - pypi-deploy - - pypi-deploy-gpu - filters: - branches: - only: - - master - - docker-conda-cpu: - requires: - - conda-forge-validation - filters: - branches: - only: - - master - - docker-conda-gpu: - requires: - - conda-forge-validation - filters: - branches: - only: - - master - - docker-3.6-cpu: - requires: - - pypi-deploy - filters: - branches: - only: - - master - - docker-3.6-gpu: - requires: - - pypi-deploy-gpu - filters: - branches: - only: - - master - - docker-3.7-cpu: - requires: - - pypi-deploy - filters: - branches: - only: - - master - - docker-3.7-gpu: - requires: - - pypi-deploy-gpu - filters: - branches: - only: - - master diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..16cd8ab --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,78 @@ +name: docker +on: + - workflow_dispatch +jobs: + build-test-push: + runs-on: ubuntu-latest + strategy: + matrix: + platform: [cpu, gpu] + distribution: [3.6, 3.7, conda] + model: [modelless, 2stems, 4stems, 5stems] + fail-fast: true + steps: + - uses: actions/checkout@v2 + # ---------------------------------------------------------------------- + # Note: base image building and env setup. + - name: Setup Python distribution + run: | + echo "::set-env name=base::python:${{ matrix.distribution }}" + echo "::set-env name=tag::${{ matrix.distribution }}" + echo "::set-env name=file::spleeter" + - if: ${{ matrix.distribution }} == 'conda' + name: Build Conda base image + run: | + docker build -t python:conda -f docker/conda.dockerfile . + echo "::set-env name=file::spleeter-conda" + - if: ${{ matrix.platform }} == 'gpu' + name: Build CUDA base image + run: | + docker build \ + --build-arg BASE=python:${{ matrix.distribution }} + -t cuda:${{ matrix.distribution }} \ + -f docker/cuda-10-0.dockerfile . + echo "::set-env name=base::cuda:${{ matrix.distribution }}" + echo "::set-env name=tag::${{ matrix.distribution }}-gpu" + # ---------------------------------------------------------------------- + # Note: image building. + - name: Build researchdeezer/spleeter:${{ env.tag }} image + run: | + docker build \ + --build-arg BASE=${{ env.base }} \ + -t researchdeezer/spleeter:${{ env.tag }} \ + -f docker/${{ env.file }}.dockerfile . + echo "::set-env name=modelargs::" + - if: ${{ matrix.model }} != 'modelless' + name: Build researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} image + run: | + docker build \ + --build-arg BASE=researchdeezer/spleeter:${{ env.tag }} \ + --build-arg MODEL=${{ matrix.model }} \ + -t researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ + -f docker/spleeter-model.dockerfile . + echo "::set-env name=tag::${{ env.tag }}-${{ matrix.model }}" + echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}" + # ---------------------------------------------------------------------- + # Note: image testing. + - name: Test researchdeezer/spleeter:${{ env.tag }} image + run: | + docker run \ + -v $(pwd):/runtime \ + researchdeezer/spleeter:${{ env.tag }} \ + separate -i /runtime/audio_example.mp3 -o /tmp \${{ env.modelarg }} + # ---------------------------------------------------------------------- + # Note: image deploy. + - name: Docker login + run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + - name: Push researchdeezer/spleeter:${{ env.tag }} image + run: docker push researchdeezer/spleeter:${{ env.tag }} + - if: ${{ env.tag }} == 'spleeter:3.7' + name: Push researchdeezer/spleeter:latest image + run: | + docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest + docker push researchdeezer/spleeter:latest + - if: ${{ env.tag }} == 'spleeter:3.7-gpu' + name: Push researchdeezer/spleeter:gpu image + run: | + docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu + docker push researchdeezer/spleeter:gpu diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..473f74c --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,40 @@ +name: pypi +on: + push: + branches: + - master +env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} +jobs: + package-and-deploy: + strategy: + matrix: + platform: [cpu, gpu] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - uses: actions/cache@v2 + with: + path: ${{ env.GITHUB_WORKSPACE }}/dist + key: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} + restore-keys: | + sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} + sdist-${{ matrix.platform }} + sdist- + - name: Install dependencies + run: pip install --upgrade pip setuptools twine + - if: ${{ matrix.platform }} == 'cpu' + name: Package CPU distribution + run: make build + - if: ${{ matrix.platform }} == 'gpu' + name: Package GPU distribution + run: make build-gpu + - name: Deploy to pypi + run: make deploy \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..d4b2965 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,41 @@ +name: pytest +on: + pull_request: + branches: + - master +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/cache@v2 + id: spleeter-pip-cache + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - uses: actions/cache@v2 + env: + model-release: 1 + id: spleeter-model-cache + with: + path: ${{ env.GITHUB_WORKSPACE }}/pretrained_models + key: models-${{ env.model-release }} + restore-keys: | + models-${{ env.model-release }} + - name: Install dependencies + run: | + sudo apt-get update && sudo apt-get install -y ffmpeg + pip install --upgrade pip setuptools + pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval + python setup.py install + - name: Test with pytest + run: make test \ No newline at end of file diff --git a/Makefile b/Makefile index 740d6cd..26129d3 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ FEEDSTOCK = spleeter-feedstock FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK) FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml -PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv +PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked all: clean build test deploy @@ -27,14 +27,7 @@ build-gpu: clean python3 setup.py sdist test: - #$(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) - # deactivate this until we figure out why it fails on the CI container - #$(PYTEST_CMD) tests/test_eval.py - $(PYTEST_CMD) tests/test_ffmpeg_adapter.py - $(PYTEST_CMD) tests/test_github_model_provider.py - $(PYTEST_CMD) --boxed tests/test_separator.py - - + $(PYTEST_CMD) -n 2 tests/ deploy: pip install twine diff --git a/README.md b/README.md index 2249c7f..702e0bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) +[![Github actions](https://github.com/deezer/spleeter/workflows/pytest/badge.svg)](https://github.com/deezer/spleeter/actions) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) ## About diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 28f7286..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -importlib_resources; python_version<'3.7' -requests -setuptools>=41.0.0 -pandas==0.25.1 -tensorflow==1.15.2 -ffmpeg-python -norbert==0.2.1 -librosa==0.7.2 -numba==0.48.0 \ No newline at end of file