yuzu/spleeter
1
0
Fork 0
mirror of https://github.com/YuzuZensai/spleeter.git synced 2026-01-06 04:32:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #446 from deezer/github-actions

Browse Source 
Switch to GitHub actions
This commit is contained in:
Félix Voituret
2020-07-08 17:30:29 +02:00
committed by GitHub
parent 6e8f4701b2 9b9b3f3a25
commit a3df9f6d51
7 changed files with 162 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

285
.circleci/config.yml
View File

@@ -1,285 +0,0 @@
version: 2
jobs:
# =======================================================================================
# Python 3.6 testing.
# =======================================================================================
test-3.6:
docker:
- image: python:3.6
working_directory: ~/spleeter
steps:
- checkout
- restore_cache:
key: models-{{ checksum "spleeter/model/__init__.py" }}
- run: apt-get update && apt-get install -y ffmpeg
- run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval
- run: make test
- save_cache:
key: models-{{ checksum "spleeter/model/__init__.py" }}
paths:
- "pretrained_models"
# =======================================================================================
# Python 3.7 testing.
# =======================================================================================
test-3.7:
docker:
- image: python:3.7
working_directory: ~/spleeter
steps:
- checkout
- restore_cache:
key: models-{{ checksum "spleeter/model/__init__.py" }}
- run: apt-get update && apt-get install -y ffmpeg
- run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval
- run: make test
- save_cache:
key: models-{{ checksum "spleeter/model/__init__.py" }}
paths:
- "pretrained_models"
# =======================================================================================
# Source distribution packaging.
# =======================================================================================
sdist:
docker:
- image: python:3
steps:
- checkout
- run: make build
- save_cache:
key: sdist-{{ .Branch }}-{{ checksum "setup.py" }}
paths:
- dist
sdist-gpu:
docker:
- image: python:3
steps:
- checkout
- run: make build-gpu
- save_cache:
key: sdist-{{ .Branch }}-{{ checksum "setup.py" }}
paths:
- dist
# =======================================================================================
# PyPi deployment.
# =======================================================================================
pypi-deploy:
docker:
- image: python:3
steps:
- checkout
- restore_cache:
key: sdist-{{ .Branch }}-{{ checksum "setup.py" }}
- run:
name: upload to PyPi
# TODO: Infer destination regarding of branch.
# - master => production PyPi
# - other => testing PyPi
command: make build deploy
pypi-deploy-gpu:
docker:
- image: python:3
steps:
- checkout
- restore_cache:
key: sdist-{{ .Branch }}-{{ checksum "setup.py" }}
- run:
name: upload to PyPi
# TODO: Infer destination regarding of branch.
# - master => production PyPi
# - other => testing PyPi
command: make build-gpu deploy
# =======================================================================================
# Docker build.
# =======================================================================================
docker-conda-cpu:
docker:
- image: docker:17.05.0-ce-git
steps:
- checkout
- setup_remote_docker
- run: docker build -t conda -f docker/conda.dockerfile .
- run: docker build --build-arg BASE=conda -t researchdeezer/spleeter:conda -f docker/spleeter-conda.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-5stems -f docker/spleeter-model.dockerfile .
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-2stems separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:conda
- run: docker push researchdeezer/spleeter:conda-2stems
- run: docker push researchdeezer/spleeter:conda-4stems
- run: docker push researchdeezer/spleeter:conda-5stems
docker-conda-gpu:
docker:
- image: docker:17.05.0-ce-git
steps:
- checkout
- setup_remote_docker
- run: docker build -t conda -f docker/conda.dockerfile .
- run: docker build --build-arg BASE=conda -t conda-gpu -f docker/cuda-10-0.dockerfile .
- run: docker build --build-arg BASE=conda-gpu --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:conda-gpu -f docker/spleeter-conda.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-gpu-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-gpu-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-gpu-5stems -f docker/spleeter-model.dockerfile .
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:conda-gpu
- run: docker push researchdeezer/spleeter:conda-gpu-2stems
- run: docker push researchdeezer/spleeter:conda-gpu-4stems
- run: docker push researchdeezer/spleeter:conda-gpu-5stems
docker-3.6-cpu:
docker:
- image: docker:17.05.0-ce-git
steps:
- checkout
- setup_remote_docker
- run: docker build --build-arg BASE=python:3.6 -t researchdeezer/spleeter:3.6 -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-5stems -f docker/spleeter-model.dockerfile .
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6 separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-2stems separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:3.6
- run: docker push researchdeezer/spleeter:3.6-2stems
- run: docker push researchdeezer/spleeter:3.6-4stems
- run: docker push researchdeezer/spleeter:3.6-5stems
docker-3.6-gpu:
docker:
- image: docker:17.05.0-ce-git
steps:
- checkout
- setup_remote_docker
- run: docker build --build-arg BASE=python:3.6 -t cuda:3.6 -f docker/cuda-10-0.dockerfile .
- run: docker build --build-arg BASE=cuda:3.6 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.6-gpu -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-gpu-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-gpu-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-gpu-5stems -f docker/spleeter-model.dockerfile .
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:3.6-gpu
- run: docker push researchdeezer/spleeter:3.6-gpu-2stems
- run: docker push researchdeezer/spleeter:3.6-gpu-4stems
- run: docker push researchdeezer/spleeter:3.6-gpu-5stems
docker-3.7-cpu:
docker:
- image: docker:17.05.0-ce-git
steps:
- checkout
- setup_remote_docker
- run: docker build --build-arg BASE=python:3.7 -t researchdeezer/spleeter:3.7 -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-5stems -f docker/spleeter-model.dockerfile .
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7 separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-2stems separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest
- run: docker push researchdeezer/spleeter:latest
- run: docker push researchdeezer/spleeter:3.7
- run: docker push researchdeezer/spleeter:3.7-2stems
- run: docker push researchdeezer/spleeter:3.7-4stems
- run: docker push researchdeezer/spleeter:3.7-5stems
docker-3.7-gpu:
docker:
- image: docker:17.05.0-ce-git
steps:
- checkout
- setup_remote_docker
- run: docker build -t cuda:3.7 -f docker/cuda-10-0.dockerfile .
- run: docker build --build-arg BASE=cuda:3.7 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.7-gpu -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-gpu-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-gpu-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-gpu-5stems -f docker/spleeter-model.dockerfile .
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:3.7-gpu
- run: docker push researchdeezer/spleeter:3.7-gpu-2stems
- run: docker push researchdeezer/spleeter:3.7-gpu-4stems
- run: docker push researchdeezer/spleeter:3.7-gpu-5stems
- run: docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu
- run: docker push researchdeezer/spleeter:gpu
workflows:
version: 2
spleeter-release-pipeline:
jobs:
- test-3.6
- test-3.7
- sdist:
requires:
- test-3.6
- test-3.7
- sdist-gpu:
requires:
- test-3.6
- test-3.7
- pypi-deploy:
filters:
branches:
only:
- master
requires:
- sdist
- pypi-deploy-gpu:
filters:
branches:
only:
- master
requires:
- sdist-gpu
- conda-forge-validation:
type: approval
requires:
- pypi-deploy
- pypi-deploy-gpu
filters:
branches:
only:
- master
- docker-conda-cpu:
requires:
- conda-forge-validation
filters:
branches:
only:
- master
- docker-conda-gpu:
requires:
- conda-forge-validation
filters:
branches:
only:
- master
- docker-3.6-cpu:
requires:
- pypi-deploy
filters:
branches:
only:
- master
- docker-3.6-gpu:
requires:
- pypi-deploy-gpu
filters:
branches:
only:
- master
- docker-3.7-cpu:
requires:
- pypi-deploy
filters:
branches:
only:
- master
- docker-3.7-gpu:
requires:
- pypi-deploy-gpu
filters:
branches:
only:
- master

78
.github/workflows/docker.yml vendored Normal file
View File

@@ -0,0 +1,78 @@
name: docker
on:
- workflow_dispatch
jobs:
build-test-push:
runs-on: ubuntu-latest
strategy:
matrix:
platform: [cpu, gpu]
distribution: [3.6, 3.7, conda]
model: [modelless, 2stems, 4stems, 5stems]
fail-fast: true
steps:
- uses: actions/checkout@v2
# ----------------------------------------------------------------------
# Note: base image building and env setup.
- name: Setup Python distribution
run: |
echo "::set-env name=base::python:${{ matrix.distribution }}"
echo "::set-env name=tag::${{ matrix.distribution }}"
echo "::set-env name=file::spleeter"
- if: ${{ matrix.distribution }} == 'conda'
name: Build Conda base image
run: |
docker build -t python:conda -f docker/conda.dockerfile .
echo "::set-env name=file::spleeter-conda"
- if: ${{ matrix.platform }} == 'gpu'
name: Build CUDA base image
run: |
docker build \
--build-arg BASE=python:${{ matrix.distribution }}
-t cuda:${{ matrix.distribution }} \
-f docker/cuda-10-0.dockerfile .
echo "::set-env name=base::cuda:${{ matrix.distribution }}"
echo "::set-env name=tag::${{ matrix.distribution }}-gpu"
# ----------------------------------------------------------------------
# Note: image building.
- name: Build researchdeezer/spleeter:${{ env.tag }} image
run: |
docker build \
--build-arg BASE=${{ env.base }} \
-t researchdeezer/spleeter:${{ env.tag }} \
-f docker/${{ env.file }}.dockerfile .
echo "::set-env name=modelargs::"
- if: ${{ matrix.model }} != 'modelless'
name: Build researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} image
run: |
docker build \
--build-arg BASE=researchdeezer/spleeter:${{ env.tag }} \
--build-arg MODEL=${{ matrix.model }} \
-t researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} \
-f docker/spleeter-model.dockerfile .
echo "::set-env name=tag::${{ env.tag }}-${{ matrix.model }}"
echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}"
# ----------------------------------------------------------------------
# Note: image testing.
- name: Test researchdeezer/spleeter:${{ env.tag }} image
run: |
docker run \
-v $(pwd):/runtime \
researchdeezer/spleeter:${{ env.tag }} \
separate -i /runtime/audio_example.mp3 -o /tmp \${{ env.modelarg }}
# ----------------------------------------------------------------------
# Note: image deploy.
- name: Docker login
run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
- name: Push researchdeezer/spleeter:${{ env.tag }} image
run: docker push researchdeezer/spleeter:${{ env.tag }}
- if: ${{ env.tag }} == 'spleeter:3.7'
name: Push researchdeezer/spleeter:latest image
run: |
docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest
docker push researchdeezer/spleeter:latest
- if: ${{ env.tag }} == 'spleeter:3.7-gpu'
name: Push researchdeezer/spleeter:gpu image
run: |
docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu
docker push researchdeezer/spleeter:gpu

40
.github/workflows/pypi.yml vendored Normal file
View File

@@ -0,0 +1,40 @@
name: pypi
on:
push:
branches:
- master
env:
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
jobs:
package-and-deploy:
strategy:
matrix:
platform: [cpu, gpu]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/cache@v2
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- uses: actions/cache@v2
with:
path: ${{ env.GITHUB_WORKSPACE }}/dist
key: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }}
restore-keys: |
sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }}
sdist-${{ matrix.platform }}
sdist-
- name: Install dependencies
run: pip install --upgrade pip setuptools twine
- if: ${{ matrix.platform }} == 'cpu'
name: Package CPU distribution
run: make build
- if: ${{ matrix.platform }} == 'gpu'
name: Package GPU distribution
run: make build-gpu
- name: Deploy to pypi
run: make deploy

41
.github/workflows/pytest.yml vendored Normal file
View File

@@ -0,0 +1,41 @@
name: pytest
on:
pull_request:
branches:
- master
jobs:
tests:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.6, 3.7]
steps:
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- uses: actions/cache@v2
id: spleeter-pip-cache
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
restore-keys: |
${{ runner.os }}-pip-
- uses: actions/cache@v2
env:
model-release: 1
id: spleeter-model-cache
with:
path: ${{ env.GITHUB_WORKSPACE }}/pretrained_models
key: models-${{ env.model-release }}
restore-keys: |
models-${{ env.model-release }}
- name: Install dependencies
run: |
sudo apt-get update && sudo apt-get install -y ffmpeg
pip install --upgrade pip setuptools
pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval
python setup.py install
- name: Test with pytest
run: make test

11
Makefile
View File

@@ -8,7 +8,7 @@
FEEDSTOCK = spleeter-feedstock
FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK)
FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml
PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv
PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked
all: clean build test deploy
@@ -27,14 +27,7 @@ build-gpu: clean
python3 setup.py sdist
test:
#$(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);)
# deactivate this until we figure out why it fails on the CI container
#$(PYTEST_CMD) tests/test_eval.py
$(PYTEST_CMD) tests/test_ffmpeg_adapter.py
$(PYTEST_CMD) tests/test_github_model_provider.py
$(PYTEST_CMD) --boxed tests/test_separator.py
$(PYTEST_CMD) -n 2 tests/
deploy:
pip install twine

2
README.md
View File

@@ -1,6 +1,6 @@
<img src="https://github.com/deezer/spleeter/raw/master/images/spleeter_logo.png" height="80" />
[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b)
[![Github actions](https://github.com/deezer/spleeter/workflows/pytest/badge.svg)](https://github.com/deezer/spleeter/actions) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b)
## About

9
requirements.txt
View File

@@ -1,9 +0,0 @@
importlib_resources; python_version<'3.7'
requests
setuptools>=41.0.0
pandas==0.25.1
tensorflow==1.15.2
ffmpeg-python
norbert==0.2.1
librosa==0.7.2
numba==0.48.0