Merge pull request #120 from deezer/1.4.4

1.4.4

.circleci/config.yml

@@ -1,40 +1,207 @@
 version: 2
 jobs:
-  test:
+  # =======================================================================================
+  # Python 3.6 testing.
+  # =======================================================================================
+  test-3.6:
+    docker:
+      - image: python:3.6
+    working_directory: ~/spleeter
+    steps:
+      - checkout
+      - restore_cache:
+          key: models-{{ checksum "spleeter/model/__init__.py" }}
+      - run:
+          name: install ffmpeg
+          command: apt-get update && apt-get install -y ffmpeg
+      - run:
+          name: install python dependencies
+          command: pip install -r requirements.txt && pip install pytest pytest-xdist
+      - run:
+          name: run tests
+          command: make test
+      - save_cache:
+          key: models-{{ checksum "spleeter/model/__init__.py" }}
+          paths:
+            - "pretrained_models"
+  # =======================================================================================
+  # Python 3.7 testing.
+  # =======================================================================================
+  test-3.7:
     docker:
       - image: python:3.7
     working_directory: ~/spleeter
     steps:
       - checkout
+      - restore_cache:
+          key: models-{{ checksum "spleeter/model/__init__.py" }}
       - run:
           name: install ffmpeg
           command: apt-get update && apt-get install -y ffmpeg
       - run:
-          name: install spleeter
+          name: install python dependencies
-          command: pip install .
+          command: pip install -r requirements.txt && pip install pytest pytest-xdist
       - run:
-          name: test separation
+          name: run tests
-          command: spleeter separate -i audio_example.mp3 -o .
+          command: make test
-  upload:
+      - save_cache:
+          key: models-{{ checksum "spleeter/model/__init__.py" }}
+          paths:
+            - "pretrained_models"
+  # =======================================================================================
+  # Source distribution packaging.
+  # =======================================================================================
+  sdist:
     docker:
       - image: python:3
     steps:
       - checkout
       - run:  
-          name: package
+          name: package source distribution
-          command: python setup.py sdist
+          command: make build
+      - save_cache:
+          key: sdist-{{ .Branch }}-{{ checksum "setup.py" }}
+          paths:
+            - dist
+  # =======================================================================================
+  # PyPi deployment.
+  # =======================================================================================
+  pypi-deploy:
+    docker:
+      - image: python:3
+    steps:
+      - checkout
+      - restore_cache:
+          key: sdist-{{ .Branch }}-{{ checksum "setup.py" }}
       - run:
           name: upload to PyPi
-          command: pip install twine && twine upload dist/*
+          # TODO: Infer destination regarding of branch.
+          #       - master => production PyPi
+          #       - other => testing PyPi
+          command: make deploy
+  # =======================================================================================
+  # Conda distribution.
+  # =======================================================================================
+  conda-forge-deploy:
+    docker:
+      - image: python:3
+    steps:
+      - run:
+          name: install dependencies
+          command: apt-get update && apt-get install -y git openssl hub
+      - run:
+          name: checkout feedstock
+          command: make feedstock
+  # =======================================================================================
+  # Docker build.
+  # =======================================================================================
+  docker-conda-cpu:
+    docker:
+      - image: docker:17.05.0-ce-git
+    steps:
+      - checkout
+      - run: docker build -t researchdeezer/spleeter:conda -f docker/cpu/conda.dockerfile .
+      - run: docker build -t researchdeezer/spleeter:conda-2stems -f docker/cpu/conda-2stems.dockerfile .
+      - run: docker build -t researchdeezer/spleeter:conda-4stems -f docker/cpu/conda-2stems.dockerfile .
+      - run: docker build -t researchdeezer/spleeter:conda-5stems -f docker/cpu/conda-2stems.dockerfile .
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-2stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-4stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-5stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
+      - run: docker push researchdeezer/spleeter:conda
+      - run: docker push researchdeezer/spleeter:conda-2stems
+      - run: docker push researchdeezer/spleeter:conda-4stems
+      - run: docker push researchdeezer/spleeter:conda-5stems
+  docker-3.6-cpu:
+    docker:
+      - image: docker:17.05.0-ce-git
+    steps:
+      - checkout
+      - run: docker build -t researchdeezer/spleeter:3.6 -f docker/cpu/python-3.6.dockerfile .
+      - run: docker build -t researchdeezer/spleeter:3.6-2stems -f docker/cpu/python-3.6-2stems.dockerfile .
+      - run: docker build -t researchdeezer/spleeter:3.6-4stems -f docker/cpu/python-3.6-4stems.dockerfile .
+      - run: docker build -t researchdeezer/spleeter:3.6-5stems -f docker/cpu/python-3.6-5stems.dockerfile .
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6 separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-2stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-4stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-5stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
+      - run: docker push researchdeezer/spleeter:3.6
+      - run: docker push researchdeezer/spleeter:3.6-2stems
+      - run: docker push researchdeezer/spleeter:3.6-4stems
+      - run: docker push researchdeezer/spleeter:3.6-5stems
+  docker-3.7-cpu:
+    docker:
+      - image: docker:17.05.0-ce-git
+    steps:
+      - checkout
+      - run: docker build -t spleeter:3.7 -f docker/cpu/python-3.7.dockerfile .
+      - run: docker build -t spleeter:3.7-2stems -f docker/cpu/python-3.7-2stems.dockerfile .
+      - run: docker build -t spleeter:3.7-4stems -f docker/cpu/python-3.7-4stems.dockerfile .
+      - run: docker build -t spleeter:3.7-5stems -f docker/cpu/python-3.7-5stems.dockerfile .
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7 separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-2stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-5stems separate -i /runtime/audio_example.mp3 -o /tmp
+      - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
+      - run: docker tags researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest
+      - run: docker push researchdeezer/spleeter:latest
+      - run: docker push researchdeezer/spleeter:3.7
+      - run: docker push researchdeezer/spleeter:3.7-2stems
+      - run: docker push researchdeezer/spleeter:3.7-4stems
+      - run: docker push researchdeezer/spleeter:3.7-5stems
 workflows:
   version: 2
-  test-and-deploy:
+  spleeter-release-pipeline:
     jobs:
-      - test
+      - test-3.6
-      - upload:
+      - test-3.7
+      - sdist:
+          requires:
+            - test-3.6
+            - test-3.7
+      - pypi-deploy:
           filters:
             branches:
               only:
                 - master
           requires:
-            - test
+            - sdist
+      - conda-forge-deploy:
+          filters:
+            branches:
+              only:
+                - master
+          requires:
+            - pypi-deploy
+      - hold:
+          type: approval
+          requires:
+            - pypi-deploy
+            - conda-forge-deploy
+          filters:
+            branches:
+              only:
+                - master
+      - docker-conda-cpu:
+          requires:
+            - hold
+          filters:
+            branches:
+              only:
+                - master
+      - docker-3.6-cpu:
+          requires:
+            - hold
+          filters:
+            branches:
+              only:
+                - master
+      - docker-3.7-cpu:
+          requires:
+            - hold
+          filters:
+            branches:
+              only:
+                - master

.gitignore

@@ -109,4 +109,5 @@ __pycache__
 
 pretrained_models
 docs/build
-.vscode
+.vscode
+spleeter-feedstock/

Makefile

@@ -1,30 +1,40 @@
 # =======================================================
-# Build script for distribution packaging.
+# Library lifecycle management.
 #
 # @author Deezer Research <research@deezer.com>
 # @licence MIT Licence
 # =======================================================
 
+FFEDSTOCK = spleeter-feedstock
+FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK)
+FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml
+
+all: clean build test deploy
+
 clean:
 	rm -Rf *.egg-info
 	rm -Rf dist
 
 build:
-	@echo "=== Build CPU bdist package" 
+	python3 setup.py sdist
-	@python3 setup.py sdist
-	@echo "=== CPU version checksum"
-	@openssl sha256 dist/*.tar.gz
 
-build-gpu:
+test:
-	@echo "=== Build GPU bdist package" 
+	pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked
-	@python3 setup.py sdist --target gpu
-	@echo "=== GPU version checksum"
-	@openssl sha256 dist/*.tar.gz
 
-upload:
+feedstock: build
-	twine upload dist/*
+	$(eval VERSION = $(shell grep 'project_version = ' setup.py | cut -d' ' -f3 | sed "s/'//g"))
+	$(eval CHECKSUM = $(shell openssl sha256 dist/spleeter-$(VERSION).tar.gz | cut -d' ' -f2))
+	git clone $(FEEDSTOCK_REPOSITORY)
+	sed 's/{% set version = "[0-9]*\.[0-9]*\.[0-9]*" %}/{% set version = "$(VERSION)" %}/g' $(FEEDSTOCK_RECIPE)
+	sed 's/sha256: [0-9a-z]*/sha: $(CHECKSUM)/g' $(FEEDSTOCK_RECIPE)
+	git config credential.helper 'cache --timeout=120'
+	git config user.email "research@deezer.com"
+	git config user.name "spleeter-ci"
+	git add recipe/spleeter/meta.yaml
+	git commit --allow-empty -m "feat: update spleeter version from CI"
+	git push -q https://$$FEEDSTOCK_TOKEN@github.com/deezer/$(FEEDSTOCK)
+	hub pull-request -m "Update spleeter version to $(VERSION)"
 
-test-upload:
+deploy: pip-dependencies
-	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+	pip install twine
-
+	twine upload dist/*
-all: clean build build-gpu upload

README.md

@@ -1,6 +1,8 @@
 <img src="https://github.com/deezer/spleeter/raw/master/images/spleeter_logo.png" height="80" />
 
-[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb)
+[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community)
+
+
 
 ## About
 
@@ -28,8 +30,7 @@ environment to start separating audio file as follows:
 
 ```bash
 git clone https://github.com/Deezer/spleeter
-conda env create -f spleeter/conda/spleeter-cpu.yaml
+conda install -c conda-forge spleeter
-conda activate spleeter-cpu
 spleeter separate -i spleeter/audio_example.mp3 -p spleeter:2stems -o output
 ```
 You should get two separated audio files (`vocals.wav` and `accompaniment.wav`)

docker/cpu.Dockerfile

@@ -1,24 +0,0 @@
-FROM continuumio/miniconda3:4.7.10
-
-# install tensorflow
-RUN conda install -y tensorflow==1.14.0
-
-# install ffmpeg for audio loading/writing
-RUN conda  install -y -c conda-forge ffmpeg
-
-# install extra python libraries
-RUN conda install -y -c anaconda pandas==0.25.1
-RUN conda install -y -c conda-forge libsndfile
-
-# install ipython
-RUN conda install -y ipython
-
-WORKDIR /workspace/
-COPY ./ spleeter/
-
-RUN mkdir /cache/
-
-WORKDIR /workspace/spleeter
-RUN pip install .
-
-ENTRYPOINT ["python", "-m", "spleeter"]

docker/cpu/conda-2stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:conda
+FROM researchdeezer/spleeter:conda
 
 RUN mkdir -p /model/2stems \
     && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \
-    && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/
+    && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ \
+    && touch /model/5stems/.probe

docker/cpu/conda-4stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:conda
+FROM researchdeezer/spleeter:conda
 
 RUN mkdir -p /model/4stems \
     && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \
-    && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/
+    && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ \
+    && touch /model/5stems/.probe

docker/cpu/conda-5stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:conda
+FROM researchdeezer/spleeter:conda
 
 RUN mkdir -p /model/5stems \
     && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \
-    && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/
+    && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ \
+    && touch /model/5stems/.probe

docker/cpu/conda.dockerfile

@@ -1,12 +1,9 @@
 FROM continuumio/miniconda3:4.7.10
 
-RUN conda install -y ipython \
+RUN conda install -y -c conda-forge musdb
-    && conda install -y tensorflow==1.14.0 \
+# RUN conda install -y -c conda-forge museval
-    && conda install -y -c conda-forge ffmpeg \
+RUN conda install -y -c conda-forge spleeter=1.4.4
-    && conda install -y -c conda-forge libsndfile \
-    && conda install -y -c anaconda pandas==0.25.1 \
 RUN mkdir -p /model
 ENV MODEL_PATH /model
-RUN pip install spleeter
 
 ENTRYPOINT ["spleeter"]

docker/cpu/python-3.6-2stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:3.6
+FROM researchdeezer/spleeter:3.6
 
 RUN mkdir -p /model/2stems \
     && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \
-    && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/
+    && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ \
+    && touch /model/5stems/.probe

docker/cpu/python-3.6-4stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:3.6
+FROM researchdeezer/spleeter:3.6
 
 RUN mkdir -p /model/4stems \
     && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \
-    && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/
+    && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ \
+    && touch /model/5stems/.probe

docker/cpu/python-3.6-5stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:3.6
+FROM researchdeezer/spleeter:3.6
 
 RUN mkdir -p /model/5stems \
     && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \
-    && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/
+    && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ \
+    && touch /model/5stems/.probe

docker/cpu/python-3.6.dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.6
 
-RUN apt-get update && apt-get install -y ffmpeg libsndfile
+RUN apt-get update && apt-get install -y ffmpeg libsndfile1
-RUN pip install spleeter
+RUN pip install musdb museval
+RUN pip install spleeter==1.4.4
 RUN mkdir -p /model
 ENV MODEL_PATH /model
 ENTRYPOINT ["spleeter"]

docker/cpu/python-3.7-2stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:3.7
+FROM researchdeezer/spleeter:3.7
 
 RUN mkdir -p /model/2stems \
     && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \
-    && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/
+    && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ \
+    && touch /model/5stems/.probe

docker/cpu/python-3.7-4stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:3.7
+FROM researchdeezer/spleeter:3.7
 
 RUN mkdir -p /model/4stems \
     && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \
-    && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/
+    && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ \
+    && touch /model/5stems/.probe

docker/cpu/python-3.7-5stems.dockerfile

@@ -1,5 +1,6 @@
-FROM deezer/spleeter:3.7
+FROM researchdeezer/spleeter:3.7
 
 RUN mkdir -p /model/5stems \
     && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \
-    && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/
+    && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ \
+    && touch /model/5stems/.probe

docker/cpu/python-3.7.dockerfile

@@ -1,7 +1,8 @@
 FROM python:3.7
 
-RUN apt-get update && apt-get install -y ffmpeg libsndfile
+RUN apt-get update && apt-get install -y ffmpeg libsndfile1
-RUN pip install spleeter
+RUN pip install musdb museval
+RUN pip install spleeter==1.4.4
 RUN mkdir -p /model
 ENV MODEL_PATH /model
 ENTRYPOINT ["spleeter"]

docker/gpu.Dockerfile

@@ -1,35 +0,0 @@
-FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
-
-# set work directory
-WORKDIR /workspace
-
-# install anaconda
-ENV PATH /opt/conda/bin:$PATH
-COPY docker/install_miniconda.sh .
-RUN bash ./install_miniconda.sh && rm install_miniconda.sh
-
-RUN conda update -n base -c defaults conda
-
-# install tensorflow for GPU
-RUN conda install -y tensorflow-gpu==1.14.0
-
-# install ffmpeg for audio loading/writing
-RUN conda  install -y -c conda-forge ffmpeg
-
-# install extra libs
-RUN conda install -y -c anaconda pandas==0.25.1
-RUN conda install -y -c conda-forge libsndfile
-
-# install ipython
-RUN conda install -y ipython
-
-RUN mkdir /cache/
-
-# clone inside image github repository
-COPY ./ spleeter/
-
-WORKDIR /workspace/spleeter
-RUN pip install .
-
-
-ENTRYPOINT ["python", "-m", "spleeter"]

docker/gpu/conda-gpu-2-stems.dockerfile

@@ -1,4 +1,4 @@
-FROM deezer/spleeter:conda-gpu
+FROM researchdeezer/spleeter:conda-gpu
 
 RUN mkdir -p /model/2stems \
     && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \

docker/gpu/conda-gpu-4-stems.dockerfile

@@ -1,4 +1,4 @@
-FROM deezer/spleeter:conda-gpu
+FROM researchdeezer/spleeter:conda-gpu
 
 RUN mkdir -p /model/4stems \
     && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \

docker/gpu/conda-gpu-5-stems.dockerfile

@@ -1,4 +1,4 @@
-FROM deezer/spleeter:conda-gpu
+FROM researchdeezer/spleeter:conda-gpu
 
 RUN mkdir -p /model/5stems \
     && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \

docker/gpu/conda-gpu.dockerfile

@@ -1,12 +1,22 @@
-FROM continuumio/miniconda3:4.7.10
+FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04
 
-RUN conda install -y ipython \
+RUN apt-get update --fix-missing \
+    && apt-get install -y wget bzip2 ca-certificates curl git \
+    && apt-get clean && \
+    && rm -rf /var/lib/apt/lists/* \
+    && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh \
+    && /bin/bash ~/miniconda.sh -b -p /opt/conda \
+    && rm ~/miniconda.sh \
+    && /opt/conda/bin/conda clean -tipsy \
+    && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
+    && echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc \
+    && echo "conda activate base" >> ~/.bashrc
+
+RUN conda install -y cudatoolkit=9.0 \
     && conda install -y tensorflow-gpu==1.14.0 \
-    && conda install -y -c conda-forge ffmpeg \
+    && conda install -y -c conda-forge musdb
-    && conda install -y -c conda-forge libsndfile \
+# RUN conda install -y -c conda-forge museval
-    && conda install -y -c anaconda pandas==0.25.1 \
+# Note: switch to spleeter GPU once published.
-RUN mkdir -p /model
+RUN conda install -y -c conda-forge spleeter=1.4.4
-ENV MODEL_PATH /model
-RUN pip install spleeter
 
 ENTRYPOINT ["spleeter"]

docker/install_miniconda.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-apt-get update --fix-missing && \
-apt-get install -y wget bzip2 ca-certificates curl git && \
-apt-get clean && \
-rm -rf /var/lib/apt/lists/*
-
-wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \
-/bin/bash ~/miniconda.sh -b -p /opt/conda && \
-rm ~/miniconda.sh && \
-/opt/conda/bin/conda clean -tipsy && \
-ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-echo "conda activate base" >> ~/.bashrc

requirements.txt

@@ -0,0 +1,7 @@
+importlib_resources; python_version<'3.7'
+requests
+setuptools>=41.0.0
+pandas==0.25.1
+tensorflow==1.14.0
+ffmpeg-python
+norbert==0.2.1

setup.py

@@ -14,7 +14,7 @@ __license__ = 'MIT License'
 
 # Default project values.
 project_name = 'spleeter'
-project_version = '1.4.3'
+project_version = '1.4.4'
 device_target = 'cpu'
 tensorflow_dependency = 'tensorflow'
 tensorflow_version = '1.14.0'
@@ -51,13 +51,13 @@ setup(
     license='MIT License',
     packages=[
         'spleeter',
+        'spleeter.audio',
         'spleeter.commands',
         'spleeter.model',
         'spleeter.model.functions',
         'spleeter.model.provider',
         'spleeter.resources',
         'spleeter.utils',
-        'spleeter.utils.audio',
     ],
     package_data={'spleeter.resources': ['*.json']},
     python_requires='>=3.6, <3.8',

spleeter/__init__.py

@@ -16,3 +16,9 @@
 __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'
 __license__ = 'MIT License'
+
+
+class SpleeterError(Exception):
+    """ Custom exception for Spleeter related error. """
+
+    pass

spleeter/__main__.py

@@ -10,9 +10,13 @@
 import sys
 import warnings
 
+from . import SpleeterError
 from .commands import create_argument_parser
 from .utils.configuration import load_configuration
-from .utils.logging import enable_logging, enable_tensorflow_logging
+from .utils.logging import (
+    enable_logging,
+    enable_tensorflow_logging,
+    get_logger)
 
 __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'
@@ -26,19 +30,22 @@ def main(argv):
 
     :param argv: Provided command line arguments.
     """
-    parser = create_argument_parser()
+    try:
-    arguments = parser.parse_args(argv[1:])
+        parser = create_argument_parser()
-    enable_logging()
+        arguments = parser.parse_args(argv[1:])
-    if arguments.verbose:
+        enable_logging()
-        enable_tensorflow_logging()
+        if arguments.verbose:
-    if arguments.command == 'separate':
+            enable_tensorflow_logging()
-        from .commands.separate import entrypoint
+        if arguments.command == 'separate':
-    elif arguments.command == 'train':
+            from .commands.separate import entrypoint
-        from .commands.train import entrypoint
+        elif arguments.command == 'train':
-    elif arguments.command == 'evaluate':
+            from .commands.train import entrypoint
-        from .commands.evaluate import entrypoint
+        elif arguments.command == 'evaluate':
-    params = load_configuration(arguments.params_filename)
+            from .commands.evaluate import entrypoint
-    entrypoint(arguments, params)
+        params = load_configuration(arguments.configuration)
+        entrypoint(arguments, params)
+    except SpleeterError as e:
+        get_logger().error(e)
 
 
 def entrypoint():

spleeter/audio/adapter.py

@@ -16,7 +16,8 @@ import tensorflow as tf
 from tensorflow.contrib.signal import stft, hann_window
 # pylint: enable=import-error
 
-from ..logging import get_logger
+from .. import SpleeterError
+from ..utils.logging import get_logger
 
 __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'
@@ -73,7 +74,8 @@ class AudioAdapter(ABC):
 
         # Defined safe loading function.
         def safe_load(path, offset, duration, sample_rate, dtype):
-            get_logger().info(
+            logger = get_logger()
+            logger.info(
                 f'Loading audio {path} from {offset} to {offset + duration}')
             try:
                 (data, _) = self.load(
@@ -82,10 +84,12 @@ class AudioAdapter(ABC):
                     duration.numpy(),
                     sample_rate.numpy(),
                     dtype=dtype.numpy())
-                get_logger().info('Audio data loaded successfully')
+                logger.info('Audio data loaded successfully')
                 return (data, False)
             except Exception as e:
-                get_logger().warning(e)
+                logger.exception(
+                    'An error occurs while loading audio',
+                    exc_info=e)
             return (np.float32(-1.0), True)
 
         # Execute function and format results.
@@ -140,6 +144,6 @@ def get_audio_adapter(descriptor):
     adapter_module = import_module(module_path)
     adapter_class = getattr(adapter_module, adapter_class_name)
     if not isinstance(adapter_class, AudioAdapter):
-        raise ValueError(
+        raise SpleeterError(
             f'{adapter_class_name} is not a valid AudioAdapter class')
     return adapter_class()

spleeter/audio/convertor.py

@@ -8,7 +8,7 @@ import numpy as np
 import tensorflow as tf
 # pylint: enable=import-error
 
-from ..tensor import from_float32_to_uint8, from_uint8_to_float32
+from ..utils.tensor import from_float32_to_uint8, from_uint8_to_float32
 
 __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'

spleeter/audio/ffmpeg.py

@@ -16,7 +16,8 @@ import numpy as np
 # pylint: enable=import-error
 
 from .adapter import AudioAdapter
-from ..logging import get_logger
+from .. import SpleeterError
+from ..utils.logging import get_logger
 
 __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'
@@ -54,12 +55,18 @@ class FFMPEGProcessAudioAdapter(AudioAdapter):
         :param sample_rate: (Optional) Sample rate to load audio with.
         :param dtype: (Optional) Numpy data type to use, default to float32.
         :returns: Loaded data a (waveform, sample_rate) tuple.
+        :raise SpleeterError: If any error occurs while loading audio.
         """
         if not isinstance(path, str):
             path = path.decode()
-        probe = ffmpeg.probe(path)
+        try:
+            probe = ffmpeg.probe(path)
+        except ffmpeg._run.Error as e:
+            raise SpleeterError(
+                'An error occurs with ffprobe (see ffprobe output below)\n\n{}'
+                .format(e.stderr.decode()))
         if 'streams' not in probe or len(probe['streams']) == 0:
-            raise IOError('No stream was found with ffprobe')
+            raise SpleeterError('No stream was found with ffprobe')
         metadata = next(
             stream
             for stream in probe['streams']
@@ -117,5 +124,5 @@ class FFMPEGProcessAudioAdapter(AudioAdapter):
             process.stdin.close()
             process.wait()
         except IOError:
-            raise IOError(f'FFMPEG error: {process.stderr.read()}')
+            raise SpleeterError(f'FFMPEG error: {process.stderr.read()}')
         get_logger().info('File %s written', path)

spleeter/commands/__init__.py

@@ -13,67 +13,77 @@ __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'
 __license__ = 'MIT License'
 
-# -i opt specification.
+# -i opt specification (separate).
 OPT_INPUT = {
-    'dest': 'audio_filenames',
+    'dest': 'inputs',
     'nargs': '+',
     'help': 'List of input audio filenames',
     'required': True
 }
 
-# -o opt specification.
+# -o opt specification (evaluate and separate).
 OPT_OUTPUT = {
     'dest': 'output_path',
     'default': join(gettempdir(), 'separated_audio'),
     'help': 'Path of the output directory to write audio files in'
 }
 
-# -p opt specification.
+# -f opt specification (separate).
+OPT_FORMAT = {
+    'dest': 'filename_format',
+    'default': '{filename}/{instrument}.{codec}',
+    'help': (
+        'Template string that will be formatted to generated'
+        'output filename. Such template should be Python formattable'
+        'string, and could use {filename}, {instrument}, and {codec}'
+        'variables.'
+    )
+}
+
+# -p opt specification (train, evaluate and separate).
 OPT_PARAMS = {
-    'dest': 'params_filename',
+    'dest': 'configuration',
     'default': 'spleeter:2stems',
     'type': str,
     'action': 'store',
     'help': 'JSON filename that contains params'
 }
 
-# -n opt specification.
+# -s opt specification (separate).
-OPT_OUTPUT_NAMING = {
+OPT_OFFSET = {
-    'dest': 'output_naming',
+    'dest': 'offset',
-    'default': 'filename',
+    'type': float,
-    'choices': ('directory', 'filename'),
+    'default': 0.,
-    'help': (
+    'help': 'Set the starting offset to separate audio from.'
-        'Choice for naming the output base path: '
-        '"filename" (use the input filename, i.e '
-        '/path/to/audio/mix.wav will be separated to '
-        '<output_path>/mix/<instument1>.wav, '
-        '<output_path>/mix/<instument2>.wav...) or '
-        '"directory" (use the name of the input last level'
-        ' directory, for instance /path/to/audio/mix.wav '
-        'will be separated to <output_path>/audio/<instument1>.wav'
-        ', <output_path>/audio/<instument2>.wav)')
 }
 
 # -d opt specification (separate).
 OPT_DURATION = {
-    'dest': 'max_duration',
+    'dest': 'duration',
     'type': float,
     'default': 600.,
     'help': (
         'Set a maximum duration for processing audio '
-        '(only separate max_duration first seconds of '
+        '(only separate offset + duration first seconds of '
         'the input file)')
 }
 
-# -c opt specification.
+# -c opt specification (separate).
 OPT_CODEC = {
-    'dest': 'audio_codec',
+    'dest': 'codec',
     'choices': ('wav', 'mp3', 'ogg', 'm4a', 'wma', 'flac'),
     'default': 'wav',
     'help': 'Audio codec to be used for the separated output'
 }
 
-# -m opt specification.
+# -b opt specification (separate).
+OPT_BITRATE = {
+    'dest': 'bitrate',
+    'default': '128k',
+    'help': 'Audio bitrate to be used for the separated output'
+}
+
+# -m opt specification (evaluate and separate).
 OPT_MWF = {
     'dest': 'MWF',
     'action': 'store_const',
@@ -82,7 +92,7 @@ OPT_MWF = {
     'help': 'Whether to use multichannel Wiener filtering for separation',
 }
 
-# --mus_dir opt specification.
+# --mus_dir opt specification (evaluate).
 OPT_MUSDB = {
     'dest': 'mus_dir',
     'type': str,
@@ -98,14 +108,14 @@ OPT_DATA = {
     'help': 'Path of the folder containing audio data for training'
 }
 
-# -a opt specification.
+# -a opt specification (train, evaluate and separate).
 OPT_ADAPTER = {
     'dest': 'audio_adapter',
     'type': str,
     'help': 'Name of the audio adapter to use for audio I/O'
 }
 
-# -a opt specification.
+# -a opt specification (train, evaluate and separate).
 OPT_VERBOSE = {
     'action': 'store_true',
     'help': 'Shows verbose logs'
@@ -158,11 +168,13 @@ def _create_separate_parser(parser_factory):
     """
     parser = parser_factory('separate', help='Separate audio files')
     _add_common_options(parser)
-    parser.add_argument('-i', '--audio_filenames', **OPT_INPUT)
+    parser.add_argument('-i', '--inputs', **OPT_INPUT)
     parser.add_argument('-o', '--output_path', **OPT_OUTPUT)
-    parser.add_argument('-n', '--output_naming', **OPT_OUTPUT_NAMING)
+    parser.add_argument('-f', '--filename_format', **OPT_FORMAT)
-    parser.add_argument('-d', '--max_duration', **OPT_DURATION)
+    parser.add_argument('-d', '--duration', **OPT_DURATION)
-    parser.add_argument('-c', '--audio_codec', **OPT_CODEC)
+    parser.add_argument('-s', '--offset', **OPT_OFFSET)
+    parser.add_argument('-c', '--codec', **OPT_CODEC)
+    parser.add_argument('-b', '--birate', **OPT_BITRATE)
     parser.add_argument('-m', '--mwf', **OPT_MWF)
     return parser
 

spleeter/commands/evaluate.py

@@ -44,7 +44,6 @@ __license__ = 'MIT License'
 
 _SPLIT = 'test'
 _MIXTURE = 'mixture.wav'
-_NAMING = 'directory'
 _AUDIO_DIRECTORY = 'audio'
 _METRICS_DIRECTORY = 'metrics'
 _INSTRUMENTS = ('vocals', 'drums', 'bass', 'other')
@@ -71,7 +70,6 @@ def _separate_evaluation_dataset(arguments, musdb_root_directory, params):
             audio_filenames=mixtures,
             audio_codec='wav',
             output_path=join(audio_output_directory, _SPLIT),
-            output_naming=_NAMING,
             max_duration=600.,
             MWF=arguments.MWF,
             verbose=arguments.verbose),

spleeter/commands/separate.py

@@ -11,168 +11,35 @@
         -i /path/to/audio1.wav /path/to/audio2.mp3
 """
 
-from multiprocessing import Pool
+from ..audio.adapter import get_audio_adapter
-from os.path import isabs, join, split, splitext
+from ..separator import Separator
-from tempfile import gettempdir
-
-# pylint: disable=import-error
-import tensorflow as tf
-import numpy as np
-# pylint: enable=import-error
-
-from ..utils.audio.adapter import get_audio_adapter
-from ..utils.audio.convertor import to_n_channels
-from ..utils.estimator import create_estimator
-from ..utils.tensor import set_tensor_shape
 
 __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'
 __license__ = 'MIT License'
 
 
-def get_dataset(audio_adapter, filenames_and_crops, sample_rate, n_channels):
-    """"
-        Build a tensorflow dataset of waveform from a filename list wit crop
-        information.
-
-        Params:
-        - audio_adapter:        An AudioAdapter instance to load audio from.
-        - filenames_and_crops:  list of (audio_filename, start, duration)
-                                tuples separation is performed on each filaneme
-                                from start (in seconds) to start + duration
-                                (in seconds).
-        - sample_rate:          audio sample_rate of the input and output audio
-                                signals
-        - n_channels:           int, number of channels of the input and output
-                                audio signals
-
-        Returns
-        A tensorflow dataset of waveform to feed a tensorflow estimator in
-        predict mode.
-    """
-    filenames, starts, ends = list(zip(*filenames_and_crops))
-    dataset = tf.data.Dataset.from_tensor_slices({
-        'audio_id': list(filenames),
-        'start': list(starts),
-        'end': list(ends)
-    })
-    # Load waveform.
-    dataset = dataset.map(
-        lambda sample: dict(
-            sample,
-            **audio_adapter.load_tf_waveform(
-                sample['audio_id'],
-                sample_rate=sample_rate,
-                offset=sample['start'],
-                duration=sample['end'] - sample['start'])),
-        num_parallel_calls=2)
-    # Filter out error.
-    dataset = dataset.filter(
-        lambda sample: tf.logical_not(sample['waveform_error']))
-    # Convert waveform to the right number of channels.
-    dataset = dataset.map(
-        lambda sample: dict(
-            sample,
-            waveform=to_n_channels(sample['waveform'], n_channels)))
-    # Set number of channels (required for the model).
-    dataset = dataset.map(
-        lambda sample: dict(
-            sample,
-            waveform=set_tensor_shape(sample['waveform'], (None, n_channels))))
-    return dataset
-
-
-def process_audio(
-        audio_adapter,
-        filenames_and_crops, estimator, output_path,
-        sample_rate, n_channels, codec, output_naming):
-    """
-        Perform separation on a list of audio ids.
-
-        Params:
-        - audio_adapter:        Audio adapter to use for audio I/O.
-        - filenames_and_crops:  list of (audio_filename, start, duration)
-                                tuples separation is performed on each filaneme
-                                from start (in seconds) to start + duration
-                                (in seconds).
-        - estimator:            the tensorflow estimator that performs the
-                                source separation.
-        - output_path:          output_path where to export separated files.
-        - sample_rate:          audio sample_rate of the input and output audio
-                                signals
-        - n_channels:           int, number of channels of the input and output
-                                audio signals
-        - codec:                string codec to be used for export (could be
-                                "wav", "mp3", "ogg", "m4a") could be anything
-                                supported by ffmpeg.
-        - output_naming: string (= "filename" of "directory")
-            naming convention for output.
-            for an input file /path/to/audio/input_file.wav:
-                * if output_naming is equal to "filename":
-        output files will be put in the directory <output_path>/input_file
-        (<output_path>/input_file/<instrument1>.<codec>,
-         <output_path>/input_file/<instrument2>.<codec>...).
-                * if output_naming is equal to "directory":
-        output files will be put in the directory <output_path>/audio/
-        (<output_path>/audio/<instrument1>.<codec>,
-         <output_path>/audio/<instrument2>.<codec>...)
-        Use "directory" when separating the MusDB dataset.
-
-    """
-    # Get estimator
-    prediction = estimator.predict(
-        lambda: get_dataset(
-            audio_adapter,
-            filenames_and_crops,
-            sample_rate,
-            n_channels),
-        yield_single_examples=False)
-    # initialize pool for audio export
-    pool = Pool(16)
-    for sample in prediction:
-        sample_filename = sample.pop('audio_id', 'unknown_filename').decode()
-        input_directory, input_filename = split(sample_filename)
-        if output_naming == 'directory':
-            output_dirname = split(input_directory)[1]
-        elif output_naming == 'filename':
-            output_dirname = splitext(input_filename)[0]
-        else:
-            raise ValueError(f'Unknown output naming {output_naming}')
-        for instrument, waveform in sample.items():
-            filename = join(
-                output_path,
-                output_dirname,
-                f'{instrument}.{codec}')
-            pool.apply_async(
-                audio_adapter.save,
-                (filename, waveform, sample_rate, codec))
-    # Wait for everything to be written
-    pool.close()
-    pool.join()
-
-
 def entrypoint(arguments, params):
     """ Command entrypoint.
 
     :param arguments: Command line parsed argument as argparse.Namespace.
     :param params: Deserialized JSON configuration file provided in CLI args.
     """
+    # TODO: check with output naming.
     audio_adapter = get_audio_adapter(arguments.audio_adapter)
-    filenames = arguments.audio_filenames
+    separator = Separator(
-    output_path = arguments.output_path
+        arguments.configuration,
-    max_duration = arguments.max_duration
+        arguments.MWF)
-    audio_codec = arguments.audio_codec
+    for filename in arguments.inputs:
-    output_naming = arguments.output_naming
+        separator.separate_to_file(
-    estimator = create_estimator(params, arguments.MWF)
+            filename,
-    filenames_and_crops = [
+            arguments.output_path,
-        (filename, 0., max_duration)
+            audio_adapter=audio_adapter,
-        for filename in filenames]
+            offset=arguments.offset,
-    process_audio(
+            duration=arguments.duration,
-        audio_adapter,
+            codec=arguments.codec,
-        filenames_and_crops,
+            bitrate=arguments.bitrate,
-        estimator,
+            filename_format=arguments.filename_format,
-        output_path,
+            synchronous=False
-        params['sample_rate'],
+        )
-        params['n_channels'],
+    separator.join()
-        codec=audio_codec,
-        output_naming=output_naming)

spleeter/commands/train.py

@@ -13,9 +13,10 @@ from functools import partial
 import tensorflow as tf
 # pylint: enable=import-error
 
+from ..audio.adapter import get_audio_adapter
 from ..dataset import get_training_dataset, get_validation_dataset
 from ..model import model_fn
-from ..utils.audio.adapter import get_audio_adapter
+from ..model.provider import ModelProvider
 from ..utils.logging import get_logger
 
 __email__ = 'research@deezer.com'
@@ -95,4 +96,5 @@ def entrypoint(arguments, params):
         estimator,
         train_spec,
         evaluation_spec)
+    ModelProvider.writeProbe(params['model_dir'])
     get_logger().info('Model training done')

spleeter/dataset.py

@@ -2,15 +2,16 @@
 # coding: utf8
 
 """
-    Module for building data preprocessing pipeline using the tensorflow data
+    Module for building data preprocessing pipeline using the tensorflow
-    API.
+    data API. Data preprocessing such as audio loading, spectrogram
-    Data preprocessing such as audio loading, spectrogram computation, cropping,
+    computation, cropping, feature caching or data augmentation is done
-    feature caching or data augmentation is done using a tensorflow dataset object
+    using a tensorflow dataset object that output a tuple (input_, output)
-    that output a tuple (input_, output) where:
+    where:
-    - input_ is a dictionary with a single key that contains the (batched) mix
-    spectrogram of audio samples
-    - output is a dictionary of spectrogram of the isolated tracks (ground truth)
 
+    -   input is a dictionary with a single key that contains the (batched)
+        mix spectrogram of audio samples
+    -   output is a dictionary of spectrogram of the isolated tracks
+        (ground truth)
 """
 
 import time
@@ -23,10 +24,10 @@ import numpy as np
 import tensorflow as tf
 # pylint: enable=import-error
 
-from .utils.audio.convertor import (
+from .audio.convertor import (
     db_uint_spectrogram_to_gain,
     spectrogram_to_db_uint)
-from .utils.audio.spectrogram import (
+from .audio.spectrogram import (
     compute_spectrogram_tf,
     random_pitch_shift,
     random_time_stretch)
@@ -41,15 +42,6 @@ __email__ = 'research@deezer.com'
 __author__ = 'Deezer Research'
 __license__ = 'MIT License'
 
-# Default datasets path parameter to use.
-DEFAULT_DATASETS_PATH = join(
-    'audio_database',
-    'separated_sources',
-    'experiments',
-    'karaoke_vocal_extraction',
-    'tensorflow_experiment'
-)
-
 # Default audio parameters to use.
 DEFAULT_AUDIO_PARAMS = {
     'instrument_list': ('vocals', 'accompaniment'),

spleeter/model/provider/__init__.py

@@ -38,12 +38,14 @@ class ModelProvider(ABC):
         """
         pass
 
-    def writeProbe(self, directory):
+    @staticmethod
+    def writeProbe(directory):
         """ Write a model probe file into the given directory.
 
         :param directory: Directory to write probe into.
         """
-        with open(join(directory, self.MODEL_PROBE_PATH), 'w') as stream:
+        probe = join(directory, ModelProvider.MODEL_PROBE_PATH)
+        with open(probe, 'w') as stream:
             stream.write('OK')
 
     def get(self, model_directory):

spleeter/model/provider/github.py

@@ -14,11 +14,10 @@
     >>> provider.download('2stems', '/path/to/local/storage')
 """
 
+import hashlib
 import tarfile
 
-from os import environ
+from tempfile import NamedTemporaryFile
-from tempfile import TemporaryFile
-from shutil import copyfileobj
 
 import requests
 
@@ -30,11 +29,25 @@ __author__ = 'Deezer Research'
 __license__ = 'MIT License'
 
 
+def compute_file_checksum(path):
+    """ Computes given path file sha256.
+
+    :param path: Path of the file to compute checksum for.
+    :returns: File checksum.
+    """
+    sha256 = hashlib.sha256()
+    with open(path, 'rb') as stream:
+        for chunk in iter(lambda: stream.read(4096), b''):
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+
 class GithubModelProvider(ModelProvider):
     """ A ModelProvider implementation backed on Github for remote storage. """
 
     LATEST_RELEASE = 'v1.4.0'
     RELEASE_PATH = 'releases/download'
+    CHECKSUM_INDEX = 'checksum.json'
 
     def __init__(self, host, repository, release):
         """ Default constructor.
@@ -47,6 +60,26 @@ class GithubModelProvider(ModelProvider):
         self._repository = repository
         self._release = release
 
+    def checksum(self, name):
+        """ Downloads and returns reference checksum for the given model name.
+
+        :param name: Name of the model to get checksum for.
+        :returns: Checksum of the required model.
+        :raise ValueError: If the given model name is not indexed.
+        """
+        url = '{}/{}/{}/{}/{}'.format(
+            self._host,
+            self._repository,
+            self.RELEASE_PATH,
+            self._release,
+            self.CHECKSUM_INDEX)
+        response = requests.get(url)
+        response.raise_for_status()
+        index = response.json()
+        if name not in index:
+            raise ValueError('No checksum for model {}'.format(name))
+        return index[name]
+
     def download(self, name, path):
         """ Download model denoted by the given name to disk.
 
@@ -60,14 +93,19 @@ class GithubModelProvider(ModelProvider):
             self._release,
             name)
         get_logger().info('Downloading model archive %s', url)
-        response = requests.get(url, stream=True)
+        with requests.get(url, stream=True) as response:
-        if response.status_code != 200:
+            response.raise_for_status()
-            raise IOError(f'Resource {url} not found')
+            archive = NamedTemporaryFile(delete=False)
-        with TemporaryFile() as stream:
+            with archive as stream:
-            copyfileobj(response.raw, stream)
+                # Note: check for chunk size parameters ?
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        stream.write(chunk)
+            get_logger().info('Validating archive checksum')
+            if compute_file_checksum(archive.name) != self.checksum(name):
+                raise IOError('Downloaded file is corrupted, please retry')
             get_logger().info('Extracting downloaded %s archive', name)
-            stream.seek(0)
+            tar = tarfile.open(name=archive.name)
-            tar = tarfile.open(fileobj=stream)
             tar.extractall(path=path)
             tar.close()
         get_logger().info('%s model file(s) extracted', name)

spleeter/separator.py

@@ -18,11 +18,12 @@ import json
 from functools import partial
 from multiprocessing import Pool
 from pathlib import Path
-from os.path import join
+from os.path import basename, join
 
+from . import SpleeterError
+from .audio.adapter import get_default_audio_adapter
+from .audio.convertor import to_stereo
 from .model import model_fn
-from .utils.audio.adapter import get_default_audio_adapter
-from .utils.audio.convertor import to_stereo
 from .utils.configuration import load_configuration
 from .utils.estimator import create_estimator, to_predictor
 
@@ -57,7 +58,7 @@ class Separator(object):
             self._predictor = to_predictor(estimator)
         return self._predictor
 
-    def join(self, timeout=20):
+    def join(self, timeout=200):
         """ Wait for all pending tasks to be finished.
 
         :param timeout: (Optional) task waiting timeout.
@@ -93,10 +94,13 @@ class Separator(object):
             self, audio_descriptor, destination,
             audio_adapter=get_default_audio_adapter(),
             offset=0, duration=600., codec='wav', bitrate='128k',
-            synchronous=True):
+            filename_format='{filename}/{instrument}.{codec}', synchronous=True):
         """ Performs source separation and export result to file using
         given audio adapter.
 
+        Filename format should be a Python formattable string that could use
+        following parameters : {instrument}, {filename} and {codec}.
+
         :param audio_descriptor:    Describe song to separate, used by audio
                                     adapter to retrieve and load audio data,
                                     in case of file based audio adapter, such
@@ -107,6 +111,7 @@ class Separator(object):
         :param duration:            (Optional) Duration of loaded song.
         :param codec:               (Optional) Export codec.
         :param bitrate:             (Optional) Export bitrate.
+        :param filename_format:     (Optional) Filename format.
         :param synchronous:         (Optional) True is should by synchronous.
         """
         waveform, _ = audio_adapter.load(
@@ -115,9 +120,20 @@ class Separator(object):
             duration=duration,
             sample_rate=self._sample_rate)
         sources = self.separate(waveform)
+        filename = basename(audio_descriptor)
+        generated = []
         for instrument, data in sources.items():
+            path = join(destination, filename_format.format(
+                filename=filename,
+                instrument=instrument,
+                codec=codec))
+            if path in generated:
+                raise SpleeterError((
+                    f'Separated source path conflict : {path},'
+                    'please check your filename format'))
+            generated.append(path)
             task = self._pool.apply_async(audio_adapter.save, (
-                join(destination, f'{instrument}.{codec}'),
+                path,
                 data,
                 self._sample_rate,
                 codec,

spleeter/utils/configuration.py

@@ -13,7 +13,7 @@ except ImportError:
 
 from os.path import exists
 
-from .. import resources
+from .. import resources, SpleeterError
 
 
 __email__ = 'research@deezer.com'
@@ -31,17 +31,17 @@ def load_configuration(descriptor):
     :param descriptor: Configuration descriptor to use for lookup.
     :returns: Loaded description as dict.
     :raise ValueError: If required embedded configuration does not exists.
-    :raise IOError: If required configuration file does not exists.
+    :raise SpleeterError: If required configuration file does not exists.
     """
     # Embedded configuration reading.
     if descriptor.startswith(_EMBEDDED_CONFIGURATION_PREFIX):
         name = descriptor[len(_EMBEDDED_CONFIGURATION_PREFIX):]
         if not loader.is_resource(resources, f'{name}.json'):
-            raise ValueError(f'No embedded configuration {name} found')
+            raise SpleeterError(f'No embedded configuration {name} found')
         with loader.open_text(resources, f'{name}.json') as stream:
             return json.load(stream)
     # Standard file reading.
     if not exists(descriptor):
-        raise IOError(f'Configuration file {descriptor} not found')
+        raise SpleeterError(f'Configuration file {descriptor} not found')
     with open(descriptor, 'r') as stream:
         return json.load(stream)

tests/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# coding: utf8
+
+""" Unit testing package. """
+
+__email__ = 'research@deezer.com'
+__author__ = 'Deezer Research'
+__license__ = 'MIT License'

tests/test_ffmpeg_adapter.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# coding: utf8
+
+""" Unit testing for audio adapter. """
+
+__email__ = 'research@deezer.com'
+__author__ = 'Deezer Research'
+__license__ = 'MIT License'
+
+from os.path import join
+from tempfile import TemporaryDirectory
+
+# pylint: disable=import-error
+from pytest import fixture, raises
+
+import numpy as np
+import ffmpeg
+# pylint: enable=import-error
+
+from spleeter import SpleeterError
+from spleeter.audio.adapter import AudioAdapter
+from spleeter.audio.adapter import get_default_audio_adapter
+from spleeter.audio.adapter import get_audio_adapter
+from spleeter.audio.ffmpeg import FFMPEGProcessAudioAdapter
+
+TEST_AUDIO_DESCRIPTOR = 'audio_example.mp3'
+TEST_OFFSET = 0
+TEST_DURATION = 600.
+TEST_SAMPLE_RATE = 44100
+
+
+@fixture(scope='session')
+def adapter():
+    """ Target test audio adapter fixture. """
+    return get_default_audio_adapter()
+
+
+@fixture(scope='session')
+def audio_data(adapter):
+    """ Audio data fixture based on sample loading from adapter. """
+    return adapter.load(
+        TEST_AUDIO_DESCRIPTOR,
+        TEST_OFFSET,
+        TEST_DURATION,
+        TEST_SAMPLE_RATE)
+
+
+def test_default_adapter(adapter):
+    """ Test adapter as default adapter. """
+    assert isinstance(adapter, FFMPEGProcessAudioAdapter)
+    assert adapter is AudioAdapter.DEFAULT
+
+
+def test_load(audio_data):
+    """ Test audio loading. """
+    waveform, sample_rate = audio_data
+    assert sample_rate == TEST_SAMPLE_RATE
+    assert waveform is not None
+    assert waveform.dtype == np.dtype('float32')
+    assert len(waveform.shape) == 2
+    assert waveform.shape[0] == 479832
+    assert waveform.shape[1] == 2
+
+
+def test_load_error(adapter):
+    """ Test load ffprobe exception """
+    with raises(SpleeterError):
+        adapter.load(
+            'Paris City Jazz',
+            TEST_OFFSET,
+            TEST_DURATION,
+            TEST_SAMPLE_RATE)
+
+
+def test_save(adapter, audio_data):
+    """ Test audio saving. """
+    with TemporaryDirectory() as directory:
+        path = join(directory, 'ffmpeg-save.mp3')
+        adapter.save(
+            path,
+            audio_data[0],
+            audio_data[1])
+        probe = ffmpeg.probe(TEST_AUDIO_DESCRIPTOR)
+        assert len(probe['streams']) == 1
+        stream = probe['streams'][0]
+        assert stream['codec_type'] == 'audio'
+        assert stream['channels'] == 2
+        assert stream['duration'] == '10.919184'

tests/test_github_model_provider.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# coding: utf8
+
+""" TO DOCUMENT """
+
+from pytest import raises
+
+from spleeter.model.provider import get_default_model_provider
+
+
+def test_checksum():
+    """ Test archive checksum index retrieval. """
+    provider = get_default_model_provider()
+    assert provider.checksum('2stems') == \
+        'f3a90b39dd2874269e8b05a48a86745df897b848c61f3958efc80a39152bd692'
+    assert provider.checksum('4stems') == \
+        '3adb4a50ad4eb18c7c4d65fcf4cf2367a07d48408a5eb7d03cd20067429dfaa8'
+    assert provider.checksum('5stems') == \
+        '25a1e87eb5f75cc72a4d2d5467a0a50ac75f05611f877c278793742513cc7218'
+    with raises(ValueError):
+        provider.checksum('laisse moi stems stems stems')

tests/test_separator.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# coding: utf8
+
+""" Unit testing for Separator class. """
+
+__email__ = 'research@deezer.com'
+__author__ = 'Deezer Research'
+__license__ = 'MIT License'
+
+import filecmp
+
+from os.path import basename, exists, join
+from tempfile import TemporaryDirectory
+
+import pytest
+
+from spleeter import SpleeterError
+from spleeter.audio.adapter import get_default_audio_adapter
+from spleeter.separator import Separator
+
+TEST_AUDIO_DESCRIPTOR = 'audio_example.mp3'
+TEST_AUDIO_BASENAME = basename(TEST_AUDIO_DESCRIPTOR)
+TEST_CONFIGURATIONS = [
+    ('spleeter:2stems', ('vocals', 'accompaniment')),
+    ('spleeter:4stems', ('vocals', 'drums', 'bass', 'other')),
+    ('spleeter:5stems', ('vocals', 'drums', 'bass', 'piano', 'other'))
+]
+
+
+@pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS)
+def test_separate(configuration, instruments):
+    """ Test separation from raw data. """
+    adapter = get_default_audio_adapter()
+    waveform, _ = adapter.load(TEST_AUDIO_DESCRIPTOR)
+    separator = Separator(configuration)
+    prediction = separator.separate(waveform)
+    assert len(prediction) == len(instruments)
+    for instrument in instruments:
+        assert instrument in prediction
+    for instrument in instruments:
+        track = prediction[instrument]
+        assert not (waveform == track).all()
+        for compared in instruments:
+            if instrument != compared:
+                assert not (track == prediction[compared]).all()
+
+
+@pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS)
+def test_separate_to_file(configuration, instruments):
+    """ Test file based separation. """
+    separator = Separator(configuration)
+    with TemporaryDirectory() as directory:
+        separator.separate_to_file(
+            TEST_AUDIO_DESCRIPTOR,
+            directory)
+        for instrument in instruments:
+            assert exists(join(
+                directory,
+                '{}/{}.wav'.format(TEST_AUDIO_BASENAME, instrument)))
+
+
+@pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS)
+def test_filename_format(configuration, instruments):
+    """ Test custom filename format. """
+    separator = Separator(configuration)
+    with TemporaryDirectory() as directory:
+        separator.separate_to_file(
+            TEST_AUDIO_DESCRIPTOR,
+            directory,
+            filename_format='export/{filename}/{instrument}.{codec}')
+        for instrument in instruments:
+            assert exists(join(
+                directory,
+                'export/{}/{}.wav'.format(TEST_AUDIO_BASENAME, instrument)))
+
+
+def test_filename_confilct():
+    """ Test error handling with static pattern. """
+    separator = Separator(TEST_CONFIGURATIONS[0][0])
+    with TemporaryDirectory() as directory:
+        with pytest.raises(SpleeterError):
+            separator.separate_to_file(
+                TEST_AUDIO_DESCRIPTOR,
+                directory,
+                filename_format='I wanna be your lover')