From c4d1343897e172981350dd8bb4e224b7431fd80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 17:09:17 -0500 Subject: [PATCH 01/40] refactor: move audio package from utils to top --- setup.py | 2 +- spleeter/{utils => }/audio/__init__.py | 0 spleeter/{utils => }/audio/adapter.py | 0 spleeter/{utils => }/audio/convertor.py | 0 spleeter/{utils => }/audio/ffmpeg.py | 0 spleeter/{utils => }/audio/spectrogram.py | 0 spleeter/commands/separate.py | 4 ++-- spleeter/commands/train.py | 2 +- spleeter/dataset.py | 21 +++++++++++---------- spleeter/separator.py | 4 ++-- 10 files changed, 17 insertions(+), 16 deletions(-) rename spleeter/{utils => }/audio/__init__.py (100%) rename spleeter/{utils => }/audio/adapter.py (100%) rename spleeter/{utils => }/audio/convertor.py (100%) rename spleeter/{utils => }/audio/ffmpeg.py (100%) rename spleeter/{utils => }/audio/spectrogram.py (100%) diff --git a/setup.py b/setup.py index 6680413..b9abf78 100644 --- a/setup.py +++ b/setup.py @@ -51,13 +51,13 @@ setup( license='MIT License', packages=[ 'spleeter', + 'spleeter.audio', 'spleeter.commands', 'spleeter.model', 'spleeter.model.functions', 'spleeter.model.provider', 'spleeter.resources', 'spleeter.utils', - 'spleeter.utils.audio', ], package_data={'spleeter.resources': ['*.json']}, python_requires='>=3.6, <3.8', diff --git a/spleeter/utils/audio/__init__.py b/spleeter/audio/__init__.py similarity index 100% rename from spleeter/utils/audio/__init__.py rename to spleeter/audio/__init__.py diff --git a/spleeter/utils/audio/adapter.py b/spleeter/audio/adapter.py similarity index 100% rename from spleeter/utils/audio/adapter.py rename to spleeter/audio/adapter.py diff --git a/spleeter/utils/audio/convertor.py b/spleeter/audio/convertor.py similarity index 100% rename from spleeter/utils/audio/convertor.py rename to spleeter/audio/convertor.py diff --git a/spleeter/utils/audio/ffmpeg.py b/spleeter/audio/ffmpeg.py similarity index 100% rename from spleeter/utils/audio/ffmpeg.py rename to spleeter/audio/ffmpeg.py diff --git a/spleeter/utils/audio/spectrogram.py b/spleeter/audio/spectrogram.py similarity index 100% rename from spleeter/utils/audio/spectrogram.py rename to spleeter/audio/spectrogram.py diff --git a/spleeter/commands/separate.py b/spleeter/commands/separate.py index 71eed2d..10da43c 100644 --- a/spleeter/commands/separate.py +++ b/spleeter/commands/separate.py @@ -20,8 +20,8 @@ import tensorflow as tf import numpy as np # pylint: enable=import-error -from ..utils.audio.adapter import get_audio_adapter -from ..utils.audio.convertor import to_n_channels +from ..audio.adapter import get_audio_adapter +from ..audio.convertor import to_n_channels from ..utils.estimator import create_estimator from ..utils.tensor import set_tensor_shape diff --git a/spleeter/commands/train.py b/spleeter/commands/train.py index 2814ae6..bb48c41 100644 --- a/spleeter/commands/train.py +++ b/spleeter/commands/train.py @@ -13,9 +13,9 @@ from functools import partial import tensorflow as tf # pylint: enable=import-error +from ..audio.adapter import get_audio_adapter from ..dataset import get_training_dataset, get_validation_dataset from ..model import model_fn -from ..utils.audio.adapter import get_audio_adapter from ..utils.logging import get_logger __email__ = 'research@deezer.com' diff --git a/spleeter/dataset.py b/spleeter/dataset.py index dc65652..4f8c3e1 100644 --- a/spleeter/dataset.py +++ b/spleeter/dataset.py @@ -2,15 +2,16 @@ # coding: utf8 """ - Module for building data preprocessing pipeline using the tensorflow data - API. - Data preprocessing such as audio loading, spectrogram computation, cropping, - feature caching or data augmentation is done using a tensorflow dataset object - that output a tuple (input_, output) where: - - input_ is a dictionary with a single key that contains the (batched) mix - spectrogram of audio samples - - output is a dictionary of spectrogram of the isolated tracks (ground truth) + Module for building data preprocessing pipeline using the tensorflow + data API. Data preprocessing such as audio loading, spectrogram + computation, cropping, feature caching or data augmentation is done + using a tensorflow dataset object that output a tuple (input_, output) + where: + - input is a dictionary with a single key that contains the (batched) + mix spectrogram of audio samples + - output is a dictionary of spectrogram of the isolated tracks + (ground truth) """ import time @@ -23,10 +24,10 @@ import numpy as np import tensorflow as tf # pylint: enable=import-error -from .utils.audio.convertor import ( +from .audio.convertor import ( db_uint_spectrogram_to_gain, spectrogram_to_db_uint) -from .utils.audio.spectrogram import ( +from .audio.spectrogram import ( compute_spectrogram_tf, random_pitch_shift, random_time_stretch) diff --git a/spleeter/separator.py b/spleeter/separator.py index a238037..a5f4efd 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -20,9 +20,9 @@ from multiprocessing import Pool from pathlib import Path from os.path import join +from .audio.adapter import get_default_audio_adapter +from .audio.convertor import to_stereo from .model import model_fn -from .utils.audio.adapter import get_default_audio_adapter -from .utils.audio.convertor import to_stereo from .utils.configuration import load_configuration from .utils.estimator import create_estimator, to_predictor From d19c13bf66df1cc57b33dbd308dfd5e001a536d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 18:01:47 -0500 Subject: [PATCH 02/40] test: add unit test for audio adapter fix: import issues --- spleeter/audio/adapter.py | 2 +- spleeter/audio/convertor.py | 2 +- spleeter/audio/ffmpeg.py | 2 +- tests/__init__.py | 4 ++ tests/test_ffmpeg_adapter.py | 77 ++++++++++++++++++++++++++++++++++++ 5 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_ffmpeg_adapter.py diff --git a/spleeter/audio/adapter.py b/spleeter/audio/adapter.py index d1a64c0..59bce42 100644 --- a/spleeter/audio/adapter.py +++ b/spleeter/audio/adapter.py @@ -16,7 +16,7 @@ import tensorflow as tf from tensorflow.contrib.signal import stft, hann_window # pylint: enable=import-error -from ..logging import get_logger +from ..utils.logging import get_logger __email__ = 'research@deezer.com' __author__ = 'Deezer Research' diff --git a/spleeter/audio/convertor.py b/spleeter/audio/convertor.py index b6a7953..550345a 100644 --- a/spleeter/audio/convertor.py +++ b/spleeter/audio/convertor.py @@ -8,7 +8,7 @@ import numpy as np import tensorflow as tf # pylint: enable=import-error -from ..tensor import from_float32_to_uint8, from_uint8_to_float32 +from ..utils.tensor import from_float32_to_uint8, from_uint8_to_float32 __email__ = 'research@deezer.com' __author__ = 'Deezer Research' diff --git a/spleeter/audio/ffmpeg.py b/spleeter/audio/ffmpeg.py index d9c5506..8246a58 100644 --- a/spleeter/audio/ffmpeg.py +++ b/spleeter/audio/ffmpeg.py @@ -16,7 +16,7 @@ import numpy as np # pylint: enable=import-error from .adapter import AudioAdapter -from ..logging import get_logger +from ..utils.logging import get_logger __email__ = 'research@deezer.com' __author__ = 'Deezer Research' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..3b800d9 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +# coding: utf8 + +""" TO DOCUMENT """ \ No newline at end of file diff --git a/tests/test_ffmpeg_adapter.py b/tests/test_ffmpeg_adapter.py new file mode 100644 index 0000000..cd503c2 --- /dev/null +++ b/tests/test_ffmpeg_adapter.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# coding: utf8 + +""" Unit testing for audio adapter. """ + +__email__ = 'research@deezer.com' +__author__ = 'Deezer Research' +__license__ = 'MIT License' + +from os.path import join +from tempfile import TemporaryDirectory + +# pylint: disable=import-error +from pytest import fixture + +import numpy as np +import ffmpeg +# pylint: enable=import-error + +from spleeter.audio.adapter import AudioAdapter +from spleeter.audio.adapter import get_default_audio_adapter +from spleeter.audio.adapter import get_audio_adapter +from spleeter.audio.ffmpeg import FFMPEGProcessAudioAdapter + +TEST_AUDIO_DESCRIPTOR = 'audio_example.mp3' +TEST_OFFSET = 0 +TEST_DURATION = 600. +TEST_SAMPLE_RATE = 44100 + + +@fixture(scope='session') +def adapter(): + """ Target test audio adapter fixture. """ + return get_default_audio_adapter() + + +@fixture(scope='session') +def audio_data(adapter): + """ Audio data fixture based on sample loading from adapter. """ + return adapter.load( + TEST_AUDIO_DESCRIPTOR, + TEST_OFFSET, + TEST_DURATION, + TEST_SAMPLE_RATE) + + +def test_default_adapter(adapter): + """ Test adapter as default adapter. """ + assert isinstance(adapter, FFMPEGProcessAudioAdapter) + assert adapter is AudioAdapter.DEFAULT + + +def test_load(audio_data): + """ Test audio loading. """ + waveform, sample_rate = audio_data + assert sample_rate == TEST_SAMPLE_RATE + assert waveform is not None + assert waveform.dtype == np.dtype('float32') + assert len(waveform.shape) == 2 + assert waveform.shape[0] == 479832 + assert waveform.shape[1] == 2 + + +def test_save(adapter, audio_data): + """ Test audio saving. """ + with TemporaryDirectory() as directory: + path = join(directory, 'ffmpeg-save.mp3') + adapter.save( + path, + audio_data[0], + audio_data[1]) + probe = ffmpeg.probe(TEST_AUDIO_DESCRIPTOR) + assert len(probe['streams']) == 1 + stream = probe['streams'][0] + assert stream['codec_type'] == 'audio' + assert stream['channels'] == 2 + assert stream['duration'] == '10.919184' From 1e90bfb6ff941accf7b0a147f1ae36cc5726b5c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 18:35:02 -0500 Subject: [PATCH 03/40] test: add unit test for Separator class --- tests/test_separator.py | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tests/test_separator.py diff --git a/tests/test_separator.py b/tests/test_separator.py new file mode 100644 index 0000000..3115314 --- /dev/null +++ b/tests/test_separator.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# coding: utf8 + +""" Unit testing for Separator class. """ + +__email__ = 'research@deezer.com' +__author__ = 'Deezer Research' +__license__ = 'MIT License' + +from os.path import exists, join +from tempfile import TemporaryDirectory + +from spleeter.audio.adapter import get_default_audio_adapter +from spleeter.separator import Separator + +TEST_AUDIO_DESCRIPTOR = 'audio_example.mp3' +TEST_CONFIGURATIONS = { + 'spleeter:2stems': ('vocals', 'accompaniament'), + 'spleeter:4stems': ('vocals', 'drums', 'bass', 'other'), + 'spleeter:5stems': ('vocals', 'drums', 'bass', 'piano', 'other') +} + + +def test_separate(): + """ Test separation from raw data. """ + adapter = get_default_audio_adapter() + waveform, _ = adapter.load(TEST_AUDIO_DESCRIPTOR) + for configuration, instruments in TEST_CONFIGURATIONS: + separator = Separator(configuration) + prediction = separator.separate(waveform) + assert len(prediction) == 2 + for instrument in instruments: + assert instrument in prediction + + +def test_separate_to_file(): + """ Test file based separation. """ + for configuration, instruments in TEST_CONFIGURATIONS: + separator = Separator(configuration) + with TemporaryDirectory() as directory: + separator.separate_to_file( + TEST_AUDIO_DESCRIPTOR, + directory) + for instrument in instruments: + assert exists(join(directory, '{}.wav'.format(instrument))) + # TODO: Consider testing generated file as well. From 5f67eec9a22eb416d5aa407f1b37210997c16db5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:23:26 -0500 Subject: [PATCH 04/40] fix: add python version wf --- .circleci/config.yml | 53 ++++++++++++++++++++++++++++++++++++-------- requirements.txt | 7 ++++++ 2 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index b769876..d5b5267 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,21 +1,54 @@ version: 2 jobs: - test: + test-3.6: + docker: + - image: python:3.6 + working_directory: ~/spleeter + steps: + - checkout + - restore_cache: + keys: + - python-3.6-cache-{{ checksum requirements.txt }} + - python-3.6-cache- + - run: + name: install ffmpeg + command: apt-get update && apt-get install -y ffmpeg + - run: + name: install python dependencies + command: pip install -r requirements.txt && pip install pytest + - save_cache: + key: python-3.6-cache-{{ checksum requirements.txt }} + paths: + - "/usr/local/bin" + - "/usr/local/lib/python3.6/site-packages" + - run: + name: pytest + command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + test-3.7: docker: - image: python:3.7 working_directory: ~/spleeter steps: - checkout + - restore_cache: + keys: + - python-3.7-cache-{{ checksum requirements.txt }} + - python-3.7-cache- - run: name: install ffmpeg command: apt-get update && apt-get install -y ffmpeg - run: - name: install spleeter - command: pip install . + name: install python dependencies + command: pip install -r requirements.txt && pip install pytest + - save_cache: + key: python-3.7-cache-{{ checksum requirements.txt }} + paths: + - "/usr/local/bin" + - "/usr/local/lib/python3.7/site-packages" - run: - name: test separation - command: spleeter separate -i audio_example.mp3 -o . - upload: + name: pytest + command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + pypi-deploy: docker: - image: python:3 steps: @@ -30,11 +63,13 @@ workflows: version: 2 test-and-deploy: jobs: - - test - - upload: + - test-3.6 + - test-3.7 + - pypi-deploy: filters: branches: only: - master requires: - - test \ No newline at end of file + - test-3.6 + - test-3.7 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8e1f7be --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +importlib_resources; python_version<'3.7' +requests +setuptools>=41.0.0 +pandas==0.25.1 +tensorflow==1.14.0 +ffmpeg-python +norbert==0.2.1 \ No newline at end of file From 647a1636352b27f3bff9f967004e7b1a7251ddf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:35:30 -0500 Subject: [PATCH 05/40] feat: add sdist intermediate stage --- .circleci/config.yml | 53 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5b5267..fcb8709 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,8 @@ version: 2 jobs: + # ======================================================================================= + # Python 3.6 testing. + # ======================================================================================= test-3.6: docker: - image: python:3.6 @@ -8,7 +11,8 @@ jobs: - checkout - restore_cache: keys: - - python-3.6-cache-{{ checksum requirements.txt }} + - python-3.6-cache-{{ .branch }}-{{ checksum requirements.txt }} + - python-3.6-cache-{{ .branch }}- - python-3.6-cache- - run: name: install ffmpeg @@ -17,13 +21,17 @@ jobs: name: install python dependencies command: pip install -r requirements.txt && pip install pytest - save_cache: - key: python-3.6-cache-{{ checksum requirements.txt }} + key: python-3.6-cache-{{ .branch }}-{{ checksum requirements.txt }} paths: + - "pretrained_models" - "/usr/local/bin" - "/usr/local/lib/python3.6/site-packages" - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + # ======================================================================================= + # Python 3.7 testing. + # ======================================================================================= test-3.7: docker: - image: python:3.7 @@ -32,7 +40,8 @@ jobs: - checkout - restore_cache: keys: - - python-3.7-cache-{{ checksum requirements.txt }} + - python-3.7-cache-{{ .branch }}-{{ checksum requirements.txt }} + - python-3.7-cache-{{ .branch }}- - python-3.7-cache- - run: name: install ffmpeg @@ -40,15 +49,19 @@ jobs: - run: name: install python dependencies command: pip install -r requirements.txt && pip install pytest - - save_cache: - key: python-3.7-cache-{{ checksum requirements.txt }} - paths: - - "/usr/local/bin" - - "/usr/local/lib/python3.7/site-packages" - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning - pypi-deploy: + - save_cache: + key: python-3.7-cache-{{ .branch }}-{{ checksum requirements.txt }} + paths: + - "pretrained_models" + - "/usr/local/bin" + - "/usr/local/lib/python3.7/site-packages" + # ======================================================================================= + # Source distribution packaging. + # ======================================================================================= + sdist: docker: - image: python:3 steps: @@ -56,6 +69,20 @@ jobs: - run: name: package command: python setup.py sdist + - save_cache: + key: sdist-{{ .branch }}-{{ checksum setup.py }} + paths: + - dist + # ======================================================================================= + # PyPi deployment. + # ======================================================================================= + pypi-deploy: + docker: + - image: python:3 + steps: + - checkout + - restore_cache: + key: sdist-{{ .branch }}-{{ checksum setup.py }} - run: name: upload to PyPi command: pip install twine && twine upload dist/* @@ -65,11 +92,15 @@ workflows: jobs: - test-3.6 - test-3.7 + - sdist: + requires: + - test-3.6 + - test-3.7 - pypi-deploy: filters: branches: only: - master + - development requires: - - test-3.6 - - test-3.7 \ No newline at end of file + - sdist \ No newline at end of file From ebd03c8565eb15e8350c7459080169aecbc2a0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:38:20 -0500 Subject: [PATCH 06/40] fix: cache keys --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcb8709..0a257a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,8 +11,8 @@ jobs: - checkout - restore_cache: keys: - - python-3.6-cache-{{ .branch }}-{{ checksum requirements.txt }} - - python-3.6-cache-{{ .branch }}- + - python-3.6-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} + - python-3.6-cache-{{ .Branch }}- - python-3.6-cache- - run: name: install ffmpeg @@ -21,7 +21,7 @@ jobs: name: install python dependencies command: pip install -r requirements.txt && pip install pytest - save_cache: - key: python-3.6-cache-{{ .branch }}-{{ checksum requirements.txt }} + key: python-3.6-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} paths: - "pretrained_models" - "/usr/local/bin" @@ -40,8 +40,8 @@ jobs: - checkout - restore_cache: keys: - - python-3.7-cache-{{ .branch }}-{{ checksum requirements.txt }} - - python-3.7-cache-{{ .branch }}- + - python-3.7-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} + - python-3.7-cache-{{ .Branch }}- - python-3.7-cache- - run: name: install ffmpeg @@ -53,7 +53,7 @@ jobs: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning - save_cache: - key: python-3.7-cache-{{ .branch }}-{{ checksum requirements.txt }} + key: python-3.7-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} paths: - "pretrained_models" - "/usr/local/bin" @@ -70,7 +70,7 @@ jobs: name: package command: python setup.py sdist - save_cache: - key: sdist-{{ .branch }}-{{ checksum setup.py }} + key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} paths: - dist # ======================================================================================= @@ -82,7 +82,7 @@ jobs: steps: - checkout - restore_cache: - key: sdist-{{ .branch }}-{{ checksum setup.py }} + key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - run: name: upload to PyPi command: pip install twine && twine upload dist/* From 7322b3bdea43241e0f72586edffa0fd8960c5d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:40:45 -0500 Subject: [PATCH 07/40] fix: test iterations --- .circleci/config.yml | 2 +- tests/test_separator.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a257a6..fbad562 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,7 +88,7 @@ jobs: command: pip install twine && twine upload dist/* workflows: version: 2 - test-and-deploy: + spleeter-workflow: jobs: - test-3.6 - test-3.7 diff --git a/tests/test_separator.py b/tests/test_separator.py index 3115314..15493cd 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -25,7 +25,7 @@ def test_separate(): """ Test separation from raw data. """ adapter = get_default_audio_adapter() waveform, _ = adapter.load(TEST_AUDIO_DESCRIPTOR) - for configuration, instruments in TEST_CONFIGURATIONS: + for configuration, instruments in TEST_CONFIGURATIONS.items(): separator = Separator(configuration) prediction = separator.separate(waveform) assert len(prediction) == 2 @@ -35,7 +35,7 @@ def test_separate(): def test_separate_to_file(): """ Test file based separation. """ - for configuration, instruments in TEST_CONFIGURATIONS: + for configuration, instruments in TEST_CONFIGURATIONS.items(): separator = Separator(configuration) with TemporaryDirectory() as directory: separator.separate_to_file( From d311068dfb01c670688aa62f1220c1cb0877b4c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:45:38 -0500 Subject: [PATCH 08/40] fix: model caching --- .circleci/config.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fbad562..b6d4fad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,8 @@ jobs: - python-3.6-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} - python-3.6-cache-{{ .Branch }}- - python-3.6-cache- + - restore_cache: + key: models-{{ checksum "spleeter/model/__init__.py" }} - run: name: install ffmpeg command: apt-get update && apt-get install -y ffmpeg @@ -23,12 +25,14 @@ jobs: - save_cache: key: python-3.6-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} paths: - - "pretrained_models" - - "/usr/local/bin" - - "/usr/local/lib/python3.6/site-packages" + - "/usr" - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + - save_cache: + key: models-{{ checksum "spleeter/model/__init__.py" }} + paths: + - "pretrained_models" # ======================================================================================= # Python 3.7 testing. # ======================================================================================= @@ -43,21 +47,25 @@ jobs: - python-3.7-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} - python-3.7-cache-{{ .Branch }}- - python-3.7-cache- + - restore_cache: + key: models-{{ checksum "spleeter/model/__init__.py" }} - run: name: install ffmpeg command: apt-get update && apt-get install -y ffmpeg - run: name: install python dependencies command: pip install -r requirements.txt && pip install pytest + - save_cache: + key: python-3.7-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} + paths: + - "/usr" - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning - save_cache: - key: python-3.7-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} + key: models-{{ checksum "spleeter/model/__init__.py" }} paths: - "pretrained_models" - - "/usr/local/bin" - - "/usr/local/lib/python3.7/site-packages" # ======================================================================================= # Source distribution packaging. # ======================================================================================= From 05deded08915d8620cdac21250d7aae53c2e7bdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:46:14 -0500 Subject: [PATCH 09/40] fix: separation testing --- tests/test_separator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_separator.py b/tests/test_separator.py index 15493cd..76011af 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -15,7 +15,7 @@ from spleeter.separator import Separator TEST_AUDIO_DESCRIPTOR = 'audio_example.mp3' TEST_CONFIGURATIONS = { - 'spleeter:2stems': ('vocals', 'accompaniament'), + 'spleeter:2stems': ('vocals', 'accompaniment'), 'spleeter:4stems': ('vocals', 'drums', 'bass', 'other'), 'spleeter:5stems': ('vocals', 'drums', 'bass', 'piano', 'other') } From 29afdc32f74ffa813fa3bcc3da86aced98539c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:48:57 -0500 Subject: [PATCH 10/40] fix: remove useless caching --- .circleci/config.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b6d4fad..e379d20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,30 +9,15 @@ jobs: working_directory: ~/spleeter steps: - checkout - - restore_cache: - keys: - - python-3.6-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} - - python-3.6-cache-{{ .Branch }}- - - python-3.6-cache- - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - run: name: install ffmpeg command: apt-get update && apt-get install -y ffmpeg - run: name: install python dependencies command: pip install -r requirements.txt && pip install pytest - - save_cache: - key: python-3.6-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} - paths: - - "/usr" - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" # ======================================================================================= # Python 3.7 testing. # ======================================================================================= @@ -42,30 +27,15 @@ jobs: working_directory: ~/spleeter steps: - checkout - - restore_cache: - keys: - - python-3.7-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} - - python-3.7-cache-{{ .Branch }}- - - python-3.7-cache- - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - run: name: install ffmpeg command: apt-get update && apt-get install -y ffmpeg - run: name: install python dependencies command: pip install -r requirements.txt && pip install pytest - - save_cache: - key: python-3.7-cache-{{ .Branch }}-{{ checksum "requirements.txt" }} - paths: - - "/usr" - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" # ======================================================================================= # Source distribution packaging. # ======================================================================================= From 05571be6610a6084130eaf0c0f6f2e1be2308996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 19:54:33 -0500 Subject: [PATCH 11/40] fix: cache model only --- .circleci/config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e379d20..3e0ac71 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,8 @@ jobs: working_directory: ~/spleeter steps: - checkout + - restore_cache: + key: models-{{ checksum "spleeter/model/__init__.py" }} - run: name: install ffmpeg command: apt-get update && apt-get install -y ffmpeg @@ -18,6 +20,10 @@ jobs: - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + - save_cache: + key: models-{{ checksum "spleeter/model/__init__.py" }} + paths: + - "pretrained_models" # ======================================================================================= # Python 3.7 testing. # ======================================================================================= @@ -27,6 +33,8 @@ jobs: working_directory: ~/spleeter steps: - checkout + - restore_cache: + key: models-{{ checksum "spleeter/model/__init__.py" }} - run: name: install ffmpeg command: apt-get update && apt-get install -y ffmpeg @@ -36,6 +44,10 @@ jobs: - run: name: pytest command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + - save_cache: + key: models-{{ checksum "spleeter/model/__init__.py" }} + paths: + - "pretrained_models" # ======================================================================================= # Source distribution packaging. # ======================================================================================= From e9f1ba9450890ac3e707ad0adc81d2af6f96596c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 20:03:16 -0500 Subject: [PATCH 12/40] fix: test parameters --- tests/test_separator.py | 44 +++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/test_separator.py b/tests/test_separator.py index 76011af..976cf6c 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -10,37 +10,39 @@ __license__ = 'MIT License' from os.path import exists, join from tempfile import TemporaryDirectory +import pytest + from spleeter.audio.adapter import get_default_audio_adapter from spleeter.separator import Separator TEST_AUDIO_DESCRIPTOR = 'audio_example.mp3' -TEST_CONFIGURATIONS = { - 'spleeter:2stems': ('vocals', 'accompaniment'), - 'spleeter:4stems': ('vocals', 'drums', 'bass', 'other'), - 'spleeter:5stems': ('vocals', 'drums', 'bass', 'piano', 'other') -} +TEST_CONFIGURATIONS = [ + ('spleeter:2stems', ('vocals', 'accompaniment')), + ('spleeter:4stems', ('vocals', 'drums', 'bass', 'other')), + ('spleeter:5stems', ('vocals', 'drums', 'bass', 'piano', 'other')) +] +@pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS) def test_separate(): """ Test separation from raw data. """ adapter = get_default_audio_adapter() waveform, _ = adapter.load(TEST_AUDIO_DESCRIPTOR) - for configuration, instruments in TEST_CONFIGURATIONS.items(): - separator = Separator(configuration) - prediction = separator.separate(waveform) - assert len(prediction) == 2 - for instrument in instruments: - assert instrument in prediction + separator = Separator(configuration) + prediction = separator.separate(waveform) + assert len(prediction) == 2 + for instrument in instruments: + assert instrument in prediction -def test_separate_to_file(): +@pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS) +def test_separate_to_file(configuration, instruments): """ Test file based separation. """ - for configuration, instruments in TEST_CONFIGURATIONS.items(): - separator = Separator(configuration) - with TemporaryDirectory() as directory: - separator.separate_to_file( - TEST_AUDIO_DESCRIPTOR, - directory) - for instrument in instruments: - assert exists(join(directory, '{}.wav'.format(instrument))) - # TODO: Consider testing generated file as well. + separator = Separator(configuration) + with TemporaryDirectory() as directory: + separator.separate_to_file( + TEST_AUDIO_DESCRIPTOR, + directory) + for instrument in instruments: + assert exists(join(directory, '{}.wav'.format(instrument))) + # TODO: Consider testing generated file as well. From 7773d1df2702a68f5dced7e92e3a7b9157c35ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 20:04:44 -0500 Subject: [PATCH 13/40] fix: function spec error --- tests/test_separator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_separator.py b/tests/test_separator.py index 976cf6c..5358d06 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -24,7 +24,7 @@ TEST_CONFIGURATIONS = [ @pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS) -def test_separate(): +def test_separate(configuration, instruments): """ Test separation from raw data. """ adapter = get_default_audio_adapter() waveform, _ = adapter.load(TEST_AUDIO_DESCRIPTOR) From 6d5398bda70682fc64d3d826f0f0c9b3c3ee1dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 20:11:03 -0500 Subject: [PATCH 14/40] fix: add verbose test for avoiding timeout --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e0ac71..d20f72a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: command: pip install -r requirements.txt && pip install pytest - run: name: pytest - command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} paths: @@ -43,7 +43,7 @@ jobs: command: pip install -r requirements.txt && pip install pytest - run: name: pytest - command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning + command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} paths: From e912ae0fcecb26f5751d0c47511ea10ddd8260dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 8 Nov 2019 20:21:37 -0500 Subject: [PATCH 15/40] fix: use xdist testing --- .circleci/config.yml | 8 ++++---- tests/test_separator.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d20f72a..5e7368a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,10 +16,10 @@ jobs: command: apt-get update && apt-get install -y ffmpeg - run: name: install python dependencies - command: pip install -r requirements.txt && pip install pytest + command: pip install -r requirements.txt && pip install pytest pytest-xdist - run: name: pytest - command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv + command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} paths: @@ -40,10 +40,10 @@ jobs: command: apt-get update && apt-get install -y ffmpeg - run: name: install python dependencies - command: pip install -r requirements.txt && pip install pytest + command: pip install -r requirements.txt && pip install pytest pytest-xdist - run: name: pytest - command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv + command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} paths: diff --git a/tests/test_separator.py b/tests/test_separator.py index 5358d06..54c9c18 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -30,7 +30,7 @@ def test_separate(configuration, instruments): waveform, _ = adapter.load(TEST_AUDIO_DESCRIPTOR) separator = Separator(configuration) prediction = separator.separate(waveform) - assert len(prediction) == 2 + assert len(prediction) == len(instruments) for instrument in instruments: assert instrument in prediction From 4a743cd61871667a996acf3115dd0b8da391ee47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 14 Nov 2019 13:03:55 -0500 Subject: [PATCH 16/40] refactor: use separator feat: add bitrate opt feat: add offset opt --- .circleci/config.yml | 21 ++++- setup.py | 2 +- spleeter/commands/__init__.py | 52 ++++++++---- spleeter/commands/separate.py | 154 ++++------------------------------ 4 files changed, 71 insertions(+), 158 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e7368a..c8cdce3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -75,7 +75,27 @@ jobs: key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - run: name: upload to PyPi + # TODO: Infer destination regarding of branch. + # - master => production PyPi + # - development => testing PyPi command: pip install twine && twine upload dist/* + + # ======================================================================================= + # Conda distribution. + # ======================================================================================= + conda-deploy: + docker: + - image: null # TODO: use circle ci python image. + steps: + - run: + name: checkout feedstock + command: git clone https://github.com/conda-forge/spleeter-feedstock + - run: + name: update version + command: sed -i 's///g' + - run: + name: deploy + command: git add recipe && git commit -m "" && git push workflows: version: 2 spleeter-workflow: @@ -91,6 +111,5 @@ workflows: branches: only: - master - - development requires: - sdist \ No newline at end of file diff --git a/setup.py b/setup.py index b9abf78..c4665e6 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ __license__ = 'MIT License' # Default project values. project_name = 'spleeter' -project_version = '1.4.3' +project_version = '1.4.4' device_target = 'cpu' tensorflow_dependency = 'tensorflow' tensorflow_version = '1.14.0' diff --git a/spleeter/commands/__init__.py b/spleeter/commands/__init__.py index 331ee2d..6428d6d 100644 --- a/spleeter/commands/__init__.py +++ b/spleeter/commands/__init__.py @@ -13,31 +13,31 @@ __email__ = 'research@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' -# -i opt specification. +# -i opt specification (separate). OPT_INPUT = { - 'dest': 'audio_filenames', + 'dest': 'inputs', 'nargs': '+', 'help': 'List of input audio filenames', 'required': True } -# -o opt specification. +# -o opt specification (evaluate and separate). OPT_OUTPUT = { 'dest': 'output_path', 'default': join(gettempdir(), 'separated_audio'), 'help': 'Path of the output directory to write audio files in' } -# -p opt specification. +# -p opt specification (train, evaluate and separate). OPT_PARAMS = { - 'dest': 'params_filename', + 'dest': 'configuration', 'default': 'spleeter:2stems', 'type': str, 'action': 'store', 'help': 'JSON filename that contains params' } -# -n opt specification. +# -n opt specification (separate). OPT_OUTPUT_NAMING = { 'dest': 'output_naming', 'default': 'filename', @@ -54,26 +54,42 @@ OPT_OUTPUT_NAMING = { ', /audio/.wav)') } +# -s opt specification (separate). +OPT_OFFSET = { + 'dest': 'offset', + 'type': float, + 'default': 0., + 'help': 'Set the starting offset to separate audio from.' +} + # -d opt specification (separate). OPT_DURATION = { - 'dest': 'max_duration', + 'dest': 'duration', 'type': float, 'default': 600., 'help': ( 'Set a maximum duration for processing audio ' - '(only separate max_duration first seconds of ' + '(only separate offset + duration first seconds of ' 'the input file)') } -# -c opt specification. +# -c opt specification (separate). OPT_CODEC = { - 'dest': 'audio_codec', + 'dest': 'codec', 'choices': ('wav', 'mp3', 'ogg', 'm4a', 'wma', 'flac'), 'default': 'wav', 'help': 'Audio codec to be used for the separated output' } -# -m opt specification. +# -b opt specification (separate). +OPT_BITRATE = { + 'dest': 'bitrate', + 'type': int, + 'default': '128k', + 'help': 'Audio bitrate to be used for the separated output' +} + +# -m opt specification (evaluate and separate). OPT_MWF = { 'dest': 'MWF', 'action': 'store_const', @@ -82,7 +98,7 @@ OPT_MWF = { 'help': 'Whether to use multichannel Wiener filtering for separation', } -# --mus_dir opt specification. +# --mus_dir opt specification (evaluate). OPT_MUSDB = { 'dest': 'mus_dir', 'type': str, @@ -98,14 +114,14 @@ OPT_DATA = { 'help': 'Path of the folder containing audio data for training' } -# -a opt specification. +# -a opt specification (train, evaluate and separate). OPT_ADAPTER = { 'dest': 'audio_adapter', 'type': str, 'help': 'Name of the audio adapter to use for audio I/O' } -# -a opt specification. +# -a opt specification (train, evaluate and separate). OPT_VERBOSE = { 'action': 'store_true', 'help': 'Shows verbose logs' @@ -158,11 +174,13 @@ def _create_separate_parser(parser_factory): """ parser = parser_factory('separate', help='Separate audio files') _add_common_options(parser) - parser.add_argument('-i', '--audio_filenames', **OPT_INPUT) + parser.add_argument('-i', '--inputs', **OPT_INPUT) parser.add_argument('-o', '--output_path', **OPT_OUTPUT) parser.add_argument('-n', '--output_naming', **OPT_OUTPUT_NAMING) - parser.add_argument('-d', '--max_duration', **OPT_DURATION) - parser.add_argument('-c', '--audio_codec', **OPT_CODEC) + parser.add_ + parser.add_argument('-d', '--duration', **OPT_DURATION) + parser.add_argument('-c', '--codec', **OPT_CODEC) + parser.add_argument('-b', '--birate', **OPT_BITRATE) parser.add_argument('-m', '--mwf', **OPT_MWF) return parser diff --git a/spleeter/commands/separate.py b/spleeter/commands/separate.py index 10da43c..df88229 100644 --- a/spleeter/commands/separate.py +++ b/spleeter/commands/separate.py @@ -22,6 +22,7 @@ import numpy as np from ..audio.adapter import get_audio_adapter from ..audio.convertor import to_n_channels +from ..separator import Separator from ..utils.estimator import create_estimator from ..utils.tensor import set_tensor_shape @@ -30,149 +31,24 @@ __author__ = 'Deezer Research' __license__ = 'MIT License' -def get_dataset(audio_adapter, filenames_and_crops, sample_rate, n_channels): - """" - Build a tensorflow dataset of waveform from a filename list wit crop - information. - - Params: - - audio_adapter: An AudioAdapter instance to load audio from. - - filenames_and_crops: list of (audio_filename, start, duration) - tuples separation is performed on each filaneme - from start (in seconds) to start + duration - (in seconds). - - sample_rate: audio sample_rate of the input and output audio - signals - - n_channels: int, number of channels of the input and output - audio signals - - Returns - A tensorflow dataset of waveform to feed a tensorflow estimator in - predict mode. - """ - filenames, starts, ends = list(zip(*filenames_and_crops)) - dataset = tf.data.Dataset.from_tensor_slices({ - 'audio_id': list(filenames), - 'start': list(starts), - 'end': list(ends) - }) - # Load waveform. - dataset = dataset.map( - lambda sample: dict( - sample, - **audio_adapter.load_tf_waveform( - sample['audio_id'], - sample_rate=sample_rate, - offset=sample['start'], - duration=sample['end'] - sample['start'])), - num_parallel_calls=2) - # Filter out error. - dataset = dataset.filter( - lambda sample: tf.logical_not(sample['waveform_error'])) - # Convert waveform to the right number of channels. - dataset = dataset.map( - lambda sample: dict( - sample, - waveform=to_n_channels(sample['waveform'], n_channels))) - # Set number of channels (required for the model). - dataset = dataset.map( - lambda sample: dict( - sample, - waveform=set_tensor_shape(sample['waveform'], (None, n_channels)))) - return dataset - - -def process_audio( - audio_adapter, - filenames_and_crops, estimator, output_path, - sample_rate, n_channels, codec, output_naming): - """ - Perform separation on a list of audio ids. - - Params: - - audio_adapter: Audio adapter to use for audio I/O. - - filenames_and_crops: list of (audio_filename, start, duration) - tuples separation is performed on each filaneme - from start (in seconds) to start + duration - (in seconds). - - estimator: the tensorflow estimator that performs the - source separation. - - output_path: output_path where to export separated files. - - sample_rate: audio sample_rate of the input and output audio - signals - - n_channels: int, number of channels of the input and output - audio signals - - codec: string codec to be used for export (could be - "wav", "mp3", "ogg", "m4a") could be anything - supported by ffmpeg. - - output_naming: string (= "filename" of "directory") - naming convention for output. - for an input file /path/to/audio/input_file.wav: - * if output_naming is equal to "filename": - output files will be put in the directory /input_file - (/input_file/., - /input_file/....). - * if output_naming is equal to "directory": - output files will be put in the directory /audio/ - (/audio/., - /audio/....) - Use "directory" when separating the MusDB dataset. - - """ - # Get estimator - prediction = estimator.predict( - lambda: get_dataset( - audio_adapter, - filenames_and_crops, - sample_rate, - n_channels), - yield_single_examples=False) - # initialize pool for audio export - pool = Pool(16) - for sample in prediction: - sample_filename = sample.pop('audio_id', 'unknown_filename').decode() - input_directory, input_filename = split(sample_filename) - if output_naming == 'directory': - output_dirname = split(input_directory)[1] - elif output_naming == 'filename': - output_dirname = splitext(input_filename)[0] - else: - raise ValueError(f'Unknown output naming {output_naming}') - for instrument, waveform in sample.items(): - filename = join( - output_path, - output_dirname, - f'{instrument}.{codec}') - pool.apply_async( - audio_adapter.save, - (filename, waveform, sample_rate, codec)) - # Wait for everything to be written - pool.close() - pool.join() - - def entrypoint(arguments, params): """ Command entrypoint. :param arguments: Command line parsed argument as argparse.Namespace. :param params: Deserialized JSON configuration file provided in CLI args. """ + # TODO: check with output naming. audio_adapter = get_audio_adapter(arguments.audio_adapter) - filenames = arguments.audio_filenames - output_path = arguments.output_path - max_duration = arguments.max_duration - audio_codec = arguments.audio_codec - output_naming = arguments.output_naming - estimator = create_estimator(params, arguments.MWF) - filenames_and_crops = [ - (filename, 0., max_duration) - for filename in filenames] - process_audio( - audio_adapter, - filenames_and_crops, - estimator, - output_path, - params['sample_rate'], - params['n_channels'], - codec=audio_codec, - output_naming=output_naming) + separator = Separator(arguments.configuration, arguments.MWF) + for filename in arguments.audio_filenames: + separator.separate_to_file( + filename, + arguments.output_path, + audio_adapter=audio_adapter, + offset=arguments.offset, + duration=arguments.max_duration, + codec=arguments.codec, + bitrate=arguments.bitrate, + synchronous=False + ) + separator.join() From 24334446836e621c2528a8b7c7557847ed9563a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 14 Nov 2019 13:04:40 -0500 Subject: [PATCH 17/40] fix: disable conda deployment --- .circleci/config.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8cdce3..5be77c7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,19 +83,19 @@ jobs: # ======================================================================================= # Conda distribution. # ======================================================================================= - conda-deploy: - docker: - - image: null # TODO: use circle ci python image. - steps: - - run: - name: checkout feedstock - command: git clone https://github.com/conda-forge/spleeter-feedstock - - run: - name: update version - command: sed -i 's///g' - - run: - name: deploy - command: git add recipe && git commit -m "" && git push + #conda-deploy: + # docker: + # - image: null # TODO: use circle ci python image. + # steps: + # - run: + # name: checkout feedstock + # command: git clone https://github.com/conda-forge/spleeter-feedstock + # - run: + # name: update version + # command: sed -i 's///g' + # - run: + # name: deploy + # command: git add recipe && git commit -m "" && git push workflows: version: 2 spleeter-workflow: From da3947e49d549209ed15b92b534453d2ad008e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 14 Nov 2019 13:16:14 -0500 Subject: [PATCH 18/40] docs: add gitter badge --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84494d7..fde7987 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ -[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) +[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) + + ## About @@ -28,8 +30,7 @@ environment to start separating audio file as follows: ```bash git clone https://github.com/Deezer/spleeter -conda env create -f spleeter/conda/spleeter-cpu.yaml -conda activate spleeter-cpu +conda install -c conda-forge spleeter spleeter separate -i spleeter/audio_example.mp3 -p spleeter:2stems -o output ``` You should get two separated audio files (`vocals.wav` and `accompaniment.wav`) From ff539bd26cd7ce2d2a9467239880e7fce836cbf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 14 Nov 2019 14:44:19 -0500 Subject: [PATCH 19/40] refactor: Makefile test: improve separator test test: start pypi dist test --- .circleci/config.yml | 14 +++++++------- Makefile | 23 +++++++++++++++-------- tests/test_pypi_sdist.sh | 10 ++++++++++ tests/test_separator.py | 15 ++++++++++++++- 4 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 tests/test_pypi_sdist.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 5be77c7..244ff1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -18,8 +18,8 @@ jobs: name: install python dependencies command: pip install -r requirements.txt && pip install pytest pytest-xdist - run: - name: pytest - command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked + name: run tests + command: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} paths: @@ -42,8 +42,8 @@ jobs: name: install python dependencies command: pip install -r requirements.txt && pip install pytest pytest-xdist - run: - name: pytest - command: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked + name: run tests + command: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} paths: @@ -57,8 +57,8 @@ jobs: steps: - checkout - run: - name: package - command: python setup.py sdist + name: package source distribution + command: make build - save_cache: key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} paths: @@ -78,7 +78,7 @@ jobs: # TODO: Infer destination regarding of branch. # - master => production PyPi # - development => testing PyPi - command: pip install twine && twine upload dist/* + command: make deploy # ======================================================================================= # Conda distribution. diff --git a/Makefile b/Makefile index 989f5ce..c2019ce 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # ======================================================= -# Build script for distribution packaging. +# Library lifecycle management. # # @author Deezer Research # @licence MIT Licence @@ -9,22 +9,29 @@ clean: rm -Rf *.egg-info rm -Rf dist + build: @echo "=== Build CPU bdist package" - @python3 setup.py sdist + python3 setup.py sdist @echo "=== CPU version checksum" @openssl sha256 dist/*.tar.gz build-gpu: @echo "=== Build GPU bdist package" - @python3 setup.py sdist --target gpu + python3 setup.py sdist --target gpu @echo "=== GPU version checksum" @openssl sha256 dist/*.tar.gz -upload: +pip-dependencies: + pip install twine + +test: pip-dependencies + pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked + +test-distribution: pip-dependencies + bash tests/test_pypi_sdist.sh + +deploy: pip-dependencies twine upload dist/* -test-upload: - twine upload --repository-url https://test.pypi.org/legacy/ dist/* - -all: clean build build-gpu upload \ No newline at end of file +all: clean test build build-gpu upload diff --git a/tests/test_pypi_sdist.sh b/tests/test_pypi_sdist.sh new file mode 100644 index 0000000..c514062 --- /dev/null +++ b/tests/test_pypi_sdist.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +###################################################################### +# Script that performs PyPi packaging test. +# +# @author Deezer Research +# @version 1.0.0 +###################################################################### + + twine upload --repository-url https://test.pypi.org/legacy/ dist/* \ No newline at end of file diff --git a/tests/test_separator.py b/tests/test_separator.py index 54c9c18..c570e64 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -7,6 +7,8 @@ __email__ = 'research@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' +import filecmp + from os.path import exists, join from tempfile import TemporaryDirectory @@ -33,6 +35,12 @@ def test_separate(configuration, instruments): assert len(prediction) == len(instruments) for instrument in instruments: assert instrument in prediction + for instrument in instruments: + track = prediction[instrument] + assert not (waveform == track).all() + for compared in instruments: + if instrument != compared: + assert not (track == prediction[compared]).all() @pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS) @@ -45,4 +53,9 @@ def test_separate_to_file(configuration, instruments): directory) for instrument in instruments: assert exists(join(directory, '{}.wav'.format(instrument))) - # TODO: Consider testing generated file as well. + for instrument in instruments: + for compared in instrument: + if instrument != compared: + assert not filecmp.cmp( + join(directory, '{}.wav'.format(instrument)), + join(directory, '{}.wav'.format(compared))) From 2c6c927f4718595f31d7c4e951c14596130aa2e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 11:27:34 +0100 Subject: [PATCH 20/40] fix: conda deploy --- .circleci/config.yml | 34 +++++++++++++++++++--------------- .gitignore | 3 ++- Makefile | 41 ++++++++++++++++++++++------------------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 244ff1a..0cab9bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,25 +77,22 @@ jobs: name: upload to PyPi # TODO: Infer destination regarding of branch. # - master => production PyPi - # - development => testing PyPi + # - other => testing PyPi command: make deploy # ======================================================================================= # Conda distribution. # ======================================================================================= - #conda-deploy: - # docker: - # - image: null # TODO: use circle ci python image. - # steps: - # - run: - # name: checkout feedstock - # command: git clone https://github.com/conda-forge/spleeter-feedstock - # - run: - # name: update version - # command: sed -i 's///g' - # - run: - # name: deploy - # command: git add recipe && git commit -m "" && git push + conda-forge-deploy: + docker: + - image: python:3 + steps: + - run: + name: install dependencies + command: apt-get update && apt-get install -y git openssl hub + - run: + name: checkout feedstock + command: make feedstock workflows: version: 2 spleeter-workflow: @@ -112,4 +109,11 @@ workflows: only: - master requires: - - sdist \ No newline at end of file + - sdist + - conda-forge-deploy: + filters: + branches: + only: + - master + requires: + - pypi-deploy diff --git a/.gitignore b/.gitignore index 29d7036..3660fc0 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,5 @@ __pycache__ pretrained_models docs/build -.vscode \ No newline at end of file +.vscode +spleeter-feedstock/ \ No newline at end of file diff --git a/Makefile b/Makefile index c2019ce..6bde98d 100644 --- a/Makefile +++ b/Makefile @@ -5,33 +5,36 @@ # @licence MIT Licence # ======================================================= +FFEDSTOCK = spleeter-feedstock +FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK) +FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml + +all: clean build test deploy + clean: rm -Rf *.egg-info rm -Rf dist - build: - @echo "=== Build CPU bdist package" python3 setup.py sdist - @echo "=== CPU version checksum" - @openssl sha256 dist/*.tar.gz -build-gpu: - @echo "=== Build GPU bdist package" - python3 setup.py sdist --target gpu - @echo "=== GPU version checksum" - @openssl sha256 dist/*.tar.gz - -pip-dependencies: - pip install twine - -test: pip-dependencies +test: pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked -test-distribution: pip-dependencies - bash tests/test_pypi_sdist.sh +feedstock: build + $(eval VERSION = $(shell grep 'project_version = ' setup.py | cut -d' ' -f3 | sed "s/'//g")) + $(eval CHECKSUM = $(shell openssl sha256 dist/spleeter-$(VERSION).tar.gz | cut -d' ' -f2)) + git clone $(FEEDSTOCK_REPOSITORY) + sed 's/{% set version = "[0-9]*\.[0-9]*\.[0-9]*" %}/{% set version = "$(VERSION)" %}/g' $(FEEDSTOCK_RECIPE) + sed 's/sha256: [0-9a-z]*/sha: $(CHECKSUM)/g' $(FEEDSTOCK_RECIPE) + git config credential.helper 'cache --timeout=120' + git config user.email "research@deezer.com" + git config user.name "spleeter-ci" + git add recipe/spleeter/meta.yaml + git commit --allow-empty -m "feat: update spleeter version from CI" + git push -q https://$$FEEDSTOCK_TOKEN@github.com/deezer/$(FEEDSTOCK) + hub pull-request -m "Update spleeter version to $(VERSION)" deploy: pip-dependencies - twine upload dist/* - -all: clean test build build-gpu upload + pip install twine + twine upload dist/* \ No newline at end of file From b3f5085d5c30150f532b7a58362ee22e432149df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 11:50:57 +0100 Subject: [PATCH 21/40] feat: add docker workflow --- .circleci/config.yml | 211 +++++++++++++++++- ...ems.dockerfile => conda-2stems.dockerfile} | 0 ...ems.dockerfile => conda-4stems.dockerfile} | 0 ...ems.dockerfile => conda-5stems.dockerfile} | 0 docker/cpu/conda.dockerfile | 7 +- 5 files changed, 210 insertions(+), 8 deletions(-) rename docker/cpu/{conda-2-stems.dockerfile => conda-2stems.dockerfile} (100%) rename docker/cpu/{conda-4-stems.dockerfile => conda-4stems.dockerfile} (100%) rename docker/cpu/{conda-5-stems.dockerfile => conda-5stems.dockerfile} (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0cab9bd..0443970 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,6 @@ jobs: # - master => production PyPi # - other => testing PyPi command: make deploy - # ======================================================================================= # Conda distribution. # ======================================================================================= @@ -93,9 +92,128 @@ jobs: - run: name: checkout feedstock command: make feedstock + # ======================================================================================= + # Docker build. + # ======================================================================================= + docker-conda-cpu: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:conda -f docker/cpu/conda.dockerfile . + docker-conda-cpu-2stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:2-stems-conda -f docker/cpu/conda-2stems.dockerfile . + docker-conda-cpu-4stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:4-stems-conda -f docker/cpu/conda-4stems.dockerfile . + docker-conda-cpu-5stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:5-stems-conda -f docker/cpu/conda-5stems.dockerfile . + docker-3.6-cpu: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6.dockerfile . + docker-3.6-cpu-2stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6-2stems.dockerfile . + docker-3.6-cpu-4stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6-4stems.dockerfile . + docker-3.6-cpu-5stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6-5stems.dockerfile . + docker-3.7-cpu: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7.dockerfile . + docker-3.7-cpu-2stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7-2stems.dockerfile . + docker-3.7-cpu-4stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7-4stems.dockerfile . + docker-3.7-cpu-5stems: + docker: + - image: docker:17.05.0-ce-git + steps: + - setup_remote_docker: + docker_layer_caching: true + - run: + name: build image + command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7-5stems.dockerfile . workflows: version: 2 - spleeter-workflow: + development-workflow: + jobs: + - test-3.6 + - test-3.7 + - sdist: + requires: + - test-3.6 + - test-3.7 + release-workflow: jobs: - test-3.6 - test-3.7 @@ -117,3 +235,92 @@ workflows: - master requires: - pypi-deploy + - hold: + type: approval + requires: + - pypi-deploy + - conda-forge-deploy + - docker-conda-cpu: + requires: + - hold + filters: + branches: + only: + - master + - docker-conda-cpu-2stems: + requires: + - docker-conda-cpu + filters: + branches: + only: + - master + - docker-conda-cpu-4stems: + requires: + - docker-conda-cpu + filters: + branches: + only: + - master + - docker-conda-cpu-5stems: + requires: + - docker-conda-cpu + filters: + branches: + only: + - master + - docker-3.6-cpu: + requires: + - hold + filters: + branches: + only: + - master + - docker-3.6-cpu-2stems: + requires: + - docker-3.6-cpu + filters: + branches: + only: + - master + - docker-3.6-cpu-4stems: + requires: + - docker-3.6-cpu + filters: + branches: + only: + - master + - docker-3.6-cpu-5stems: + requires: + - docker-3.6-cpu + filters: + branches: + only: + - master + - docker-3.7-cpu: + requires: + - hold + filters: + branches: + only: + - master + - docker-3.7-cpu-2stems: + requires: + - docker-3.7-cpu + filters: + branches: + only: + - master + - docker-3.7-cpu-4stems: + requires: + - docker-3.7-cpu + filters: + branches: + only: + - master + - docker-3.7-cpu-5stems: + requires: + - docker-3.7-cpu + filters: + branches: + only: + - master \ No newline at end of file diff --git a/docker/cpu/conda-2-stems.dockerfile b/docker/cpu/conda-2stems.dockerfile similarity index 100% rename from docker/cpu/conda-2-stems.dockerfile rename to docker/cpu/conda-2stems.dockerfile diff --git a/docker/cpu/conda-4-stems.dockerfile b/docker/cpu/conda-4stems.dockerfile similarity index 100% rename from docker/cpu/conda-4-stems.dockerfile rename to docker/cpu/conda-4stems.dockerfile diff --git a/docker/cpu/conda-5-stems.dockerfile b/docker/cpu/conda-5stems.dockerfile similarity index 100% rename from docker/cpu/conda-5-stems.dockerfile rename to docker/cpu/conda-5stems.dockerfile diff --git a/docker/cpu/conda.dockerfile b/docker/cpu/conda.dockerfile index a9b9d45..133273a 100644 --- a/docker/cpu/conda.dockerfile +++ b/docker/cpu/conda.dockerfile @@ -1,12 +1,7 @@ FROM continuumio/miniconda3:4.7.10 -RUN conda install -y ipython \ - && conda install -y tensorflow==1.14.0 \ - && conda install -y -c conda-forge ffmpeg \ - && conda install -y -c conda-forge libsndfile \ - && conda install -y -c anaconda pandas==0.25.1 \ +RUN conda install -y -c conda-forge spleeter RUN mkdir -p /model ENV MODEL_PATH /model -RUN pip install spleeter ENTRYPOINT ["spleeter"] \ No newline at end of file From de30d2929c39c8c43a2c19891bddfc4a460bd83e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 11:52:53 +0100 Subject: [PATCH 22/40] fix: update docker file naming --- .circleci/config.yml | 14 +++++++------- docker/cpu/conda-2stems.dockerfile | 2 +- docker/cpu/conda-4stems.dockerfile | 2 +- docker/cpu/conda-5stems.dockerfile | 2 +- docker/cpu/python-3.6-2stems.dockerfile | 2 +- docker/cpu/python-3.6-4stems.dockerfile | 2 +- docker/cpu/python-3.6-5stems.dockerfile | 2 +- docker/cpu/python-3.7-2stems.dockerfile | 2 +- docker/cpu/python-3.7-4stems.dockerfile | 2 +- docker/cpu/python-3.7-5stems.dockerfile | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0443970..845c25e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: docker_layer_caching: true - run: name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6-2stems.dockerfile . + command: docker build -t spleeter:3.6-2stems -f docker/cpu/python-3.6-2stems.dockerfile . docker-3.6-cpu-4stems: docker: - image: docker:17.05.0-ce-git @@ -157,7 +157,7 @@ jobs: docker_layer_caching: true - run: name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6-4stems.dockerfile . + command: docker build -t spleeter:3.6-4stems -f docker/cpu/python-3.6-4stems.dockerfile . docker-3.6-cpu-5stems: docker: - image: docker:17.05.0-ce-git @@ -166,7 +166,7 @@ jobs: docker_layer_caching: true - run: name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6-5stems.dockerfile . + command: docker build -t spleeter:3.6-5stems -f docker/cpu/python-3.6-5stems.dockerfile . docker-3.7-cpu: docker: - image: docker:17.05.0-ce-git @@ -175,7 +175,7 @@ jobs: docker_layer_caching: true - run: name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7.dockerfile . + command: docker build -t spleeter:3.7 -f docker/cpu/python-3.7.dockerfile . docker-3.7-cpu-2stems: docker: - image: docker:17.05.0-ce-git @@ -184,7 +184,7 @@ jobs: docker_layer_caching: true - run: name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7-2stems.dockerfile . + command: docker build -t spleeter:3.7-2stems -f docker/cpu/python-3.7-2stems.dockerfile . docker-3.7-cpu-4stems: docker: - image: docker:17.05.0-ce-git @@ -193,7 +193,7 @@ jobs: docker_layer_caching: true - run: name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7-4stems.dockerfile . + command: docker build -t spleeter:3.7-4stems -f docker/cpu/python-3.7-4stems.dockerfile . docker-3.7-cpu-5stems: docker: - image: docker:17.05.0-ce-git @@ -202,7 +202,7 @@ jobs: docker_layer_caching: true - run: name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.7-5stems.dockerfile . + command: docker build -t spleeter:3.7-5stems -f docker/cpu/python-3.7-5stems.dockerfile . workflows: version: 2 development-workflow: diff --git a/docker/cpu/conda-2stems.dockerfile b/docker/cpu/conda-2stems.dockerfile index bf7e33a..c38a9ca 100644 --- a/docker/cpu/conda-2stems.dockerfile +++ b/docker/cpu/conda-2stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:conda +FROM spleeter:conda RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ diff --git a/docker/cpu/conda-4stems.dockerfile b/docker/cpu/conda-4stems.dockerfile index f91fdf4..e53fce5 100644 --- a/docker/cpu/conda-4stems.dockerfile +++ b/docker/cpu/conda-4stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:conda +FROM spleeter:conda RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ diff --git a/docker/cpu/conda-5stems.dockerfile b/docker/cpu/conda-5stems.dockerfile index 2fac5dd..b1cbc3a 100644 --- a/docker/cpu/conda-5stems.dockerfile +++ b/docker/cpu/conda-5stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:conda +FROM spleeter:conda RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ diff --git a/docker/cpu/python-3.6-2stems.dockerfile b/docker/cpu/python-3.6-2stems.dockerfile index b43e785..2a2012d 100644 --- a/docker/cpu/python-3.6-2stems.dockerfile +++ b/docker/cpu/python-3.6-2stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:3.6 +FROM spleeter:3.6 RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ diff --git a/docker/cpu/python-3.6-4stems.dockerfile b/docker/cpu/python-3.6-4stems.dockerfile index 9708880..4524de0 100644 --- a/docker/cpu/python-3.6-4stems.dockerfile +++ b/docker/cpu/python-3.6-4stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:3.6 +FROM spleeter:3.6 RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ diff --git a/docker/cpu/python-3.6-5stems.dockerfile b/docker/cpu/python-3.6-5stems.dockerfile index 57f3b90..1eaf104 100644 --- a/docker/cpu/python-3.6-5stems.dockerfile +++ b/docker/cpu/python-3.6-5stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:3.6 +FROM spleeter:3.6 RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ diff --git a/docker/cpu/python-3.7-2stems.dockerfile b/docker/cpu/python-3.7-2stems.dockerfile index 7e9f2ad..748cd44 100644 --- a/docker/cpu/python-3.7-2stems.dockerfile +++ b/docker/cpu/python-3.7-2stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:3.7 +FROM spleeter:3.7 RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ diff --git a/docker/cpu/python-3.7-4stems.dockerfile b/docker/cpu/python-3.7-4stems.dockerfile index 1ccb911..4f8ce06 100644 --- a/docker/cpu/python-3.7-4stems.dockerfile +++ b/docker/cpu/python-3.7-4stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:3.7 +FROM spleeter:3.7 RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ diff --git a/docker/cpu/python-3.7-5stems.dockerfile b/docker/cpu/python-3.7-5stems.dockerfile index a69611c..b5dd68b 100644 --- a/docker/cpu/python-3.7-5stems.dockerfile +++ b/docker/cpu/python-3.7-5stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:3.7 +FROM spleeter:3.7 RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ From 326fc323d10f051d14e4d0985c9328292ff15622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 11:54:24 +0100 Subject: [PATCH 23/40] fix: worflow --- .circleci/config.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 845c25e..16a9ff4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,15 +205,7 @@ jobs: command: docker build -t spleeter:3.7-5stems -f docker/cpu/python-3.7-5stems.dockerfile . workflows: version: 2 - development-workflow: - jobs: - - test-3.6 - - test-3.7 - - sdist: - requires: - - test-3.6 - - test-3.7 - release-workflow: + spleeter-release-pipeline: jobs: - test-3.6 - test-3.7 @@ -240,6 +232,10 @@ workflows: requires: - pypi-deploy - conda-forge-deploy + filters: + branches: + only: + - master - docker-conda-cpu: requires: - hold From 584c39908e16fa1fc8340f24c2fd29427f583eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 11:56:58 +0100 Subject: [PATCH 24/40] fix: remove deprecated dockerfile --- docker/{cpu.Dockerfile => cpu/cpu.dockerfile} | 0 docker/{gpu.Dockerfile => gpu/gpu.dockerfile} | 0 docker/{ => gpu}/install_miniconda.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename docker/{cpu.Dockerfile => cpu/cpu.dockerfile} (100%) rename docker/{gpu.Dockerfile => gpu/gpu.dockerfile} (100%) rename docker/{ => gpu}/install_miniconda.sh (100%) diff --git a/docker/cpu.Dockerfile b/docker/cpu/cpu.dockerfile similarity index 100% rename from docker/cpu.Dockerfile rename to docker/cpu/cpu.dockerfile diff --git a/docker/gpu.Dockerfile b/docker/gpu/gpu.dockerfile similarity index 100% rename from docker/gpu.Dockerfile rename to docker/gpu/gpu.dockerfile diff --git a/docker/install_miniconda.sh b/docker/gpu/install_miniconda.sh similarity index 100% rename from docker/install_miniconda.sh rename to docker/gpu/install_miniconda.sh From c2f412b1d21a4283a7f585d67aef038e278100b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 11:58:48 +0100 Subject: [PATCH 25/40] fix: syntax error --- tests/test_separator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_separator.py b/tests/test_separator.py index c570e64..d2648c2 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -54,7 +54,7 @@ def test_separate_to_file(configuration, instruments): for instrument in instruments: assert exists(join(directory, '{}.wav'.format(instrument))) for instrument in instruments: - for compared in instrument: + for compared in instruments: if instrument != compared: assert not filecmp.cmp( join(directory, '{}.wav'.format(instrument)), From ab099f2256ed90fef5948f4ca1bb8d3c0001c208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 12:11:47 +0100 Subject: [PATCH 26/40] fix: move to chunk based writing --- spleeter/model/provider/github.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/spleeter/model/provider/github.py b/spleeter/model/provider/github.py index aad0c44..3afecd1 100644 --- a/spleeter/model/provider/github.py +++ b/spleeter/model/provider/github.py @@ -18,7 +18,6 @@ import tarfile from os import environ from tempfile import TemporaryFile -from shutil import copyfileobj import requests @@ -60,14 +59,18 @@ class GithubModelProvider(ModelProvider): self._release, name) get_logger().info('Downloading model archive %s', url) - response = requests.get(url, stream=True) - if response.status_code != 200: - raise IOError(f'Resource {url} not found') - with TemporaryFile() as stream: - copyfileobj(response.raw, stream) - get_logger().info('Extracting downloaded %s archive', name) - stream.seek(0) - tar = tarfile.open(fileobj=stream) - tar.extractall(path=path) - tar.close() + with requests.get(url, stream=True) as response: + # Note: check for error logging here or upstream ? + response.raise_for_status() + with TemporaryFile() as stream: + # Note: check for chunk size parameters ? + for chunk in response.iter_content(chunk_size=8192): + if chunk: + stream.write(chunk) + get_logger().info('Extracting downloaded %s archive', name) + stream.seek(0) + tar = tarfile.open(fileobj=stream) + tar.extractall(path=path) + tar.close() + # TODO: perform checksum control get_logger().info('%s model file(s) extracted', name) From 5158dedf3b5cd8c2346a2d1f55cedcab05a0b515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 14:07:27 +0100 Subject: [PATCH 27/40] fix: remove unused dataset default path --- spleeter/dataset.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/spleeter/dataset.py b/spleeter/dataset.py index 41ca79e..fe2d349 100644 --- a/spleeter/dataset.py +++ b/spleeter/dataset.py @@ -42,15 +42,6 @@ __email__ = 'research@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' -# Default datasets path parameter to use. -DEFAULT_DATASETS_PATH = join( - 'audio_database', - 'separated_sources', - 'experiments', - 'karaoke_vocal_extraction', - 'tensorflow_experiment' -) - # Default audio parameters to use. DEFAULT_AUDIO_PARAMS = { 'instrument_list': ('vocals', 'accompaniment'), From 7ea5325fbfd35ef0f8dde558861f13461ef8400b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 14:43:10 +0100 Subject: [PATCH 28/40] fix: remove unused dep feat: add checksum control --- spleeter/commands/separate.py | 12 -------- spleeter/model/provider/github.py | 48 ++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/spleeter/commands/separate.py b/spleeter/commands/separate.py index df88229..2fbbfee 100644 --- a/spleeter/commands/separate.py +++ b/spleeter/commands/separate.py @@ -11,20 +11,8 @@ -i /path/to/audio1.wav /path/to/audio2.mp3 """ -from multiprocessing import Pool -from os.path import isabs, join, split, splitext -from tempfile import gettempdir - -# pylint: disable=import-error -import tensorflow as tf -import numpy as np -# pylint: enable=import-error - from ..audio.adapter import get_audio_adapter -from ..audio.convertor import to_n_channels from ..separator import Separator -from ..utils.estimator import create_estimator -from ..utils.tensor import set_tensor_shape __email__ = 'research@deezer.com' __author__ = 'Deezer Research' diff --git a/spleeter/model/provider/github.py b/spleeter/model/provider/github.py index 3afecd1..2f6d7a3 100644 --- a/spleeter/model/provider/github.py +++ b/spleeter/model/provider/github.py @@ -14,10 +14,10 @@ >>> provider.download('2stems', '/path/to/local/storage') """ +import hashlib import tarfile -from os import environ -from tempfile import TemporaryFile +from tempfile import NamedTemporaryFile import requests @@ -34,6 +34,7 @@ class GithubModelProvider(ModelProvider): LATEST_RELEASE = 'v1.4.0' RELEASE_PATH = 'releases/download' + CHECKSUM_INDEX = 'checksum.json' def __init__(self, host, repository, release): """ Default constructor. @@ -46,6 +47,33 @@ class GithubModelProvider(ModelProvider): self._repository = repository self._release = release + def checksum(self, name, path): + """ Computes given path file sha256 and compares it to reference index + from release. Raise an exception if not matching. + + :param name: Name of the model to compute checksum for. + :param path: Path of the file to compare checksum with. + :raise ValueError: If the given model name is not indexed. + :raise IOerror: if checksum is not valid or index cannot be downloaded. + """ + url = '{}/{}/{}/{}/{}'.format( + self._host, + self._repository, + self.RELEASE_PATH, + self._release, + self.CHECKSUM_INDEX) + response = requests.get(url) + response.raise_for_status() + index = response.json() + if name not in index: + raise ValueError('No checksum for model {}'.format(name)) + sha256 = hashlib.sha256() + with open(path, 'rb') as stream: + for chunk in iter(lambda: stream.read(4096), b''): + sha256.update(chunk) + if sha256.hexdigest() != index[name]: + raise IOError('Downloaded file is corrupted, please retry') + def download(self, name, path): """ Download model denoted by the given name to disk. @@ -60,17 +88,17 @@ class GithubModelProvider(ModelProvider): name) get_logger().info('Downloading model archive %s', url) with requests.get(url, stream=True) as response: - # Note: check for error logging here or upstream ? response.raise_for_status() - with TemporaryFile() as stream: + archive = NamedTemporaryFile(delete=False) + with archive as stream: # Note: check for chunk size parameters ? for chunk in response.iter_content(chunk_size=8192): if chunk: stream.write(chunk) - get_logger().info('Extracting downloaded %s archive', name) - stream.seek(0) - tar = tarfile.open(fileobj=stream) - tar.extractall(path=path) - tar.close() - # TODO: perform checksum control + get_logger().info('Validating archive checksum') + self.checksum(name, archive.name) + get_logger().info('Extracting downloaded %s archive', name) + tar = tarfile.open(name=archive.name) + tar.extractall(path=path) + tar.close() get_logger().info('%s model file(s) extracted', name) From 52cefd1daed7785c9dfbab7db919fc9107889759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 14:56:12 +0100 Subject: [PATCH 29/40] feat: add checksum control --- spleeter/model/provider/github.py | 24 ++++++++++++++++-------- tests/test_github_model_provider.py | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 tests/test_github_model_provider.py diff --git a/spleeter/model/provider/github.py b/spleeter/model/provider/github.py index 2f6d7a3..d409bca 100644 --- a/spleeter/model/provider/github.py +++ b/spleeter/model/provider/github.py @@ -47,14 +47,12 @@ class GithubModelProvider(ModelProvider): self._repository = repository self._release = release - def checksum(self, name, path): - """ Computes given path file sha256 and compares it to reference index - from release. Raise an exception if not matching. + def checksum(self, name): + """ Downloads and returns reference checksum for the given model name. - :param name: Name of the model to compute checksum for. - :param path: Path of the file to compare checksum with. + :param name: Name of the model to get checksum for. + :returns: Checksum of the required model. :raise ValueError: If the given model name is not indexed. - :raise IOerror: if checksum is not valid or index cannot be downloaded. """ url = '{}/{}/{}/{}/{}'.format( self._host, @@ -67,11 +65,21 @@ class GithubModelProvider(ModelProvider): index = response.json() if name not in index: raise ValueError('No checksum for model {}'.format(name)) + return index[name] + + def check_integrity(self, name, path): + """ Computes given path file sha256 and compares it to reference index + from release. Raise an exception if not matching. + + :param name: Name of the model to compute checksum for. + :param path: Path of the file to compare checksum with. + :raise IOerror: if checksum is not valid or index cannot be downloaded. + """ sha256 = hashlib.sha256() with open(path, 'rb') as stream: for chunk in iter(lambda: stream.read(4096), b''): sha256.update(chunk) - if sha256.hexdigest() != index[name]: + if sha256.hexdigest() != self.checksum(name): raise IOError('Downloaded file is corrupted, please retry') def download(self, name, path): @@ -96,7 +104,7 @@ class GithubModelProvider(ModelProvider): if chunk: stream.write(chunk) get_logger().info('Validating archive checksum') - self.checksum(name, archive.name) + self.check_integrity(name, archive.name) get_logger().info('Extracting downloaded %s archive', name) tar = tarfile.open(name=archive.name) tar.extractall(path=path) diff --git a/tests/test_github_model_provider.py b/tests/test_github_model_provider.py new file mode 100644 index 0000000..248b1d5 --- /dev/null +++ b/tests/test_github_model_provider.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# coding: utf8 + +""" TO DOCUMENT """ + +from pytest import raises + +from spleeter.model.provider import get_default_model_provider + + +def test_checksum(): + """ Test archive checksum index retrieval. """ + provider = get_default_model_provider() + assert provider.checksum('2stems') == \ + 'f3a90b39dd2874269e8b05a48a86745df897b848c61f3958efc80a39152bd692' + assert provider.checksum('4stems') == \ + '3adb4a50ad4eb18c7c4d65fcf4cf2367a07d48408a5eb7d03cd20067429dfaa8' + assert provider.checksum('5stems') == \ + '25a1e87eb5f75cc72a4d2d5467a0a50ac75f05611f877c278793742513cc7218' + with raises(ValueError): + provider.checksum('laisse moi stems stems stems') From eb32348c348b965b345c28e64769a9df999658c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Tue, 19 Nov 2019 15:04:32 +0100 Subject: [PATCH 30/40] fix: checksum testing --- spleeter/model/provider/github.py | 31 +++++++++++++++---------------- tests/test_separator.py | 6 ------ 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/spleeter/model/provider/github.py b/spleeter/model/provider/github.py index d409bca..fdd3d80 100644 --- a/spleeter/model/provider/github.py +++ b/spleeter/model/provider/github.py @@ -29,6 +29,19 @@ __author__ = 'Deezer Research' __license__ = 'MIT License' +def compute_file_checksum(path): + """ Computes given path file sha256. + + :param path: Path of the file to compute checksum for. + :returns: File checksum. + """ + sha256 = hashlib.sha256() + with open(path, 'rb') as stream: + for chunk in iter(lambda: stream.read(4096), b''): + sha256.update(chunk) + return sha256.hexdigest() + + class GithubModelProvider(ModelProvider): """ A ModelProvider implementation backed on Github for remote storage. """ @@ -67,21 +80,6 @@ class GithubModelProvider(ModelProvider): raise ValueError('No checksum for model {}'.format(name)) return index[name] - def check_integrity(self, name, path): - """ Computes given path file sha256 and compares it to reference index - from release. Raise an exception if not matching. - - :param name: Name of the model to compute checksum for. - :param path: Path of the file to compare checksum with. - :raise IOerror: if checksum is not valid or index cannot be downloaded. - """ - sha256 = hashlib.sha256() - with open(path, 'rb') as stream: - for chunk in iter(lambda: stream.read(4096), b''): - sha256.update(chunk) - if sha256.hexdigest() != self.checksum(name): - raise IOError('Downloaded file is corrupted, please retry') - def download(self, name, path): """ Download model denoted by the given name to disk. @@ -104,7 +102,8 @@ class GithubModelProvider(ModelProvider): if chunk: stream.write(chunk) get_logger().info('Validating archive checksum') - self.check_integrity(name, archive.name) + if compute_file_checksum(archive.name) != self.checksum(name): + raise IOError('Downloaded file is corrupted, please retry') get_logger().info('Extracting downloaded %s archive', name) tar = tarfile.open(name=archive.name) tar.extractall(path=path) diff --git a/tests/test_separator.py b/tests/test_separator.py index d2648c2..a95128b 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -53,9 +53,3 @@ def test_separate_to_file(configuration, instruments): directory) for instrument in instruments: assert exists(join(directory, '{}.wav'.format(instrument))) - for instrument in instruments: - for compared in instruments: - if instrument != compared: - assert not filecmp.cmp( - join(directory, '{}.wav'.format(instrument)), - join(directory, '{}.wav'.format(compared))) From 7b5222a64a4fa43e15b5cbe3e18427bac05c82d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 20 Nov 2019 14:33:18 +0100 Subject: [PATCH 31/40] feat: add detailed ffprobe logs --- spleeter/__init__.py | 6 ++++++ spleeter/__main__.py | 35 ++++++++++++++++++++------------- spleeter/audio/adapter.py | 12 +++++++---- spleeter/audio/ffmpeg.py | 13 +++++++++--- spleeter/commands/__init__.py | 3 +-- spleeter/commands/separate.py | 4 ++-- spleeter/separator.py | 2 +- spleeter/utils/configuration.py | 8 ++++---- tests/__init__.py | 6 +++++- tests/test_pypi_sdist.sh | 10 ---------- 10 files changed, 58 insertions(+), 41 deletions(-) delete mode 100644 tests/test_pypi_sdist.sh diff --git a/spleeter/__init__.py b/spleeter/__init__.py index e369371..0650c97 100644 --- a/spleeter/__init__.py +++ b/spleeter/__init__.py @@ -16,3 +16,9 @@ __email__ = 'research@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' + + +class SpleeterError(Exception): + """ Custom exception for Spleeter related error. """ + + pass diff --git a/spleeter/__main__.py b/spleeter/__main__.py index 5f72040..5c18b21 100644 --- a/spleeter/__main__.py +++ b/spleeter/__main__.py @@ -10,9 +10,13 @@ import sys import warnings +from . import SpleeterError from .commands import create_argument_parser from .utils.configuration import load_configuration -from .utils.logging import enable_logging, enable_tensorflow_logging +from .utils.logging import ( + enable_logging, + enable_tensorflow_logging, + get_logger) __email__ = 'research@deezer.com' __author__ = 'Deezer Research' @@ -26,19 +30,22 @@ def main(argv): :param argv: Provided command line arguments. """ - parser = create_argument_parser() - arguments = parser.parse_args(argv[1:]) - enable_logging() - if arguments.verbose: - enable_tensorflow_logging() - if arguments.command == 'separate': - from .commands.separate import entrypoint - elif arguments.command == 'train': - from .commands.train import entrypoint - elif arguments.command == 'evaluate': - from .commands.evaluate import entrypoint - params = load_configuration(arguments.params_filename) - entrypoint(arguments, params) + try: + parser = create_argument_parser() + arguments = parser.parse_args(argv[1:]) + enable_logging() + if arguments.verbose: + enable_tensorflow_logging() + if arguments.command == 'separate': + from .commands.separate import entrypoint + elif arguments.command == 'train': + from .commands.train import entrypoint + elif arguments.command == 'evaluate': + from .commands.evaluate import entrypoint + params = load_configuration(arguments.configuration) + entrypoint(arguments, params) + except SpleeterError as e: + get_logger().error(e) def entrypoint(): diff --git a/spleeter/audio/adapter.py b/spleeter/audio/adapter.py index 59bce42..bda1441 100644 --- a/spleeter/audio/adapter.py +++ b/spleeter/audio/adapter.py @@ -16,6 +16,7 @@ import tensorflow as tf from tensorflow.contrib.signal import stft, hann_window # pylint: enable=import-error +from .. import SpleeterError from ..utils.logging import get_logger __email__ = 'research@deezer.com' @@ -73,7 +74,8 @@ class AudioAdapter(ABC): # Defined safe loading function. def safe_load(path, offset, duration, sample_rate, dtype): - get_logger().info( + logger = get_logger() + logger.info( f'Loading audio {path} from {offset} to {offset + duration}') try: (data, _) = self.load( @@ -82,10 +84,12 @@ class AudioAdapter(ABC): duration.numpy(), sample_rate.numpy(), dtype=dtype.numpy()) - get_logger().info('Audio data loaded successfully') + logger.info('Audio data loaded successfully') return (data, False) except Exception as e: - get_logger().warning(e) + logger.exception( + 'An error occurs while loading audio', + exc_info=e) return (np.float32(-1.0), True) # Execute function and format results. @@ -140,6 +144,6 @@ def get_audio_adapter(descriptor): adapter_module = import_module(module_path) adapter_class = getattr(adapter_module, adapter_class_name) if not isinstance(adapter_class, AudioAdapter): - raise ValueError( + raise SpleeterError( f'{adapter_class_name} is not a valid AudioAdapter class') return adapter_class() diff --git a/spleeter/audio/ffmpeg.py b/spleeter/audio/ffmpeg.py index 8246a58..df837b3 100644 --- a/spleeter/audio/ffmpeg.py +++ b/spleeter/audio/ffmpeg.py @@ -16,6 +16,7 @@ import numpy as np # pylint: enable=import-error from .adapter import AudioAdapter +from .. import SpleeterError from ..utils.logging import get_logger __email__ = 'research@deezer.com' @@ -54,12 +55,18 @@ class FFMPEGProcessAudioAdapter(AudioAdapter): :param sample_rate: (Optional) Sample rate to load audio with. :param dtype: (Optional) Numpy data type to use, default to float32. :returns: Loaded data a (waveform, sample_rate) tuple. + :raise SpleeterError: If any error occurs while loading audio. """ if not isinstance(path, str): path = path.decode() - probe = ffmpeg.probe(path) + try: + probe = ffmpeg.probe(path) + except ffmpeg._run.Error as e: + raise SpleeterError( + 'An error occurs with ffprobe (see ffprobe output below)\n\n{}' + .format(e.stderr.decode())) if 'streams' not in probe or len(probe['streams']) == 0: - raise IOError('No stream was found with ffprobe') + raise SpleeterError('No stream was found with ffprobe') metadata = next( stream for stream in probe['streams'] @@ -117,5 +124,5 @@ class FFMPEGProcessAudioAdapter(AudioAdapter): process.stdin.close() process.wait() except IOError: - raise IOError(f'FFMPEG error: {process.stderr.read()}') + raise SpleeterError(f'FFMPEG error: {process.stderr.read()}') get_logger().info('File %s written', path) diff --git a/spleeter/commands/__init__.py b/spleeter/commands/__init__.py index 6428d6d..47e8f53 100644 --- a/spleeter/commands/__init__.py +++ b/spleeter/commands/__init__.py @@ -84,7 +84,6 @@ OPT_CODEC = { # -b opt specification (separate). OPT_BITRATE = { 'dest': 'bitrate', - 'type': int, 'default': '128k', 'help': 'Audio bitrate to be used for the separated output' } @@ -177,8 +176,8 @@ def _create_separate_parser(parser_factory): parser.add_argument('-i', '--inputs', **OPT_INPUT) parser.add_argument('-o', '--output_path', **OPT_OUTPUT) parser.add_argument('-n', '--output_naming', **OPT_OUTPUT_NAMING) - parser.add_ parser.add_argument('-d', '--duration', **OPT_DURATION) + parser.add_argument('-s', '--offset', **OPT_OFFSET) parser.add_argument('-c', '--codec', **OPT_CODEC) parser.add_argument('-b', '--birate', **OPT_BITRATE) parser.add_argument('-m', '--mwf', **OPT_MWF) diff --git a/spleeter/commands/separate.py b/spleeter/commands/separate.py index 2fbbfee..b501205 100644 --- a/spleeter/commands/separate.py +++ b/spleeter/commands/separate.py @@ -28,13 +28,13 @@ def entrypoint(arguments, params): # TODO: check with output naming. audio_adapter = get_audio_adapter(arguments.audio_adapter) separator = Separator(arguments.configuration, arguments.MWF) - for filename in arguments.audio_filenames: + for filename in arguments.inputs: separator.separate_to_file( filename, arguments.output_path, audio_adapter=audio_adapter, offset=arguments.offset, - duration=arguments.max_duration, + duration=arguments.duration, codec=arguments.codec, bitrate=arguments.bitrate, synchronous=False diff --git a/spleeter/separator.py b/spleeter/separator.py index a5f4efd..79478d9 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -57,7 +57,7 @@ class Separator(object): self._predictor = to_predictor(estimator) return self._predictor - def join(self, timeout=20): + def join(self, timeout=200): """ Wait for all pending tasks to be finished. :param timeout: (Optional) task waiting timeout. diff --git a/spleeter/utils/configuration.py b/spleeter/utils/configuration.py index 03db200..d1fb167 100644 --- a/spleeter/utils/configuration.py +++ b/spleeter/utils/configuration.py @@ -13,7 +13,7 @@ except ImportError: from os.path import exists -from .. import resources +from .. import resources, SpleeterError __email__ = 'research@deezer.com' @@ -31,17 +31,17 @@ def load_configuration(descriptor): :param descriptor: Configuration descriptor to use for lookup. :returns: Loaded description as dict. :raise ValueError: If required embedded configuration does not exists. - :raise IOError: If required configuration file does not exists. + :raise SpleeterError: If required configuration file does not exists. """ # Embedded configuration reading. if descriptor.startswith(_EMBEDDED_CONFIGURATION_PREFIX): name = descriptor[len(_EMBEDDED_CONFIGURATION_PREFIX):] if not loader.is_resource(resources, f'{name}.json'): - raise ValueError(f'No embedded configuration {name} found') + raise SpleeterError(f'No embedded configuration {name} found') with loader.open_text(resources, f'{name}.json') as stream: return json.load(stream) # Standard file reading. if not exists(descriptor): - raise IOError(f'Configuration file {descriptor} not found') + raise SpleeterError(f'Configuration file {descriptor} not found') with open(descriptor, 'r') as stream: return json.load(stream) diff --git a/tests/__init__.py b/tests/__init__.py index 3b800d9..f584f49 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,8 @@ #!/usr/bin/env python # coding: utf8 -""" TO DOCUMENT """ \ No newline at end of file +""" Unit testing package. """ + +__email__ = 'research@deezer.com' +__author__ = 'Deezer Research' +__license__ = 'MIT License' diff --git a/tests/test_pypi_sdist.sh b/tests/test_pypi_sdist.sh deleted file mode 100644 index c514062..0000000 --- a/tests/test_pypi_sdist.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -###################################################################### -# Script that performs PyPi packaging test. -# -# @author Deezer Research -# @version 1.0.0 -###################################################################### - - twine upload --repository-url https://test.pypi.org/legacy/ dist/* \ No newline at end of file From bfb02250835a2c7bbd8010686b32482fa58dedb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 20 Nov 2019 14:36:41 +0100 Subject: [PATCH 32/40] test: add error test --- tests/test_ffmpeg_adapter.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_ffmpeg_adapter.py b/tests/test_ffmpeg_adapter.py index cd503c2..195b737 100644 --- a/tests/test_ffmpeg_adapter.py +++ b/tests/test_ffmpeg_adapter.py @@ -11,12 +11,13 @@ from os.path import join from tempfile import TemporaryDirectory # pylint: disable=import-error -from pytest import fixture +from pytest import fixture, raises import numpy as np import ffmpeg # pylint: enable=import-error +from spleeter import SpleeterError from spleeter.audio.adapter import AudioAdapter from spleeter.audio.adapter import get_default_audio_adapter from spleeter.audio.adapter import get_audio_adapter @@ -61,6 +62,16 @@ def test_load(audio_data): assert waveform.shape[1] == 2 +def test_load_error(adapter): + """ Test load ffprobe exception """ + with raises(SpleeterError): + adapter.load( + 'Paris City Jazz', + TEST_OFFSET, + TEST_DURATION, + TEST_SAMPLE_RATE) + + def test_save(adapter, audio_data): """ Test audio saving. """ with TemporaryDirectory() as directory: From d771fbae34a5e4cd9dcc732e42c72491afa42b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 20 Nov 2019 15:07:12 +0100 Subject: [PATCH 33/40] feat: add filename format --- spleeter/commands/__init__.py | 31 +++++++++++++------------------ spleeter/commands/evaluate.py | 2 -- spleeter/commands/separate.py | 5 ++++- spleeter/separator.py | 22 +++++++++++++++++++--- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/spleeter/commands/__init__.py b/spleeter/commands/__init__.py index 47e8f53..2bbc974 100644 --- a/spleeter/commands/__init__.py +++ b/spleeter/commands/__init__.py @@ -28,6 +28,18 @@ OPT_OUTPUT = { 'help': 'Path of the output directory to write audio files in' } +# -f opt specification (separate). +OPT_FORMAT = { + 'dest': 'filename_format', + 'default': '{filename}/{instrument}.{codec}', + 'help': ( + 'Template string that will be formatted to generated' + 'output filename. Such template should be Python formattable' + 'string, and could use {filename}, {instrument}, and {codec}' + 'variables.' + ) +} + # -p opt specification (train, evaluate and separate). OPT_PARAMS = { 'dest': 'configuration', @@ -37,23 +49,6 @@ OPT_PARAMS = { 'help': 'JSON filename that contains params' } -# -n opt specification (separate). -OPT_OUTPUT_NAMING = { - 'dest': 'output_naming', - 'default': 'filename', - 'choices': ('directory', 'filename'), - 'help': ( - 'Choice for naming the output base path: ' - '"filename" (use the input filename, i.e ' - '/path/to/audio/mix.wav will be separated to ' - '/mix/.wav, ' - '/mix/.wav...) or ' - '"directory" (use the name of the input last level' - ' directory, for instance /path/to/audio/mix.wav ' - 'will be separated to /audio/.wav' - ', /audio/.wav)') -} - # -s opt specification (separate). OPT_OFFSET = { 'dest': 'offset', @@ -175,7 +170,7 @@ def _create_separate_parser(parser_factory): _add_common_options(parser) parser.add_argument('-i', '--inputs', **OPT_INPUT) parser.add_argument('-o', '--output_path', **OPT_OUTPUT) - parser.add_argument('-n', '--output_naming', **OPT_OUTPUT_NAMING) + parser.add_argument('-f', '--filename_format', **OPT_FORMAT) parser.add_argument('-d', '--duration', **OPT_DURATION) parser.add_argument('-s', '--offset', **OPT_OFFSET) parser.add_argument('-c', '--codec', **OPT_CODEC) diff --git a/spleeter/commands/evaluate.py b/spleeter/commands/evaluate.py index c2fe789..9bc6d96 100644 --- a/spleeter/commands/evaluate.py +++ b/spleeter/commands/evaluate.py @@ -44,7 +44,6 @@ __license__ = 'MIT License' _SPLIT = 'test' _MIXTURE = 'mixture.wav' -_NAMING = 'directory' _AUDIO_DIRECTORY = 'audio' _METRICS_DIRECTORY = 'metrics' _INSTRUMENTS = ('vocals', 'drums', 'bass', 'other') @@ -71,7 +70,6 @@ def _separate_evaluation_dataset(arguments, musdb_root_directory, params): audio_filenames=mixtures, audio_codec='wav', output_path=join(audio_output_directory, _SPLIT), - output_naming=_NAMING, max_duration=600., MWF=arguments.MWF, verbose=arguments.verbose), diff --git a/spleeter/commands/separate.py b/spleeter/commands/separate.py index b501205..b1f41ab 100644 --- a/spleeter/commands/separate.py +++ b/spleeter/commands/separate.py @@ -27,7 +27,9 @@ def entrypoint(arguments, params): """ # TODO: check with output naming. audio_adapter = get_audio_adapter(arguments.audio_adapter) - separator = Separator(arguments.configuration, arguments.MWF) + separator = Separator( + arguments.configuration, + arguments.MWF) for filename in arguments.inputs: separator.separate_to_file( filename, @@ -37,6 +39,7 @@ def entrypoint(arguments, params): duration=arguments.duration, codec=arguments.codec, bitrate=arguments.bitrate, + filename_format=arguments.filename_format, synchronous=False ) separator.join() diff --git a/spleeter/separator.py b/spleeter/separator.py index 79478d9..6c46315 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -18,8 +18,9 @@ import json from functools import partial from multiprocessing import Pool from pathlib import Path -from os.path import join +from os.path import basename, join +from . import SpleeterError from .audio.adapter import get_default_audio_adapter from .audio.convertor import to_stereo from .model import model_fn @@ -93,10 +94,13 @@ class Separator(object): self, audio_descriptor, destination, audio_adapter=get_default_audio_adapter(), offset=0, duration=600., codec='wav', bitrate='128k', - synchronous=True): + filename_format='{filename}/{instrument}.{codec}', synchronous=True): """ Performs source separation and export result to file using given audio adapter. + Filename format should be a Python formattable string that could use + following parameters : {instrument}, {filename} and {codec}. + :param audio_descriptor: Describe song to separate, used by audio adapter to retrieve and load audio data, in case of file based audio adapter, such @@ -107,6 +111,7 @@ class Separator(object): :param duration: (Optional) Duration of loaded song. :param codec: (Optional) Export codec. :param bitrate: (Optional) Export bitrate. + :param filename_format: (Optional) Filename format. :param synchronous: (Optional) True is should by synchronous. """ waveform, _ = audio_adapter.load( @@ -115,9 +120,20 @@ class Separator(object): duration=duration, sample_rate=self._sample_rate) sources = self.separate(waveform) + filename = basename(audio_descriptor) + generated = [] for instrument, data in sources.items(): + path = join(destination, filename_format.format( + filename=filename, + instrument=instrument, + codec=codec)) + if path in generated: + raise SpleeterError(( + f'Separated source path conflict : {path},' + 'please check your filename format')) + generated.append(path) task = self._pool.apply_async(audio_adapter.save, ( - join(destination, f'{instrument}.{codec}'), + path, data, self._sample_rate, codec, From 151c2a7b9318e43a71ec5534498ebd623289ce29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 20 Nov 2019 15:18:53 +0100 Subject: [PATCH 34/40] test: add filename format tests --- tests/test_separator.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/tests/test_separator.py b/tests/test_separator.py index a95128b..d231987 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -9,15 +9,17 @@ __license__ = 'MIT License' import filecmp -from os.path import exists, join +from os.path import basename, exists, join from tempfile import TemporaryDirectory import pytest +from spleeter import SpleeterError from spleeter.audio.adapter import get_default_audio_adapter from spleeter.separator import Separator TEST_AUDIO_DESCRIPTOR = 'audio_example.mp3' +TEST_AUDIO_BASENAME = basename(TEST_AUDIO_DESCRIPTOR) TEST_CONFIGURATIONS = [ ('spleeter:2stems', ('vocals', 'accompaniment')), ('spleeter:4stems', ('vocals', 'drums', 'bass', 'other')), @@ -52,4 +54,32 @@ def test_separate_to_file(configuration, instruments): TEST_AUDIO_DESCRIPTOR, directory) for instrument in instruments: - assert exists(join(directory, '{}.wav'.format(instrument))) + assert exists(join( + directory, + '{}/{}.wav'.format(TEST_AUDIO_BASENAME, instrument))) + + +@pytest.mark.parametrize('configuration, instruments', TEST_CONFIGURATIONS) +def test_filename_format(configuration, instruments): + """ Test custom filename format. """ + separator = Separator(configuration) + with TemporaryDirectory() as directory: + separator.separate_to_file( + TEST_AUDIO_DESCRIPTOR, + directory, + filename_format='export/{filename}/{instrument}.{codec}') + for instrument in instruments: + assert exists(join( + directory, + 'export/{}/{}.wav'.format(TEST_AUDIO_BASENAME, instrument))) + + +def test_filename_confilct(): + """ Test error handling with static pattern. """ + separator = Separator(TEST_CONFIGURATIONS[0][0]) + with TemporaryDirectory() as directory: + with pytest.raises(SpleeterError): + separator.separate_to_file( + TEST_AUDIO_DESCRIPTOR, + directory, + filename_format='I wanna be your lover') From fbe5f290ef6af8d58a7c69f99841c8f925e93a18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 20 Nov 2019 15:40:25 +0100 Subject: [PATCH 35/40] refactor: add probe in train --- spleeter/commands/train.py | 2 ++ spleeter/model/provider/__init__.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spleeter/commands/train.py b/spleeter/commands/train.py index bb48c41..2a40c84 100644 --- a/spleeter/commands/train.py +++ b/spleeter/commands/train.py @@ -16,6 +16,7 @@ import tensorflow as tf from ..audio.adapter import get_audio_adapter from ..dataset import get_training_dataset, get_validation_dataset from ..model import model_fn +from ..model.provider import ModelProvider from ..utils.logging import get_logger __email__ = 'research@deezer.com' @@ -95,4 +96,5 @@ def entrypoint(arguments, params): estimator, train_spec, evaluation_spec) + ModelProvider.writeProbe(params['model_dir']) get_logger().info('Model training done') diff --git a/spleeter/model/provider/__init__.py b/spleeter/model/provider/__init__.py index 854b065..3aa3d8d 100644 --- a/spleeter/model/provider/__init__.py +++ b/spleeter/model/provider/__init__.py @@ -38,12 +38,14 @@ class ModelProvider(ABC): """ pass - def writeProbe(self, directory): + @staticmethod + def writeProbe(directory): """ Write a model probe file into the given directory. :param directory: Directory to write probe into. """ - with open(join(directory, self.MODEL_PROBE_PATH), 'w') as stream: + probe = join(directory, ModelProvider.MODEL_PROBE_PATH) + with open(probe, 'w') as stream: stream.write('OK') def get(self, model_directory): From 5602d5204110b491263fe38fab2aaddb532bfb35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 20 Nov 2019 16:37:10 +0100 Subject: [PATCH 36/40] refactor: aggregate docker jobs feat: add docker push --- .circleci/config.yml | 189 +++++------------------------ docker/cpu/conda-2stems.dockerfile | 2 +- docker/cpu/conda-4stems.dockerfile | 2 +- docker/cpu/conda-5stems.dockerfile | 2 +- 4 files changed, 33 insertions(+), 162 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16a9ff4..8ce09dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,110 +99,44 @@ jobs: docker: - image: docker:17.05.0-ce-git steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:conda -f docker/cpu/conda.dockerfile . - docker-conda-cpu-2stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:2-stems-conda -f docker/cpu/conda-2stems.dockerfile . - docker-conda-cpu-4stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:4-stems-conda -f docker/cpu/conda-4stems.dockerfile . - docker-conda-cpu-5stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:5-stems-conda -f docker/cpu/conda-5stems.dockerfile . + - checkout + - run: docker build -t researchdeezer/spleeter:conda -f docker/cpu/conda.dockerfile . + - run: docker build -t researchdeezer/spleeter:conda-2stems -f docker/cpu/conda-2stems.dockerfile . + - run: docker build -t researchdeezer/spleeter:conda-4stems -f docker/cpu/conda-2stems.dockerfile . + - run: docker build -t researchdeezer/spleeter:conda-5stems -f docker/cpu/conda-2stems.dockerfile . + - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + - run: docker push researchdeezer/spleeter:conda + - run: docker push researchdeezer/spleeter:conda-2stems + - run: docker push researchdeezer/spleeter:conda-4stems + - run: docker push researchdeezer/spleeter:conda-5stems docker-3.6-cpu: docker: - image: docker:17.05.0-ce-git steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.6 -f docker/cpu/python-3.6.dockerfile . - docker-3.6-cpu-2stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.6-2stems -f docker/cpu/python-3.6-2stems.dockerfile . - docker-3.6-cpu-4stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.6-4stems -f docker/cpu/python-3.6-4stems.dockerfile . - docker-3.6-cpu-5stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.6-5stems -f docker/cpu/python-3.6-5stems.dockerfile . + - checkout + - run: docker build -t researchdeezer/spleeter:3.6 -f docker/cpu/python-3.6.dockerfile . + - run: docker build -t researchdeezer/spleeter:3.6-2stems -f docker/cpu/python-3.6-2stems.dockerfile . + - run: docker build -t researchdeezer/spleeter:3.6-4stems -f docker/cpu/python-3.6-4stems.dockerfile . + - run: docker build -t researchdeezer/spleeter:3.6-5stems -f docker/cpu/python-3.6-5stems.dockerfile . + - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + - run: docker push researchdeezer/spleeter:3.6 + - run: docker push researchdeezer/spleeter:3.6-2stems + - run: docker push researchdeezer/spleeter:3.6-4stems + - run: docker push researchdeezer/spleeter:3.6-5stems docker-3.7-cpu: docker: - image: docker:17.05.0-ce-git steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.7 -f docker/cpu/python-3.7.dockerfile . - docker-3.7-cpu-2stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.7-2stems -f docker/cpu/python-3.7-2stems.dockerfile . - docker-3.7-cpu-4stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.7-4stems -f docker/cpu/python-3.7-4stems.dockerfile . - docker-3.7-cpu-5stems: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: build image - command: docker build -t spleeter:3.7-5stems -f docker/cpu/python-3.7-5stems.dockerfile . + - checkout + - run: docker build -t spleeter:3.7 -f docker/cpu/python-3.7.dockerfile . + - run: docker build -t spleeter:3.7-2stems -f docker/cpu/python-3.7-2stems.dockerfile . + - run: docker build -t spleeter:3.7-4stems -f docker/cpu/python-3.7-4stems.dockerfile . + - run: docker build -t spleeter:3.7-5stems -f docker/cpu/python-3.7-5stems.dockerfile . + - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + - run: docker push researchdeezer/spleeter:3.7 + - run: docker push researchdeezer/spleeter:3.7-2stems + - run: docker push researchdeezer/spleeter:3.7-4stems + - run: docker push researchdeezer/spleeter:3.7-5stems workflows: version: 2 spleeter-release-pipeline: @@ -243,27 +177,6 @@ workflows: branches: only: - master - - docker-conda-cpu-2stems: - requires: - - docker-conda-cpu - filters: - branches: - only: - - master - - docker-conda-cpu-4stems: - requires: - - docker-conda-cpu - filters: - branches: - only: - - master - - docker-conda-cpu-5stems: - requires: - - docker-conda-cpu - filters: - branches: - only: - - master - docker-3.6-cpu: requires: - hold @@ -271,51 +184,9 @@ workflows: branches: only: - master - - docker-3.6-cpu-2stems: - requires: - - docker-3.6-cpu - filters: - branches: - only: - - master - - docker-3.6-cpu-4stems: - requires: - - docker-3.6-cpu - filters: - branches: - only: - - master - - docker-3.6-cpu-5stems: - requires: - - docker-3.6-cpu - filters: - branches: - only: - - master - docker-3.7-cpu: requires: - hold - filters: - branches: - only: - - master - - docker-3.7-cpu-2stems: - requires: - - docker-3.7-cpu - filters: - branches: - only: - - master - - docker-3.7-cpu-4stems: - requires: - - docker-3.7-cpu - filters: - branches: - only: - - master - - docker-3.7-cpu-5stems: - requires: - - docker-3.7-cpu filters: branches: only: diff --git a/docker/cpu/conda-2stems.dockerfile b/docker/cpu/conda-2stems.dockerfile index c38a9ca..04afaa7 100644 --- a/docker/cpu/conda-2stems.dockerfile +++ b/docker/cpu/conda-2stems.dockerfile @@ -1,4 +1,4 @@ -FROM spleeter:conda +FROM researchdeezer/spleeter:conda RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ diff --git a/docker/cpu/conda-4stems.dockerfile b/docker/cpu/conda-4stems.dockerfile index e53fce5..c8acf4f 100644 --- a/docker/cpu/conda-4stems.dockerfile +++ b/docker/cpu/conda-4stems.dockerfile @@ -1,4 +1,4 @@ -FROM spleeter:conda +FROM researchdeezer/spleeter:conda RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ diff --git a/docker/cpu/conda-5stems.dockerfile b/docker/cpu/conda-5stems.dockerfile index b1cbc3a..934a97a 100644 --- a/docker/cpu/conda-5stems.dockerfile +++ b/docker/cpu/conda-5stems.dockerfile @@ -1,4 +1,4 @@ -FROM spleeter:conda +FROM researchdeezer/spleeter:conda RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ From 14b7eb2bab78207bfb1574ddd50f721bf2996092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 21 Nov 2019 12:25:25 +0100 Subject: [PATCH 37/40] refactor: docker images --- docker/cpu/conda-2stems.dockerfile | 3 ++- docker/cpu/conda-4stems.dockerfile | 3 ++- docker/cpu/conda-5stems.dockerfile | 3 ++- docker/cpu/conda.dockerfile | 2 ++ docker/cpu/cpu.dockerfile | 24 ----------------- docker/cpu/python-3.6-2stems.dockerfile | 5 ++-- docker/cpu/python-3.6-4stems.dockerfile | 5 ++-- docker/cpu/python-3.6-5stems.dockerfile | 5 ++-- docker/cpu/python-3.6.dockerfile | 1 + docker/cpu/python-3.7-2stems.dockerfile | 5 ++-- docker/cpu/python-3.7-4stems.dockerfile | 5 ++-- docker/cpu/python-3.7-5stems.dockerfile | 5 ++-- docker/cpu/python-3.7.dockerfile | 1 + docker/gpu/conda-gpu-2-stems.dockerfile | 2 +- docker/gpu/conda-gpu-4-stems.dockerfile | 2 +- docker/gpu/conda-gpu-5-stems.dockerfile | 2 +- docker/gpu/conda-gpu.dockerfile | 27 ++++++++++++------- docker/gpu/gpu.dockerfile | 35 ------------------------- docker/gpu/install_miniconda.sh | 13 --------- 19 files changed, 49 insertions(+), 99 deletions(-) delete mode 100644 docker/cpu/cpu.dockerfile delete mode 100644 docker/gpu/gpu.dockerfile delete mode 100644 docker/gpu/install_miniconda.sh diff --git a/docker/cpu/conda-2stems.dockerfile b/docker/cpu/conda-2stems.dockerfile index 04afaa7..0c4bbf0 100644 --- a/docker/cpu/conda-2stems.dockerfile +++ b/docker/cpu/conda-2stems.dockerfile @@ -2,4 +2,5 @@ FROM researchdeezer/spleeter:conda RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ - && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ + && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/conda-4stems.dockerfile b/docker/cpu/conda-4stems.dockerfile index c8acf4f..b32c84c 100644 --- a/docker/cpu/conda-4stems.dockerfile +++ b/docker/cpu/conda-4stems.dockerfile @@ -2,4 +2,5 @@ FROM researchdeezer/spleeter:conda RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ - && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ + && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/conda-5stems.dockerfile b/docker/cpu/conda-5stems.dockerfile index 934a97a..6b6f2ca 100644 --- a/docker/cpu/conda-5stems.dockerfile +++ b/docker/cpu/conda-5stems.dockerfile @@ -2,4 +2,5 @@ FROM researchdeezer/spleeter:conda RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ - && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ + && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/conda.dockerfile b/docker/cpu/conda.dockerfile index 133273a..32184e3 100644 --- a/docker/cpu/conda.dockerfile +++ b/docker/cpu/conda.dockerfile @@ -1,5 +1,7 @@ FROM continuumio/miniconda3:4.7.10 +RUN conda install -y -c conda-forge musdb +# RUN conda install -y -c conda-forge museval RUN conda install -y -c conda-forge spleeter RUN mkdir -p /model ENV MODEL_PATH /model diff --git a/docker/cpu/cpu.dockerfile b/docker/cpu/cpu.dockerfile deleted file mode 100644 index e3f47e7..0000000 --- a/docker/cpu/cpu.dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM continuumio/miniconda3:4.7.10 - -# install tensorflow -RUN conda install -y tensorflow==1.14.0 - -# install ffmpeg for audio loading/writing -RUN conda install -y -c conda-forge ffmpeg - -# install extra python libraries -RUN conda install -y -c anaconda pandas==0.25.1 -RUN conda install -y -c conda-forge libsndfile - -# install ipython -RUN conda install -y ipython - -WORKDIR /workspace/ -COPY ./ spleeter/ - -RUN mkdir /cache/ - -WORKDIR /workspace/spleeter -RUN pip install . - -ENTRYPOINT ["python", "-m", "spleeter"] \ No newline at end of file diff --git a/docker/cpu/python-3.6-2stems.dockerfile b/docker/cpu/python-3.6-2stems.dockerfile index 2a2012d..3fca665 100644 --- a/docker/cpu/python-3.6-2stems.dockerfile +++ b/docker/cpu/python-3.6-2stems.dockerfile @@ -1,5 +1,6 @@ -FROM spleeter:3.6 +FROM researchdeezer/spleeter:3.6 RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ - && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ + && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/python-3.6-4stems.dockerfile b/docker/cpu/python-3.6-4stems.dockerfile index 4524de0..d19171e 100644 --- a/docker/cpu/python-3.6-4stems.dockerfile +++ b/docker/cpu/python-3.6-4stems.dockerfile @@ -1,5 +1,6 @@ -FROM spleeter:3.6 +FROM researchdeezer/spleeter:3.6 RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ - && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ + && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/python-3.6-5stems.dockerfile b/docker/cpu/python-3.6-5stems.dockerfile index 1eaf104..a2a14ce 100644 --- a/docker/cpu/python-3.6-5stems.dockerfile +++ b/docker/cpu/python-3.6-5stems.dockerfile @@ -1,5 +1,6 @@ -FROM spleeter:3.6 +FROM researchdeezer/spleeter:3.6 RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ - && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ + && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/python-3.6.dockerfile b/docker/cpu/python-3.6.dockerfile index e0ec580..a10868d 100644 --- a/docker/cpu/python-3.6.dockerfile +++ b/docker/cpu/python-3.6.dockerfile @@ -1,6 +1,7 @@ FROM python:3.6 RUN apt-get update && apt-get install -y ffmpeg libsndfile +RUN pip install musdb museval RUN pip install spleeter RUN mkdir -p /model ENV MODEL_PATH /model diff --git a/docker/cpu/python-3.7-2stems.dockerfile b/docker/cpu/python-3.7-2stems.dockerfile index 748cd44..c3232a3 100644 --- a/docker/cpu/python-3.7-2stems.dockerfile +++ b/docker/cpu/python-3.7-2stems.dockerfile @@ -1,5 +1,6 @@ -FROM spleeter:3.7 +FROM researchdeezer/spleeter:3.7 RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ - && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ + && tar -xvzf /tmp/2stems.tar.gz -C /model/2stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/python-3.7-4stems.dockerfile b/docker/cpu/python-3.7-4stems.dockerfile index 4f8ce06..dfa2366 100644 --- a/docker/cpu/python-3.7-4stems.dockerfile +++ b/docker/cpu/python-3.7-4stems.dockerfile @@ -1,5 +1,6 @@ -FROM spleeter:3.7 +FROM researchdeezer/spleeter:3.7 RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ - && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ + && tar -xvzf /tmp/4stems.tar.gz -C /model/4stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/python-3.7-5stems.dockerfile b/docker/cpu/python-3.7-5stems.dockerfile index b5dd68b..0c05955 100644 --- a/docker/cpu/python-3.7-5stems.dockerfile +++ b/docker/cpu/python-3.7-5stems.dockerfile @@ -1,5 +1,6 @@ -FROM spleeter:3.7 +FROM researchdeezer/spleeter:3.7 RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ - && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ + && tar -xvzf /tmp/5stems.tar.gz -C /model/5stems/ \ + && touch /model/5stems/.probe diff --git a/docker/cpu/python-3.7.dockerfile b/docker/cpu/python-3.7.dockerfile index c4f370e..f900683 100644 --- a/docker/cpu/python-3.7.dockerfile +++ b/docker/cpu/python-3.7.dockerfile @@ -1,6 +1,7 @@ FROM python:3.7 RUN apt-get update && apt-get install -y ffmpeg libsndfile +RUN pip install musdb museval RUN pip install spleeter RUN mkdir -p /model ENV MODEL_PATH /model diff --git a/docker/gpu/conda-gpu-2-stems.dockerfile b/docker/gpu/conda-gpu-2-stems.dockerfile index f55b0e1..888b524 100644 --- a/docker/gpu/conda-gpu-2-stems.dockerfile +++ b/docker/gpu/conda-gpu-2-stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:conda-gpu +FROM researchdeezer/spleeter:conda-gpu RUN mkdir -p /model/2stems \ && wget -O /tmp/2stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/2stems.tar.gz \ diff --git a/docker/gpu/conda-gpu-4-stems.dockerfile b/docker/gpu/conda-gpu-4-stems.dockerfile index 98952d1..3d9c4f7 100644 --- a/docker/gpu/conda-gpu-4-stems.dockerfile +++ b/docker/gpu/conda-gpu-4-stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:conda-gpu +FROM researchdeezer/spleeter:conda-gpu RUN mkdir -p /model/4stems \ && wget -O /tmp/4stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/4stems.tar.gz \ diff --git a/docker/gpu/conda-gpu-5-stems.dockerfile b/docker/gpu/conda-gpu-5-stems.dockerfile index 0e8135f..5d5ae3b 100644 --- a/docker/gpu/conda-gpu-5-stems.dockerfile +++ b/docker/gpu/conda-gpu-5-stems.dockerfile @@ -1,4 +1,4 @@ -FROM deezer/spleeter:conda-gpu +FROM researchdeezer/spleeter:conda-gpu RUN mkdir -p /model/5stems \ && wget -O /tmp/5stems.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/5stems.tar.gz \ diff --git a/docker/gpu/conda-gpu.dockerfile b/docker/gpu/conda-gpu.dockerfile index 3da59d0..e1fa3d0 100644 --- a/docker/gpu/conda-gpu.dockerfile +++ b/docker/gpu/conda-gpu.dockerfile @@ -1,12 +1,21 @@ -FROM continuumio/miniconda3:4.7.10 +FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 -RUN conda install -y ipython \ - && conda install -y tensorflow-gpu==1.14.0 \ - && conda install -y -c conda-forge ffmpeg \ - && conda install -y -c conda-forge libsndfile \ - && conda install -y -c anaconda pandas==0.25.1 \ -RUN mkdir -p /model -ENV MODEL_PATH /model -RUN pip install spleeter +RUN apt-get update --fix-missing && \ + apt-get install -y wget bzip2 ca-certificates curl git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + /opt/conda/bin/conda clean -tipsy && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc + +RUN conda install -y tensorflow-gpu==1.14.0 +RUN conda install -y -c conda-forge musdb +# RUN conda install -y -c conda-forge museval +# Note: switch to spleeter GPU once published. +RUN conda install -y -c conda-forge spleeter ENTRYPOINT ["spleeter"] \ No newline at end of file diff --git a/docker/gpu/gpu.dockerfile b/docker/gpu/gpu.dockerfile deleted file mode 100644 index aedeeda..0000000 --- a/docker/gpu/gpu.dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 - -# set work directory -WORKDIR /workspace - -# install anaconda -ENV PATH /opt/conda/bin:$PATH -COPY docker/install_miniconda.sh . -RUN bash ./install_miniconda.sh && rm install_miniconda.sh - -RUN conda update -n base -c defaults conda - -# install tensorflow for GPU -RUN conda install -y tensorflow-gpu==1.14.0 - -# install ffmpeg for audio loading/writing -RUN conda install -y -c conda-forge ffmpeg - -# install extra libs -RUN conda install -y -c anaconda pandas==0.25.1 -RUN conda install -y -c conda-forge libsndfile - -# install ipython -RUN conda install -y ipython - -RUN mkdir /cache/ - -# clone inside image github repository -COPY ./ spleeter/ - -WORKDIR /workspace/spleeter -RUN pip install . - - -ENTRYPOINT ["python", "-m", "spleeter"] \ No newline at end of file diff --git a/docker/gpu/install_miniconda.sh b/docker/gpu/install_miniconda.sh deleted file mode 100644 index 6ea58bc..0000000 --- a/docker/gpu/install_miniconda.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -apt-get update --fix-missing && \ -apt-get install -y wget bzip2 ca-certificates curl git && \ -apt-get clean && \ -rm -rf /var/lib/apt/lists/* - -wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \ -/bin/bash ~/miniconda.sh -b -p /opt/conda && \ -rm ~/miniconda.sh && \ -/opt/conda/bin/conda clean -tipsy && \ -ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ -echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ -echo "conda activate base" >> ~/.bashrc \ No newline at end of file From 897b0cf3b31821d238ff3545f751dd90c6473c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 21 Nov 2019 12:30:43 +0100 Subject: [PATCH 38/40] fix: add simple docker testing --- .circleci/config.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ce09dc..89fe945 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,6 +104,10 @@ jobs: - run: docker build -t researchdeezer/spleeter:conda-2stems -f docker/cpu/conda-2stems.dockerfile . - run: docker build -t researchdeezer/spleeter:conda-4stems -f docker/cpu/conda-2stems.dockerfile . - run: docker build -t researchdeezer/spleeter:conda-5stems -f docker/cpu/conda-2stems.dockerfile . + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-2stems separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-4stems separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-5stems separate -i /runtime/audio_example.mp3 -o /tmp - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - run: docker push researchdeezer/spleeter:conda - run: docker push researchdeezer/spleeter:conda-2stems @@ -118,6 +122,10 @@ jobs: - run: docker build -t researchdeezer/spleeter:3.6-2stems -f docker/cpu/python-3.6-2stems.dockerfile . - run: docker build -t researchdeezer/spleeter:3.6-4stems -f docker/cpu/python-3.6-4stems.dockerfile . - run: docker build -t researchdeezer/spleeter:3.6-5stems -f docker/cpu/python-3.6-5stems.dockerfile . + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6 separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-2stems separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-4stems separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-5stems separate -i /runtime/audio_example.mp3 -o /tmp - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - run: docker push researchdeezer/spleeter:3.6 - run: docker push researchdeezer/spleeter:3.6-2stems @@ -132,6 +140,10 @@ jobs: - run: docker build -t spleeter:3.7-2stems -f docker/cpu/python-3.7-2stems.dockerfile . - run: docker build -t spleeter:3.7-4stems -f docker/cpu/python-3.7-4stems.dockerfile . - run: docker build -t spleeter:3.7-5stems -f docker/cpu/python-3.7-5stems.dockerfile . + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7 separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-2stems separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -o /tmp + - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-5stems separate -i /runtime/audio_example.mp3 -o /tmp - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - run: docker push researchdeezer/spleeter:3.7 - run: docker push researchdeezer/spleeter:3.7-2stems From 20832aad99962185a1e7395c9192576f1ec939b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 21 Nov 2019 12:34:24 +0100 Subject: [PATCH 39/40] fix: libsndfile dep --- docker/cpu/python-3.6.dockerfile | 2 +- docker/cpu/python-3.7.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/cpu/python-3.6.dockerfile b/docker/cpu/python-3.6.dockerfile index a10868d..0f23184 100644 --- a/docker/cpu/python-3.6.dockerfile +++ b/docker/cpu/python-3.6.dockerfile @@ -1,6 +1,6 @@ FROM python:3.6 -RUN apt-get update && apt-get install -y ffmpeg libsndfile +RUN apt-get update && apt-get install -y ffmpeg libsndfile1 RUN pip install musdb museval RUN pip install spleeter RUN mkdir -p /model diff --git a/docker/cpu/python-3.7.dockerfile b/docker/cpu/python-3.7.dockerfile index f900683..33bdcbc 100644 --- a/docker/cpu/python-3.7.dockerfile +++ b/docker/cpu/python-3.7.dockerfile @@ -1,6 +1,6 @@ FROM python:3.7 -RUN apt-get update && apt-get install -y ffmpeg libsndfile +RUN apt-get update && apt-get install -y ffmpeg libsndfile1 RUN pip install musdb museval RUN pip install spleeter RUN mkdir -p /model From 0dce26c10e3ea317ce4ae23c368f5422cd441f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 21 Nov 2019 12:52:09 +0100 Subject: [PATCH 40/40] refactor: freeze spleeter version --- .circleci/config.yml | 2 ++ docker/cpu/conda.dockerfile | 2 +- docker/cpu/python-3.6.dockerfile | 2 +- docker/cpu/python-3.7.dockerfile | 2 +- docker/gpu/conda-gpu.dockerfile | 29 +++++++++++++++-------------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89fe945..4cae4ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,6 +145,8 @@ jobs: - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -o /tmp - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-5stems separate -i /runtime/audio_example.mp3 -o /tmp - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + - run: docker tags researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest + - run: docker push researchdeezer/spleeter:latest - run: docker push researchdeezer/spleeter:3.7 - run: docker push researchdeezer/spleeter:3.7-2stems - run: docker push researchdeezer/spleeter:3.7-4stems diff --git a/docker/cpu/conda.dockerfile b/docker/cpu/conda.dockerfile index 32184e3..1506a68 100644 --- a/docker/cpu/conda.dockerfile +++ b/docker/cpu/conda.dockerfile @@ -2,7 +2,7 @@ FROM continuumio/miniconda3:4.7.10 RUN conda install -y -c conda-forge musdb # RUN conda install -y -c conda-forge museval -RUN conda install -y -c conda-forge spleeter +RUN conda install -y -c conda-forge spleeter=1.4.4 RUN mkdir -p /model ENV MODEL_PATH /model diff --git a/docker/cpu/python-3.6.dockerfile b/docker/cpu/python-3.6.dockerfile index 0f23184..3561198 100644 --- a/docker/cpu/python-3.6.dockerfile +++ b/docker/cpu/python-3.6.dockerfile @@ -2,7 +2,7 @@ FROM python:3.6 RUN apt-get update && apt-get install -y ffmpeg libsndfile1 RUN pip install musdb museval -RUN pip install spleeter +RUN pip install spleeter==1.4.4 RUN mkdir -p /model ENV MODEL_PATH /model ENTRYPOINT ["spleeter"] \ No newline at end of file diff --git a/docker/cpu/python-3.7.dockerfile b/docker/cpu/python-3.7.dockerfile index 33bdcbc..0433e5d 100644 --- a/docker/cpu/python-3.7.dockerfile +++ b/docker/cpu/python-3.7.dockerfile @@ -2,7 +2,7 @@ FROM python:3.7 RUN apt-get update && apt-get install -y ffmpeg libsndfile1 RUN pip install musdb museval -RUN pip install spleeter +RUN pip install spleeter==1.4.4 RUN mkdir -p /model ENV MODEL_PATH /model ENTRYPOINT ["spleeter"] \ No newline at end of file diff --git a/docker/gpu/conda-gpu.dockerfile b/docker/gpu/conda-gpu.dockerfile index e1fa3d0..b98693c 100644 --- a/docker/gpu/conda-gpu.dockerfile +++ b/docker/gpu/conda-gpu.dockerfile @@ -1,21 +1,22 @@ FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 -RUN apt-get update --fix-missing && \ - apt-get install -y wget bzip2 ca-certificates curl git && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh && \ - /opt/conda/bin/conda clean -tipsy && \ - ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ - echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ - echo "conda activate base" >> ~/.bashrc +RUN apt-get update --fix-missing \ + && apt-get install -y wget bzip2 ca-certificates curl git \ + && apt-get clean && \ + && rm -rf /var/lib/apt/lists/* \ + && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh \ + && /bin/bash ~/miniconda.sh -b -p /opt/conda \ + && rm ~/miniconda.sh \ + && /opt/conda/bin/conda clean -tipsy \ + && ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh \ + && echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc \ + && echo "conda activate base" >> ~/.bashrc -RUN conda install -y tensorflow-gpu==1.14.0 -RUN conda install -y -c conda-forge musdb +RUN conda install -y cudatoolkit=9.0 \ + && conda install -y tensorflow-gpu==1.14.0 \ + && conda install -y -c conda-forge musdb # RUN conda install -y -c conda-forge museval # Note: switch to spleeter GPU once published. -RUN conda install -y -c conda-forge spleeter +RUN conda install -y -c conda-forge spleeter=1.4.4 ENTRYPOINT ["spleeter"] \ No newline at end of file