From 8af5ee2675b9b725b092838ef7c1be660d6b4eab Mon Sep 17 00:00:00 2001 From: Ali Akbar Date: Tue, 2 Jun 2020 11:18:50 +0530 Subject: [PATCH 01/57] missing command to switch to the created directory --- .github/CONTRIBUTING.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index ee16fc5..de9b1c6 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -14,7 +14,7 @@ Those are the main contributing guidelines for contributing to this project: In order to contribute, the safest is to create your [own fork of spleeter](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) first. The following set of commands will clone this new repository, create a virtual environment provisioned with the dependencies and run the tests (will take a few minutes): ```bash -git clone https://github.com//spleeter +git clone https://github.com//spleeter && cd spleeter python -m venv spleeterenv && source spleeterenv/bin/activate pip install -r requirements.txt && pip install pytest pytest-xdist make test diff --git a/README.md b/README.md index 97e7abb..c5bdb8d 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ For a detailed documentation, please check the [repository wiki](https://github. The following set of commands will clone this repository, create a virtual environment provisioned with the dependencies and run the tests (will take a few minutes): ```bash -git clone https://github.com/Deezer/spleeter +git clone https://github.com/Deezer/spleeter && cd spleeter python -m venv spleeterenv && source spleeterenv/bin/activate pip install -r requirements.txt && pip install pytest pytest-xdist make test From 8d442c88ac001af00422acd1addba2629967dfd5 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 18:01:03 +0200 Subject: [PATCH 02/57] Fixing gltches issues with Istft --- spleeter/separator.py | 10 +++++++--- tests/test_eval.py | 34 ++++++++++++++++++---------------- tests/test_separator.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 19 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index e769c28..7b6dab7 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -122,16 +122,20 @@ class Separator(object): assert not (inverse and length is None) data = np.asfortranarray(data) N = self._params["frame_length"] + pad_edges = int(N/4) H = self._params["frame_step"] win = hann(N, sym=False) fstft = istft if inverse else stft - win_len_arg = {"win_length": None, "length": length} if inverse else {"n_fft": N} + win_len_arg = {"win_length": None, "length": length + 2*pad_edges} if inverse else {"n_fft": N} n_channels = data.shape[-1] out = [] - for c in range(n_channels): - d = data[:, :, c].T if inverse else data[:, c] + for c in range(n_channels): + d = data[:, :, c].T if inverse else np.concatenate((np.zeros(pad_edges,), data[:,c], np.zeros(pad_edges,))) s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg) + if inverse: + s = s[pad_edges:-pad_edges] s = np.expand_dims(s.T, 2-inverse) + out.append(s) if len(out) == 1: return out[0] diff --git a/tests/test_eval.py b/tests/test_eval.py index 6a3634b..2227e18 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -26,28 +26,28 @@ from spleeter.commands import evaluate from spleeter.utils.configuration import load_configuration res_4stems = { "vocals": { - "SDR": -0.007, - "SAR": -19.231, - "SIR": -4.528, + "SDR": 0.000, + "SAR": -15.856, + "SIR": -6.861, "ISR": 0.000 }, "drums": { - "SDR": -0.071, - "SAR": -14.496, - "SIR": -4.987, + "SDR": -0.069, + "SAR": -15.769, + "SIR": -5.049, "ISR": 0.001 }, "bass":{ - "SDR": -0.001, - "SAR": -12.426, - "SIR": -7.198, - "ISR": -0.001 + "SDR": -0.000, + "SAR": -10.499, + "SIR": -6.988, + "ISR": -0.000 }, "other":{ - "SDR": -1.453, - "SAR": -14.899, - "SIR": -4.678, - "ISR": -0.015 + "SDR": -1.365, + "SAR": -14.591, + "SIR": -4.751, + "ISR": -0.016 } } @@ -75,5 +75,7 @@ def test_evaluate(path="FAKE_MUSDB_DIR"): params = load_configuration(arguments.configuration) metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): - for metric, value in metric.items(): - assert np.allclose(np.median(value), res_4stems[instrument][metric], atol=1e-3) \ No newline at end of file + print(instrument), print(metric) + for m, value in metric.items(): + print(np.median(value)), print(res_4stems[instrument][m]) + assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) \ No newline at end of file diff --git a/tests/test_separator.py b/tests/test_separator.py index 245571d..0b5b282 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -38,6 +38,44 @@ TEST_CONFIGURATIONS = list(itertools.product(TEST_AUDIO_DESCRIPTORS, MODELS, BAC print("RUNNING TESTS WITH TF VERSION {}".format(tf.__version__)) +@pytest.mark.parametrize('test_file', TEST_AUDIO_DESCRIPTORS) +def test_separator_backends(test_file): + adapter = get_default_audio_adapter() + waveform, _ = adapter.load(test_file) + + separator_lib = Separator("spleeter:2stems", stft_backend="librosa") + separator_tf = Separator("spleeter:2stems", stft_backend="tensorflow") + + # Test the stft and inverse stft provides exact reconstruction + stft_matrix = separator_lib._stft(waveform) + reconstructed = separator_lib._stft(stft_matrix, inverse=True, length= waveform.shape[0]) + assert np.allclose(reconstructed, waveform, atol=1e-2) + + # # now also test that tensorflow and librosa STFT provide same results + from spleeter.audio.spectrogram import compute_spectrogram_tf + tf_waveform = tf.convert_to_tensor(waveform, tf.float32) + spectrogram_tf = compute_spectrogram_tf(tf_waveform, + separator_tf._params['frame_length'], + separator_tf._params['frame_step'],) + with tf.Session() as sess: + spectrogram_tf_eval = spectrogram_tf.eval() + + # check that stfts are equivalent up to the padding in the librosa case + assert stft_matrix.shape[0] == spectrogram_tf_eval.shape[0] + 2 + assert stft_matrix.shape[1:] == spectrogram_tf_eval.shape[1:] + assert np.allclose(np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) + + # compare both separation, it should be close + out_tf = separator_tf._separate_tensorflow(waveform, test_file) + out_lib = separator_lib._separate_librosa(waveform, test_file) + + for instrument in out_lib.keys(): + # test that both outputs are not null + assert np.sum(np.abs(out_tf[instrument])) > 1000 + assert np.sum(np.abs(out_lib[instrument])) > 1000 + max_diff = np.max(np.abs(out_tf[instrument] - out_lib[instrument])) + print(f"Max diff on {instrument} is {max_diff}") + assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate(test_file, configuration, backend): From bec0555a62bf310eb1a6d55c60bed4caa23fc482 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 18:09:34 +0200 Subject: [PATCH 03/57] remove useless prints --- tests/test_eval.py | 2 -- tests/test_separator.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index 2227e18..a829706 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -75,7 +75,5 @@ def test_evaluate(path="FAKE_MUSDB_DIR"): params = load_configuration(arguments.configuration) metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): - print(instrument), print(metric) for m, value in metric.items(): - print(np.median(value)), print(res_4stems[instrument][m]) assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) \ No newline at end of file diff --git a/tests/test_separator.py b/tests/test_separator.py index 0b5b282..532d179 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -73,8 +73,6 @@ def test_separator_backends(test_file): # test that both outputs are not null assert np.sum(np.abs(out_tf[instrument])) > 1000 assert np.sum(np.abs(out_lib[instrument])) > 1000 - max_diff = np.max(np.abs(out_tf[instrument] - out_lib[instrument])) - print(f"Max diff on {instrument} is {max_diff}") assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From 45dc3566c24ff1eb3100442655e44bad9287b7c1 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 18:12:43 +0200 Subject: [PATCH 04/57] pep8 --- spleeter/separator.py | 9 +++++---- tests/test_separator.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index 7b6dab7..b204030 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -126,16 +126,17 @@ class Separator(object): H = self._params["frame_step"] win = hann(N, sym=False) fstft = istft if inverse else stft - win_len_arg = {"win_length": None, "length": length + 2*pad_edges} if inverse else {"n_fft": N} + win_len_arg = {"win_length": None, "length": length + + 2*pad_edges} if inverse else {"n_fft": N} n_channels = data.shape[-1] out = [] - for c in range(n_channels): - d = data[:, :, c].T if inverse else np.concatenate((np.zeros(pad_edges,), data[:,c], np.zeros(pad_edges,))) + for c in range(n_channels): + d = data[:, :, c].T if inverse else np.concatenate( + (np.zeros(pad_edges,), data[:, c], np.zeros(pad_edges,))) s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg) if inverse: s = s[pad_edges:-pad_edges] s = np.expand_dims(s.T, 2-inverse) - out.append(s) if len(out) == 1: return out[0] diff --git a/tests/test_separator.py b/tests/test_separator.py index 532d179..93ad148 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -38,6 +38,7 @@ TEST_CONFIGURATIONS = list(itertools.product(TEST_AUDIO_DESCRIPTORS, MODELS, BAC print("RUNNING TESTS WITH TF VERSION {}".format(tf.__version__)) + @pytest.mark.parametrize('test_file', TEST_AUDIO_DESCRIPTORS) def test_separator_backends(test_file): adapter = get_default_audio_adapter() @@ -48,22 +49,24 @@ def test_separator_backends(test_file): # Test the stft and inverse stft provides exact reconstruction stft_matrix = separator_lib._stft(waveform) - reconstructed = separator_lib._stft(stft_matrix, inverse=True, length= waveform.shape[0]) + reconstructed = separator_lib._stft( + stft_matrix, inverse=True, length=waveform.shape[0]) assert np.allclose(reconstructed, waveform, atol=1e-2) # # now also test that tensorflow and librosa STFT provide same results from spleeter.audio.spectrogram import compute_spectrogram_tf tf_waveform = tf.convert_to_tensor(waveform, tf.float32) spectrogram_tf = compute_spectrogram_tf(tf_waveform, - separator_tf._params['frame_length'], - separator_tf._params['frame_step'],) + separator_tf._params['frame_length'], + separator_tf._params['frame_step'],) with tf.Session() as sess: spectrogram_tf_eval = spectrogram_tf.eval() # check that stfts are equivalent up to the padding in the librosa case assert stft_matrix.shape[0] == spectrogram_tf_eval.shape[0] + 2 assert stft_matrix.shape[1:] == spectrogram_tf_eval.shape[1:] - assert np.allclose(np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) + assert np.allclose( + np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) # compare both separation, it should be close out_tf = separator_tf._separate_tensorflow(waveform, test_file) @@ -75,6 +78,7 @@ def test_separator_backends(test_file): assert np.sum(np.abs(out_lib[instrument])) > 1000 assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) + @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate(test_file, configuration, backend): """ Test separation from raw data. """ From 02f86e42b24a2d0028f08c1481491619ff95c035 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 20:17:03 +0200 Subject: [PATCH 05/57] investigate diff values --- spleeter/separator.py | 2 +- tests/test_separator.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index b204030..cc05a64 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -122,7 +122,7 @@ class Separator(object): assert not (inverse and length is None) data = np.asfortranarray(data) N = self._params["frame_length"] - pad_edges = int(N/4) + pad_edges = int(N/2) H = self._params["frame_step"] win = hann(N, sym=False) fstft = istft if inverse else stft diff --git a/tests/test_separator.py b/tests/test_separator.py index 93ad148..4c29dbf 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -74,9 +74,11 @@ def test_separator_backends(test_file): for instrument in out_lib.keys(): # test that both outputs are not null + print(np.sum(np.abs(out_tf[instrument]))) + print(np.sum(np.abs(out_lib[instrument]))) assert np.sum(np.abs(out_tf[instrument])) > 1000 assert np.sum(np.abs(out_lib[instrument])) > 1000 - assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) + assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.01) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From 9510e2b5f7a47c968b4c6686d338f482bece56e9 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 19 Jun 2020 00:33:19 +0200 Subject: [PATCH 06/57] dont pad the signal, only the stft matrix before inversion --- spleeter/separator.py | 13 +++++++------ tests/test_separator.py | 12 ++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index cc05a64..7cc9d9a 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -122,20 +122,21 @@ class Separator(object): assert not (inverse and length is None) data = np.asfortranarray(data) N = self._params["frame_length"] - pad_edges = int(N/2) H = self._params["frame_step"] + F = int(N/2) + 1 win = hann(N, sym=False) fstft = istft if inverse else stft - win_len_arg = {"win_length": None, "length": length + - 2*pad_edges} if inverse else {"n_fft": N} + win_len_arg = {"win_length": None, + "length": None} if inverse else {"n_fft": N} n_channels = data.shape[-1] out = [] for c in range(n_channels): - d = data[:, :, c].T if inverse else np.concatenate( - (np.zeros(pad_edges,), data[:, c], np.zeros(pad_edges,))) + d = np.concatenate((np.zeros((F, 1)), data[:, :, c].T, np.zeros( + (F, 1))), axis=1) if inverse else data[:, c] s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg) if inverse: - s = s[pad_edges:-pad_edges] + s = s[H:] + s = s[:length] s = np.expand_dims(s.T, 2-inverse) out.append(s) if len(out) == 1: diff --git a/tests/test_separator.py b/tests/test_separator.py index 4c29dbf..882acd9 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -51,7 +51,7 @@ def test_separator_backends(test_file): stft_matrix = separator_lib._stft(waveform) reconstructed = separator_lib._stft( stft_matrix, inverse=True, length=waveform.shape[0]) - assert np.allclose(reconstructed, waveform, atol=1e-2) + assert np.allclose(reconstructed, waveform, atol=3e-2) # # now also test that tensorflow and librosa STFT provide same results from spleeter.audio.spectrogram import compute_spectrogram_tf @@ -62,11 +62,10 @@ def test_separator_backends(test_file): with tf.Session() as sess: spectrogram_tf_eval = spectrogram_tf.eval() - # check that stfts are equivalent up to the padding in the librosa case - assert stft_matrix.shape[0] == spectrogram_tf_eval.shape[0] + 2 - assert stft_matrix.shape[1:] == spectrogram_tf_eval.shape[1:] + # check that stfts are equivalent + assert stft_matrix.shape == spectrogram_tf_eval.shape assert np.allclose( - np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) + np.abs(stft_matrix), spectrogram_tf_eval, atol=1e-2) # compare both separation, it should be close out_tf = separator_tf._separate_tensorflow(waveform, test_file) @@ -78,7 +77,8 @@ def test_separator_backends(test_file): print(np.sum(np.abs(out_lib[instrument]))) assert np.sum(np.abs(out_tf[instrument])) > 1000 assert np.sum(np.abs(out_lib[instrument])) > 1000 - assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.01) + print(np.max(out_tf[instrument]- out_lib[instrument])) + assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.025) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From af6dbec5cf2a8d2bbad728f9267ebf5204637053 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 19 Jun 2020 00:48:09 +0200 Subject: [PATCH 07/57] fix tests and more testing --- tests/test_eval.py | 22 +++++++++++----------- tests/test_separator.py | 9 +++------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index a829706..c39ea76 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -27,27 +27,27 @@ from spleeter.utils.configuration import load_configuration res_4stems = { "vocals": { "SDR": 0.000, - "SAR": -15.856, - "SIR": -6.861, + "SAR": -16.212, + "SIR": -4.172, "ISR": 0.000 }, "drums": { - "SDR": -0.069, - "SAR": -15.769, - "SIR": -5.049, + "SDR": -0.077, + "SAR": -15.739, + "SIR": -5.045, "ISR": 0.001 }, "bass":{ "SDR": -0.000, - "SAR": -10.499, - "SIR": -6.988, + "SAR": -10.665, + "SIR": -5.646, "ISR": -0.000 }, "other":{ - "SDR": -1.365, - "SAR": -14.591, - "SIR": -4.751, - "ISR": -0.016 + "SDR": -1.309, + "SAR": -14.573, + "SIR": -4.705, + "ISR": -0.014 } } diff --git a/tests/test_separator.py b/tests/test_separator.py index 882acd9..21840b2 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -72,13 +72,10 @@ def test_separator_backends(test_file): out_lib = separator_lib._separate_librosa(waveform, test_file) for instrument in out_lib.keys(): - # test that both outputs are not null - print(np.sum(np.abs(out_tf[instrument]))) - print(np.sum(np.abs(out_lib[instrument]))) - assert np.sum(np.abs(out_tf[instrument])) > 1000 - assert np.sum(np.abs(out_lib[instrument])) > 1000 - print(np.max(out_tf[instrument]- out_lib[instrument])) + # test that both outputs are close everywhere assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.025) + # it should be even more similar outside edges zones + assert np.allclose(out_tf[instrument][4096:-4096,:], out_lib[instrument][4096:-4096,:], atol=0.002) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From caf8f393bd02e6915194e310272c6bcfda7f977d Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 26 Jun 2020 11:03:41 +0200 Subject: [PATCH 08/57] need to force flush the RAM in between pytest calls and avoid multiprocess --- Makefile | 2 +- tests/test_separator.py | 97 +++++++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index e8f710e..8661db9 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ build-gpu: clean python3 setup.py sdist test: - $(PYTEST_CMD) + $(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) deploy: diff --git a/tests/test_separator.py b/tests/test_separator.py index c688f0f..2d2686a 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -42,67 +42,68 @@ print("RUNNING TESTS WITH TF VERSION {}".format(tf.__version__)) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate(test_file, configuration, backend): """ Test separation from raw data. """ - with tf.Session() as sess: - instruments = MODEL_TO_INST[configuration] - adapter = get_default_audio_adapter() - waveform, _ = adapter.load(test_file) - separator = Separator(configuration, stft_backend=backend) - prediction = separator.separate(waveform, test_file) - assert len(prediction) == len(instruments) - for instrument in instruments: - assert instrument in prediction - for instrument in instruments: - track = prediction[instrument] - assert waveform.shape[:-1] == track.shape[:-1] - assert not np.allclose(waveform, track) - for compared in instruments: - if instrument != compared: - assert not np.allclose(track, prediction[compared]) + tf.reset_default_graph() + instruments = MODEL_TO_INST[configuration] + adapter = get_default_audio_adapter() + waveform, _ = adapter.load(test_file) + separator = Separator(configuration, stft_backend=backend, multiprocess=False) + prediction = separator.separate(waveform, test_file) + assert len(prediction) == len(instruments) + for instrument in instruments: + assert instrument in prediction + for instrument in instruments: + track = prediction[instrument] + assert waveform.shape[:-1] == track.shape[:-1] + assert not np.allclose(waveform, track) + for compared in instruments: + if instrument != compared: + assert not np.allclose(track, prediction[compared]) + @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate_to_file(test_file, configuration, backend): """ Test file based separation. """ - with tf.Session() as sess: - instruments = MODEL_TO_INST[configuration] - separator = Separator(configuration, stft_backend=backend) - name = splitext(basename(test_file))[0] - with TemporaryDirectory() as directory: - separator.separate_to_file( - test_file, - directory) - for instrument in instruments: - assert exists(join( - directory, - '{}/{}.wav'.format(name, instrument))) + tf.reset_default_graph() + instruments = MODEL_TO_INST[configuration] + separator = Separator(configuration, stft_backend=backend, multiprocess=False) + name = splitext(basename(test_file))[0] + with TemporaryDirectory() as directory: + separator.separate_to_file( + test_file, + directory) + for instrument in instruments: + assert exists(join( + directory, + '{}/{}.wav'.format(name, instrument))) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_filename_format(test_file, configuration, backend): """ Test custom filename format. """ - with tf.Session() as sess: - instruments = MODEL_TO_INST[configuration] - separator = Separator(configuration, stft_backend=backend) - name = splitext(basename(test_file))[0] - with TemporaryDirectory() as directory: - separator.separate_to_file( - test_file, + tf.reset_default_graph() + instruments = MODEL_TO_INST[configuration] + separator = Separator(configuration, stft_backend=backend, multiprocess=False) + name = splitext(basename(test_file))[0] + with TemporaryDirectory() as directory: + separator.separate_to_file( + test_file, + directory, + filename_format='export/{filename}/{instrument}.{codec}') + for instrument in instruments: + assert exists(join( directory, - filename_format='export/{filename}/{instrument}.{codec}') - for instrument in instruments: - assert exists(join( - directory, - 'export/{}/{}.wav'.format(name, instrument))) + 'export/{}/{}.wav'.format(name, instrument))) @pytest.mark.parametrize('test_file, configuration', MODELS_AND_TEST_FILES) def test_filename_conflict(test_file, configuration): """ Test error handling with static pattern. """ - with tf.Session() as sess: - separator = Separator(configuration) - with TemporaryDirectory() as directory: - with pytest.raises(SpleeterError): - separator.separate_to_file( - test_file, - directory, - filename_format='I wanna be your lover') + tf.reset_default_graph() + separator = Separator(configuration, multiprocess=False) + with TemporaryDirectory() as directory: + with pytest.raises(SpleeterError): + separator.separate_to_file( + test_file, + directory, + filename_format='I wanna be your lover') From 1f21b5fedce5145708c1e550134a7cb282d14ce9 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 26 Jun 2020 16:31:11 +0200 Subject: [PATCH 09/57] back to forked and no reset --- tests/test_separator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_separator.py b/tests/test_separator.py index 2d2686a..691f112 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -42,7 +42,6 @@ print("RUNNING TESTS WITH TF VERSION {}".format(tf.__version__)) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate(test_file, configuration, backend): """ Test separation from raw data. """ - tf.reset_default_graph() instruments = MODEL_TO_INST[configuration] adapter = get_default_audio_adapter() waveform, _ = adapter.load(test_file) @@ -64,7 +63,6 @@ def test_separate(test_file, configuration, backend): @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate_to_file(test_file, configuration, backend): """ Test file based separation. """ - tf.reset_default_graph() instruments = MODEL_TO_INST[configuration] separator = Separator(configuration, stft_backend=backend, multiprocess=False) name = splitext(basename(test_file))[0] @@ -81,7 +79,6 @@ def test_separate_to_file(test_file, configuration, backend): @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_filename_format(test_file, configuration, backend): """ Test custom filename format. """ - tf.reset_default_graph() instruments = MODEL_TO_INST[configuration] separator = Separator(configuration, stft_backend=backend, multiprocess=False) name = splitext(basename(test_file))[0] @@ -99,7 +96,6 @@ def test_filename_format(test_file, configuration, backend): @pytest.mark.parametrize('test_file, configuration', MODELS_AND_TEST_FILES) def test_filename_conflict(test_file, configuration): """ Test error handling with static pattern. """ - tf.reset_default_graph() separator = Separator(configuration, multiprocess=False) with TemporaryDirectory() as directory: with pytest.raises(SpleeterError): From d3d83571cebd394f6daceaab938bda0c42e01f27 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 26 Jun 2020 17:16:09 +0200 Subject: [PATCH 10/57] downgrading pytest-forked version --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cd5ff97..30c5cfd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: - restore_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb museval + - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - run: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} @@ -30,7 +30,7 @@ jobs: - restore_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb museval + - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - run: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} From 3d6990291d97a11c87c2e3257716f3ec2a4eff9f Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 26 Jun 2020 21:33:09 +0200 Subject: [PATCH 11/57] parameter name could have changed --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8661db9..d226d6b 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ FEEDSTOCK = spleeter-feedstock FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK) FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml -PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked +PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --boxed -s all: clean build test deploy From e8647e3f916a0cabf82bc29d95828e10542011eb Mon Sep 17 00:00:00 2001 From: romi1502 Date: Fri, 3 Jul 2020 17:25:58 +0200 Subject: [PATCH 12/57] Added padding at the begining to avoid tf STFT reconstruction erro --- spleeter/model/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/spleeter/model/__init__.py b/spleeter/model/__init__.py index 531f84c..29763b6 100644 --- a/spleeter/model/__init__.py +++ b/spleeter/model/__init__.py @@ -275,9 +275,16 @@ class EstimatorSpecBuilder(object): spec_name = self.spectrogram_name if stft_name not in self._features: + # pad input with a frame of zeros + waveform = tf.concat([ + tf.zeros((self._frame_length, self._n_channels)), + self._features['waveform'] + ], + 0 + ) stft_feature = tf.transpose( stft( - tf.transpose(self._features['waveform']), + tf.transpose(waveform), self._frame_length, self._frame_step, window_fn=lambda frame_length, dtype: ( @@ -341,7 +348,7 @@ class EstimatorSpecBuilder(object): reshaped = tf.transpose(inversed) if time_crop is None: time_crop = tf.shape(self._features['waveform'])[0] - return reshaped[:time_crop, :] + return reshaped[self._frame_length:self._frame_length+time_crop, :] def _build_mwf_output_waveform(self): """ Perform separation with multichannel Wiener Filtering using Norbert. From 80a6a4e3ddb0f596176356a3148dfc5e0bcba646 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Mon, 6 Jul 2020 11:41:26 +0200 Subject: [PATCH 13/57] =?UTF-8?q?unboxing=20test=C3=A8eval?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d226d6b..7f6eeda 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ FEEDSTOCK = spleeter-feedstock FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK) FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml -PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --boxed -s +PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv all: clean build test deploy @@ -27,7 +27,12 @@ build-gpu: clean python3 setup.py sdist test: - $(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) + #$(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) + $(PYTEST_CMD) tests/test_eval.py + $(PYTEST_CMD) tests/test_ffmpeg_adapter.py + $(PYTEST_CMD) tests/test_github_model_provider.py + $(PYTEST_CMD) --boxed tests/test_separator.py + deploy: From 82f29601b1d1083ea44a027094e715f5081ae658 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Wed, 8 Jul 2020 11:19:32 +0200 Subject: [PATCH 14/57] deactivate test_eval --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7f6eeda..6ed5340 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ build-gpu: clean test: #$(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) - $(PYTEST_CMD) tests/test_eval.py + #$(PYTEST_CMD) tests/test_eval.py $(PYTEST_CMD) tests/test_ffmpeg_adapter.py $(PYTEST_CMD) tests/test_github_model_provider.py $(PYTEST_CMD) --boxed tests/test_separator.py From b0ec2a50943ede05815b6ed6c2791fe31fb116e3 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Wed, 8 Jul 2020 11:22:05 +0200 Subject: [PATCH 15/57] deactivate test_eval --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 6ed5340..740d6cd 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ build-gpu: clean test: #$(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) + # deactivate this until we figure out why it fails on the CI container #$(PYTEST_CMD) tests/test_eval.py $(PYTEST_CMD) tests/test_ffmpeg_adapter.py $(PYTEST_CMD) tests/test_github_model_provider.py From afce3b95f1e6b2d7dd5680155c759d48f7bcde81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 14:59:50 +0200 Subject: [PATCH 16/57] =?UTF-8?q?=F0=9F=91=B7=20setup=20Github=20actions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 78 ++++++++++++++++++++++++++++++++++++ .github/workflows/pypi.yml | 27 +++++++++++++ .github/workflows/pytest.yml | 23 +++++++++++ Makefile | 2 - 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/pypi.yml create mode 100644 .github/workflows/pytest.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..16cd8ab --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,78 @@ +name: docker +on: + - workflow_dispatch +jobs: + build-test-push: + runs-on: ubuntu-latest + strategy: + matrix: + platform: [cpu, gpu] + distribution: [3.6, 3.7, conda] + model: [modelless, 2stems, 4stems, 5stems] + fail-fast: true + steps: + - uses: actions/checkout@v2 + # ---------------------------------------------------------------------- + # Note: base image building and env setup. + - name: Setup Python distribution + run: | + echo "::set-env name=base::python:${{ matrix.distribution }}" + echo "::set-env name=tag::${{ matrix.distribution }}" + echo "::set-env name=file::spleeter" + - if: ${{ matrix.distribution }} == 'conda' + name: Build Conda base image + run: | + docker build -t python:conda -f docker/conda.dockerfile . + echo "::set-env name=file::spleeter-conda" + - if: ${{ matrix.platform }} == 'gpu' + name: Build CUDA base image + run: | + docker build \ + --build-arg BASE=python:${{ matrix.distribution }} + -t cuda:${{ matrix.distribution }} \ + -f docker/cuda-10-0.dockerfile . + echo "::set-env name=base::cuda:${{ matrix.distribution }}" + echo "::set-env name=tag::${{ matrix.distribution }}-gpu" + # ---------------------------------------------------------------------- + # Note: image building. + - name: Build researchdeezer/spleeter:${{ env.tag }} image + run: | + docker build \ + --build-arg BASE=${{ env.base }} \ + -t researchdeezer/spleeter:${{ env.tag }} \ + -f docker/${{ env.file }}.dockerfile . + echo "::set-env name=modelargs::" + - if: ${{ matrix.model }} != 'modelless' + name: Build researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} image + run: | + docker build \ + --build-arg BASE=researchdeezer/spleeter:${{ env.tag }} \ + --build-arg MODEL=${{ matrix.model }} \ + -t researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ + -f docker/spleeter-model.dockerfile . + echo "::set-env name=tag::${{ env.tag }}-${{ matrix.model }}" + echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}" + # ---------------------------------------------------------------------- + # Note: image testing. + - name: Test researchdeezer/spleeter:${{ env.tag }} image + run: | + docker run \ + -v $(pwd):/runtime \ + researchdeezer/spleeter:${{ env.tag }} \ + separate -i /runtime/audio_example.mp3 -o /tmp \${{ env.modelarg }} + # ---------------------------------------------------------------------- + # Note: image deploy. + - name: Docker login + run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + - name: Push researchdeezer/spleeter:${{ env.tag }} image + run: docker push researchdeezer/spleeter:${{ env.tag }} + - if: ${{ env.tag }} == 'spleeter:3.7' + name: Push researchdeezer/spleeter:latest image + run: | + docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest + docker push researchdeezer/spleeter:latest + - if: ${{ env.tag }} == 'spleeter:3.7-gpu' + name: Push researchdeezer/spleeter:gpu image + run: | + docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu + docker push researchdeezer/spleeter:gpu diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..8f3e457 --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,27 @@ +name: pypi +on: + pull_request: + types: + - closed + branches: + - master +env: + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} +jobs: + deploy-cpu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Package sdist + run: make build + - name: Deploy to pypi + run: make deploy + deploy-gpu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Package sdist + run: make build-gpu + - name: Deploy to pypi + run: make deploy \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..41299e7 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,23 @@ +name: pytest +on: + - push + - pull_request +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + apt-get update && apt-get install -y ffmpeg + pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval + python setup.py install + - name: Test with pytest + run: make test \ No newline at end of file diff --git a/Makefile b/Makefile index 740d6cd..a67a64d 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,6 @@ test: $(PYTEST_CMD) tests/test_ffmpeg_adapter.py $(PYTEST_CMD) tests/test_github_model_provider.py $(PYTEST_CMD) --boxed tests/test_separator.py - - deploy: pip install twine From 75ebb62dd722a892a98aca26608f4c22786da5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:02:40 +0200 Subject: [PATCH 17/57] =?UTF-8?q?=F0=9F=92=9A=20switch=20to=20sudo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 41299e7..52f07f0 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -16,7 +16,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - apt-get update && apt-get install -y ffmpeg + sudo apt-get update && sudo apt-get install -y ffmpeg pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval python setup.py install - name: Test with pytest From 6d12077b451d90ce0d6b3388966389a08ced9b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:24:26 +0200 Subject: [PATCH 18/57] =?UTF-8?q?=F0=9F=91=B7=20add=20pip=20upgrade?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 52f07f0..a6e5b42 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,7 +17,8 @@ jobs: - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -y ffmpeg + pip install --upgrade pip setuptools pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - python setup.py install + pip setup.py install --user - name: Test with pytest run: make test \ No newline at end of file From 5182795d3610afbf264c7f402907db031cc6a7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:26:14 +0200 Subject: [PATCH 19/57] =?UTF-8?q?=F0=9F=90=9B=20syntax=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index a6e5b42..49acfee 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,6 +19,6 @@ jobs: sudo apt-get update && sudo apt-get install -y ffmpeg pip install --upgrade pip setuptools pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - pip setup.py install --user + python setup.py install - name: Test with pytest run: make test \ No newline at end of file From 1fdcff31f8ac9b3a0b4d18028000d1bdb3137584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:50:12 +0200 Subject: [PATCH 20/57] =?UTF-8?q?=F0=9F=91=B7=20add=20model=20and=20packag?= =?UTF-8?q?e=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 28 +++++++++++++++++++--------- .github/workflows/pytest.yml | 10 ++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 8f3e457..26a98c8 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -9,19 +9,29 @@ env: TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} jobs: - deploy-cpu: + package-and-deploy: + strategy: + matrix: + platform: [cpu, gpu] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Package sdist + - name: Cache sdist packages + uses: actions/cache@v2 + env: + cache-name: cache-sdist-packages + with: + path: dist + key: sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + restore-keys: | + sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + sdist-${{ matrix.platform }} + sdist- + - if: ${{ matrix.platform }} == 'cpu' + name: Package CPU distribution run: make build - - name: Deploy to pypi - run: make deploy - deploy-gpu: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Package sdist + - if: ${{ matrix.platform }} == 'gpu' + name: Package GPU distribution run: make build-gpu - name: Deploy to pypi run: make deploy \ No newline at end of file diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 49acfee..2b9019b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -14,6 +14,16 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Cache spleeter models + uses: actions/cache@v2 + env: + cache-name: cache-spleeter-model + with: + path: pretrained_models + key: models-${{ hashFiles('spleeter/model/__init__.py') }} + restore-keys: | + models-${{ hashFiles('spleeter/model/__init__.py') }} + models- - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -y ffmpeg From 040a9fdcfbda892e4454b3148a585a8eb69a43e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 15:54:46 +0200 Subject: [PATCH 21/57] =?UTF-8?q?=E2=9C=85=20restore=20old=20test=20suite?= =?UTF-8?q?=20fashion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 285 ------------------------------------------- Makefile | 9 +- 2 files changed, 2 insertions(+), 292 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 30c5cfd..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,285 +0,0 @@ -version: 2 -jobs: - # ======================================================================================= - # Python 3.6 testing. - # ======================================================================================= - test-3.6: - docker: - - image: python:3.6 - working_directory: ~/spleeter - steps: - - checkout - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - - run: make test - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" - # ======================================================================================= - # Python 3.7 testing. - # ======================================================================================= - test-3.7: - docker: - - image: python:3.7 - working_directory: ~/spleeter - steps: - - checkout - - restore_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest==5.4.3 pytest-xdist==1.32.0 pytest-forked==1.1.3 musdb museval - - run: make test - - save_cache: - key: models-{{ checksum "spleeter/model/__init__.py" }} - paths: - - "pretrained_models" - # ======================================================================================= - # Source distribution packaging. - # ======================================================================================= - sdist: - docker: - - image: python:3 - steps: - - checkout - - run: make build - - save_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - dist - sdist-gpu: - docker: - - image: python:3 - steps: - - checkout - - run: make build-gpu - - save_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - paths: - - dist - # ======================================================================================= - # PyPi deployment. - # ======================================================================================= - pypi-deploy: - docker: - - image: python:3 - steps: - - checkout - - restore_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - - run: - name: upload to PyPi - # TODO: Infer destination regarding of branch. - # - master => production PyPi - # - other => testing PyPi - command: make build deploy - pypi-deploy-gpu: - docker: - - image: python:3 - steps: - - checkout - - restore_cache: - key: sdist-{{ .Branch }}-{{ checksum "setup.py" }} - - run: - name: upload to PyPi - # TODO: Infer destination regarding of branch. - # - master => production PyPi - # - other => testing PyPi - command: make build-gpu deploy - # ======================================================================================= - # Docker build. - # ======================================================================================= - - docker-conda-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t conda -f docker/conda.dockerfile . - - run: docker build --build-arg BASE=conda -t researchdeezer/spleeter:conda -f docker/spleeter-conda.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:conda - - run: docker push researchdeezer/spleeter:conda-2stems - - run: docker push researchdeezer/spleeter:conda-4stems - - run: docker push researchdeezer/spleeter:conda-5stems - docker-conda-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t conda -f docker/conda.dockerfile . - - run: docker build --build-arg BASE=conda -t conda-gpu -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=conda-gpu --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:conda-gpu -f docker/spleeter-conda.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:conda-gpu - - run: docker push researchdeezer/spleeter:conda-gpu-2stems - - run: docker push researchdeezer/spleeter:conda-gpu-4stems - - run: docker push researchdeezer/spleeter:conda-gpu-5stems - docker-3.6-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.6 -t researchdeezer/spleeter:3.6 -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6 separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.6 - - run: docker push researchdeezer/spleeter:3.6-2stems - - run: docker push researchdeezer/spleeter:3.6-4stems - - run: docker push researchdeezer/spleeter:3.6-5stems - docker-3.6-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.6 -t cuda:3.6 -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=cuda:3.6 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.6-gpu -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.6-gpu - - run: docker push researchdeezer/spleeter:3.6-gpu-2stems - - run: docker push researchdeezer/spleeter:3.6-gpu-4stems - - run: docker push researchdeezer/spleeter:3.6-gpu-5stems - docker-3.7-cpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build --build-arg BASE=python:3.7 -t researchdeezer/spleeter:3.7 -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-5stems -f docker/spleeter-model.dockerfile . - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7 separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-2stems separate -i /runtime/audio_example.mp3 -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp - - run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-5stems separate -i /runtime/audio_example.mp3 -p spleeter:5stems -o /tmp - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest - - run: docker push researchdeezer/spleeter:latest - - run: docker push researchdeezer/spleeter:3.7 - - run: docker push researchdeezer/spleeter:3.7-2stems - - run: docker push researchdeezer/spleeter:3.7-4stems - - run: docker push researchdeezer/spleeter:3.7-5stems - docker-3.7-gpu: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: docker build -t cuda:3.7 -f docker/cuda-10-0.dockerfile . - - run: docker build --build-arg BASE=cuda:3.7 --build-arg SPLEETER_PACKAGE=spleeter-gpu -t researchdeezer/spleeter:3.7-gpu -f docker/spleeter.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-gpu-2stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-gpu-4stems -f docker/spleeter-model.dockerfile . - - run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-gpu-5stems -f docker/spleeter-model.dockerfile . - - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - - run: docker push researchdeezer/spleeter:3.7-gpu - - run: docker push researchdeezer/spleeter:3.7-gpu-2stems - - run: docker push researchdeezer/spleeter:3.7-gpu-4stems - - run: docker push researchdeezer/spleeter:3.7-gpu-5stems - - run: docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu - - run: docker push researchdeezer/spleeter:gpu - -workflows: - version: 2 - spleeter-release-pipeline: - jobs: - - test-3.6 - - test-3.7 - - sdist: - requires: - - test-3.6 - - test-3.7 - - sdist-gpu: - requires: - - test-3.6 - - test-3.7 - - pypi-deploy: - filters: - branches: - only: - - master - requires: - - sdist - - pypi-deploy-gpu: - filters: - branches: - only: - - master - requires: - - sdist-gpu - - conda-forge-validation: - type: approval - requires: - - pypi-deploy - - pypi-deploy-gpu - filters: - branches: - only: - - master - - docker-conda-cpu: - requires: - - conda-forge-validation - filters: - branches: - only: - - master - - docker-conda-gpu: - requires: - - conda-forge-validation - filters: - branches: - only: - - master - - docker-3.6-cpu: - requires: - - pypi-deploy - filters: - branches: - only: - - master - - docker-3.6-gpu: - requires: - - pypi-deploy-gpu - filters: - branches: - only: - - master - - docker-3.7-cpu: - requires: - - pypi-deploy - filters: - branches: - only: - - master - - docker-3.7-gpu: - requires: - - pypi-deploy-gpu - filters: - branches: - only: - - master diff --git a/Makefile b/Makefile index a67a64d..4db4e36 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ FEEDSTOCK = spleeter-feedstock FEEDSTOCK_REPOSITORY = https://github.com/deezer/$(FEEDSTOCK) FEEDSTOCK_RECIPE = $(FEEDSTOCK)/recipe/spleeter/meta.yaml -PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv +PYTEST_CMD = pytest -W ignore::FutureWarning -W ignore::DeprecationWarning -vv --forked all: clean build test deploy @@ -27,12 +27,7 @@ build-gpu: clean python3 setup.py sdist test: - #$(foreach file, $(wildcard tests/test_*.py), $(PYTEST_CMD) $(file);) - # deactivate this until we figure out why it fails on the CI container - #$(PYTEST_CMD) tests/test_eval.py - $(PYTEST_CMD) tests/test_ffmpeg_adapter.py - $(PYTEST_CMD) tests/test_github_model_provider.py - $(PYTEST_CMD) --boxed tests/test_separator.py + $(PYTEST_CMD) tests/ deploy: pip install twine From 531f0db15e8e5361d5a1bf65dff2b5dfe3bdc9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:00:20 +0200 Subject: [PATCH 22/57] =?UTF-8?q?=F0=9F=91=B7=20add=20pip=20cache=20and=20?= =?UTF-8?q?clean=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 9 +++------ .github/workflows/pytest.yml | 15 +++++++++------ requirements.txt | 9 --------- 3 files changed, 12 insertions(+), 21 deletions(-) delete mode 100644 requirements.txt diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 26a98c8..ad9996f 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -16,15 +16,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache sdist packages - uses: actions/cache@v2 - env: - cache-name: cache-sdist-packages + - uses: actions/cache@v2 with: path: dist - key: sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + key: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} restore-keys: | - sdist-${{ matrix.platform }}-${{ hashFiles('setup.py') }} + sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} sdist-${{ matrix.platform }} sdist- - if: ${{ matrix.platform }} == 'cpu' diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2b9019b..4fd7c19 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -14,15 +14,18 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Cache spleeter models - uses: actions/cache@v2 - env: - cache-name: cache-spleeter-model + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - uses: actions/cache@v2 with: path: pretrained_models - key: models-${{ hashFiles('spleeter/model/__init__.py') }} + key: models-${{ hashFiles('**/setup.py') }} restore-keys: | - models-${{ hashFiles('spleeter/model/__init__.py') }} + models-${{ hashFiles('**/setup.py') }} models- - name: Install dependencies run: | diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 28f7286..0000000 --- a/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -importlib_resources; python_version<'3.7' -requests -setuptools>=41.0.0 -pandas==0.25.1 -tensorflow==1.15.2 -ffmpeg-python -norbert==0.2.1 -librosa==0.7.2 -numba==0.48.0 \ No newline at end of file From 99ef2b30fbf5fbd32ce8ba997abb4adb610a9a36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:06:15 +0200 Subject: [PATCH 23/57] =?UTF-8?q?=F0=9F=93=9D=20update=20CI=20badge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2249c7f..702e0bd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -[![CircleCI](https://circleci.com/gh/deezer/spleeter/tree/master.svg?style=shield)](https://circleci.com/gh/deezer/spleeter/tree/master) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) +[![Github actions](https://github.com/deezer/spleeter/workflows/pytest/badge.svg)](https://github.com/deezer/spleeter/actions) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/spleeter) [![PyPI version](https://badge.fury.io/py/spleeter.svg)](https://badge.fury.io/py/spleeter) [![Conda](https://img.shields.io/conda/vn/conda-forge/spleeter)](https://anaconda.org/conda-forge/spleeter) [![Docker Pulls](https://img.shields.io/docker/pulls/researchdeezer/spleeter)](https://hub.docker.com/r/researchdeezer/spleeter) [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deezer/spleeter/blob/master/spleeter.ipynb) [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/spleeter/community) [![status](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b/status.svg)](https://joss.theoj.org/papers/259e5efe669945a343bad6eccb89018b) ## About From d15b417ef5d2d0fcd67fba22fee14aa06e5954a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:30:45 +0200 Subject: [PATCH 24/57] =?UTF-8?q?=F0=9F=91=B7=20update=20pytest=20events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4fd7c19..fe7dff6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,7 +1,5 @@ name: pytest -on: - - push - - pull_request +on: [push, pull_request] jobs: tests: runs-on: ubuntu-latest From bd1d8d4f54c8e06790e5b2366a7a47f09afaa6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:41:51 +0200 Subject: [PATCH 25/57] =?UTF-8?q?=F0=9F=91=B7=20add=20branch=20filter=20fo?= =?UTF-8?q?r=20pytest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index fe7dff6..7895ab6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,5 +1,12 @@ name: pytest -on: [push, pull_request] +on: + push: + branches: + - '**' + - '!master' + pull_request: + branches: + - 'master' jobs: tests: runs-on: ubuntu-latest From 21a742aa1db5398c5bec6eca0c19ca47352b6b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:44:51 +0200 Subject: [PATCH 26/57] =?UTF-8?q?=F0=9F=90=9B=20branch=20ignore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7895ab6..27b175b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,9 +1,8 @@ name: pytest on: push: - branches: - - '**' - - '!master' + branches-ignore: + - 'master' pull_request: branches: - 'master' From d8c9e66e01bcd2ac94d89b60cce799a11811e320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:48:29 +0200 Subject: [PATCH 27/57] =?UTF-8?q?=F0=9F=90=9B=20remove=20pypi=20event?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 4 +--- .github/workflows/pytest.yml | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index ad9996f..015c13f 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -1,8 +1,6 @@ name: pypi on: - pull_request: - types: - - closed + push: branches: - master env: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 27b175b..1bf1576 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,11 +1,11 @@ name: pytest on: - push: - branches-ignore: - - 'master' pull_request: branches: - - 'master' + - master + push: + branches-ignore: + - master jobs: tests: runs-on: ubuntu-latest From 223e68747cd1e6dc3e448493c980bd45ad7ccd70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:49:42 +0200 Subject: [PATCH 28/57] =?UTF-8?q?=F0=9F=91=B7=20remove=20CI=20on=20push?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 1bf1576..66ef563 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -3,9 +3,6 @@ on: pull_request: branches: - master - push: - branches-ignore: - - master jobs: tests: runs-on: ubuntu-latest From 6e3b6ef78332a131ab2d3b794f65dd77186145bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 16:55:40 +0200 Subject: [PATCH 29/57] =?UTF-8?q?=F0=9F=90=9B=20=20add=20setuptools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 015c13f..bd25f52 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -14,6 +14,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- - uses: actions/cache@v2 with: path: dist @@ -22,6 +28,8 @@ jobs: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} sdist-${{ matrix.platform }} sdist- + - name: Install dependencies + run: pip install --upgrade pip setuptools twine - if: ${{ matrix.platform }} == 'cpu' name: Package CPU distribution run: make build From 02fab48cfb7978b2dd76842ab25906b50cf3ce08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:13:37 +0200 Subject: [PATCH 30/57] =?UTF-8?q?=E2=9C=A8=20add=202=20CPU=20testing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4db4e36..26129d3 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ build-gpu: clean python3 setup.py sdist test: - $(PYTEST_CMD) tests/ + $(PYTEST_CMD) -n 2 tests/ deploy: pip install twine From 9b9b3f3a252fef1290c2789b5c12801766cda0ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:19:10 +0200 Subject: [PATCH 31/57] =?UTF-8?q?=F0=9F=90=9B=20caching=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 2 +- .github/workflows/pytest.yml | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index bd25f52..473f74c 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -22,7 +22,7 @@ jobs: ${{ runner.os }}-pip- - uses: actions/cache@v2 with: - path: dist + path: ${{ env.GITHUB_WORKSPACE }}/dist key: sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} restore-keys: | sdist-${{ matrix.platform }}-${{ hashFiles('**/setup.py') }} diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 66ef563..d4b2965 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -16,18 +16,21 @@ jobs: with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 + id: spleeter-pip-cache with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- - uses: actions/cache@v2 + env: + model-release: 1 + id: spleeter-model-cache with: - path: pretrained_models - key: models-${{ hashFiles('**/setup.py') }} + path: ${{ env.GITHUB_WORKSPACE }}/pretrained_models + key: models-${{ env.model-release }} restore-keys: | - models-${{ hashFiles('**/setup.py') }} - models- + models-${{ env.model-release }} - name: Install dependencies run: | sudo apt-get update && sudo apt-get install -y ffmpeg From 5c8bb826691b970a6a45f703507f126df706f65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:33:54 +0200 Subject: [PATCH 32/57] =?UTF-8?q?=F0=9F=92=9A=20=20setup=20python=20versio?= =?UTF-8?q?n=20for=20PyPi=20wf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 473f74c..26d4e4e 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -14,6 +14,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: 3.7 - uses: actions/cache@v2 with: path: ~/.cache/pip From b114fd032c1f848f0f164b9c1d8770c52dd0d034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:38:04 +0200 Subject: [PATCH 33/57] =?UTF-8?q?=F0=9F=90=9B=20fix=20conditional=20steps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 26d4e4e..3a0594b 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -33,11 +33,11 @@ jobs: sdist- - name: Install dependencies run: pip install --upgrade pip setuptools twine - - if: ${{ matrix.platform }} == 'cpu' - name: Package CPU distribution + - name: Package CPU distribution + if: ${{ matrix.platform == 'cpu' }} run: make build - - if: ${{ matrix.platform }} == 'gpu' - name: Package GPU distribution + - name: Package GPU distribution + if: ${{ matrix.platform == 'gpu' }} run: make build-gpu - name: Deploy to pypi run: make deploy \ No newline at end of file From 486460008f64adcafbec8f244eb96a24f291428e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:40:57 +0200 Subject: [PATCH 34/57] =?UTF-8?q?=F0=9F=90=9B=20=20fix=20docker=20worfklow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 10 +++++----- .github/workflows/pypi.yml | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 16cd8ab..aa06371 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,12 +19,12 @@ jobs: echo "::set-env name=base::python:${{ matrix.distribution }}" echo "::set-env name=tag::${{ matrix.distribution }}" echo "::set-env name=file::spleeter" - - if: ${{ matrix.distribution }} == 'conda' + - if: ${{ matrix.distribution == 'conda' }} name: Build Conda base image run: | docker build -t python:conda -f docker/conda.dockerfile . echo "::set-env name=file::spleeter-conda" - - if: ${{ matrix.platform }} == 'gpu' + - if: ${{ matrix.platform == 'gpu' }} name: Build CUDA base image run: | docker build \ @@ -42,7 +42,7 @@ jobs: -t researchdeezer/spleeter:${{ env.tag }} \ -f docker/${{ env.file }}.dockerfile . echo "::set-env name=modelargs::" - - if: ${{ matrix.model }} != 'modelless' + - if: ${{ matrix.model != 'modelless' }} name: Build researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} image run: | docker build \ @@ -66,12 +66,12 @@ jobs: run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - name: Push researchdeezer/spleeter:${{ env.tag }} image run: docker push researchdeezer/spleeter:${{ env.tag }} - - if: ${{ env.tag }} == 'spleeter:3.7' + - if: ${{ env.tag == 'spleeter:3.7' }} name: Push researchdeezer/spleeter:latest image run: | docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest docker push researchdeezer/spleeter:latest - - if: ${{ env.tag }} == 'spleeter:3.7-gpu' + - if: ${{ env.tag == 'spleeter:3.7-gpu' }} name: Push researchdeezer/spleeter:gpu image run: | docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 3a0594b..e2cd0b2 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -33,11 +33,11 @@ jobs: sdist- - name: Install dependencies run: pip install --upgrade pip setuptools twine - - name: Package CPU distribution - if: ${{ matrix.platform == 'cpu' }} + - if: ${{ matrix.platform == 'cpu' }} + name: Package CPU distribution run: make build - - name: Package GPU distribution - if: ${{ matrix.platform == 'gpu' }} + - if: ${{ matrix.platform == 'gpu' }} + name: Package GPU distribution) run: make build-gpu - name: Deploy to pypi run: make deploy \ No newline at end of file From 9311941066038141ed523232b3f33299153a6754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Wed, 8 Jul 2020 17:45:55 +0200 Subject: [PATCH 35/57] =?UTF-8?q?=F0=9F=90=9B=20fix=20multi=20line=20comma?= =?UTF-8?q?nd=20for=20CUDA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index aa06371..38831c5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ jobs: name: Build CUDA base image run: | docker build \ - --build-arg BASE=python:${{ matrix.distribution }} + --build-arg BASE=python:${{ matrix.distribution }} \ -t cuda:${{ matrix.distribution }} \ -f docker/cuda-10-0.dockerfile . echo "::set-env name=base::cuda:${{ matrix.distribution }}" From a9de3eb5062b807d041acdcc3178563ef953f182 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 17 Jul 2020 09:43:31 +0200 Subject: [PATCH 36/57] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 702e0bd..1dbfb40 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ As is commonly the case with open-source projects, there are multiple forks expo ## Contributing -If you would like to participate in the development of **spleeter** your are more than welcome to do so. Don't hesitate to throw us a pull request and we'll do our best to examine it quickly. Please check out our [guidelines](.github/CONTRIBUTING.md) first. +If you would like to participate in the development of **spleeter** you are more than welcome to do so. Don't hesitate to throw us a pull request and we'll do our best to examine it quickly. Please check out our [guidelines](.github/CONTRIBUTING.md) first. ## Note From 227020c256ef5c09674238979b2a76f91b5f0c68 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 17 Jul 2020 13:30:42 +0200 Subject: [PATCH 37/57] replace mail --- Makefile | 2 +- setup.py | 4 ++-- spleeter/__init__.py | 2 +- spleeter/__main__.py | 2 +- spleeter/audio/__init__.py | 2 +- spleeter/audio/adapter.py | 2 +- spleeter/audio/convertor.py | 2 +- spleeter/audio/ffmpeg.py | 2 +- spleeter/audio/spectrogram.py | 2 +- spleeter/commands/__init__.py | 2 +- spleeter/commands/evaluate.py | 7 ++++++- spleeter/commands/separate.py | 2 +- spleeter/commands/train.py | 2 +- spleeter/dataset.py | 2 +- spleeter/model/__init__.py | 2 +- spleeter/model/functions/__init__.py | 2 +- spleeter/model/functions/blstm.py | 2 +- spleeter/model/functions/unet.py | 2 +- spleeter/model/provider/__init__.py | 2 +- spleeter/model/provider/github.py | 2 +- spleeter/resources/__init__.py | 2 +- spleeter/separator.py | 2 +- spleeter/utils/__init__.py | 2 +- spleeter/utils/configuration.py | 2 +- spleeter/utils/logging.py | 2 +- spleeter/utils/tensor.py | 2 +- tests/__init__.py | 2 +- tests/test_eval.py | 4 ++-- tests/test_ffmpeg_adapter.py | 2 +- tests/test_separator.py | 2 +- 30 files changed, 37 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 26129d3..c2b40ba 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # ======================================================= # Library lifecycle management. # -# @author Deezer Research +# @author Deezer Research # @licence MIT Licence # ======================================================= diff --git a/setup.py b/setup.py index 72f7a0b..2465cfa 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ import sys from os import path from setuptools import setup -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' @@ -33,7 +33,7 @@ setup( long_description=readme, long_description_content_type='text/markdown', author='Deezer Research', - author_email='research@deezer.com', + author_email='spleeter@deezer.com', url='https://github.com/deezer/spleeter', license='MIT License', packages=[ diff --git a/spleeter/__init__.py b/spleeter/__init__.py index 0650c97..c2329ed 100644 --- a/spleeter/__init__.py +++ b/spleeter/__init__.py @@ -13,7 +13,7 @@ by providing train, evaluation and source separation action. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/__main__.py b/spleeter/__main__.py index 5c18b21..e4f3f4a 100644 --- a/spleeter/__main__.py +++ b/spleeter/__main__.py @@ -18,7 +18,7 @@ from .utils.logging import ( enable_tensorflow_logging, get_logger) -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/audio/__init__.py b/spleeter/audio/__init__.py index 02f83c0..3d973c5 100644 --- a/spleeter/audio/__init__.py +++ b/spleeter/audio/__init__.py @@ -10,6 +10,6 @@ - Waveform convertion and transforming functions. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/audio/adapter.py b/spleeter/audio/adapter.py index bda1441..2bfcc9b 100644 --- a/spleeter/audio/adapter.py +++ b/spleeter/audio/adapter.py @@ -19,7 +19,7 @@ from tensorflow.contrib.signal import stft, hann_window from .. import SpleeterError from ..utils.logging import get_logger -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/audio/convertor.py b/spleeter/audio/convertor.py index 550345a..0751b03 100644 --- a/spleeter/audio/convertor.py +++ b/spleeter/audio/convertor.py @@ -10,7 +10,7 @@ import tensorflow as tf from ..utils.tensor import from_float32_to_uint8, from_uint8_to_float32 -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/audio/ffmpeg.py b/spleeter/audio/ffmpeg.py index 2277d87..890e02e 100644 --- a/spleeter/audio/ffmpeg.py +++ b/spleeter/audio/ffmpeg.py @@ -20,7 +20,7 @@ from .adapter import AudioAdapter from .. import SpleeterError from ..utils.logging import get_logger -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/audio/spectrogram.py b/spleeter/audio/spectrogram.py index 38d0cf5..80ff65a 100644 --- a/spleeter/audio/spectrogram.py +++ b/spleeter/audio/spectrogram.py @@ -10,7 +10,7 @@ import tensorflow as tf from tensorflow.contrib.signal import stft, hann_window # pylint: enable=import-error -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/commands/__init__.py b/spleeter/commands/__init__.py index 995e866..79773db 100644 --- a/spleeter/commands/__init__.py +++ b/spleeter/commands/__init__.py @@ -9,7 +9,7 @@ from argparse import ArgumentParser from tempfile import gettempdir from os.path import exists, join -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/commands/evaluate.py b/spleeter/commands/evaluate.py index 1e1e3f1..64dbf09 100644 --- a/spleeter/commands/evaluate.py +++ b/spleeter/commands/evaluate.py @@ -38,7 +38,7 @@ except ImportError: logger.error('Please install musdb and museval first, abort') sys.exit(1) -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' @@ -148,11 +148,16 @@ def entrypoint(arguments, params): if not exists(musdb_root_directory): raise IOError(f'musdb directory {musdb_root_directory} not found') # Separate musdb sources. + verbose = arguments.verbose + if verbose: + print("Separating audio files") audio_output_directory = _separate_evaluation_dataset( arguments, musdb_root_directory, params) # Compute metrics with musdb. + if verbose: + print("Compute Metrics with musdb") metrics_output_directory = _compute_musdb_metrics( arguments, musdb_root_directory, diff --git a/spleeter/commands/separate.py b/spleeter/commands/separate.py index 158190c..193d8f6 100644 --- a/spleeter/commands/separate.py +++ b/spleeter/commands/separate.py @@ -14,7 +14,7 @@ from ..audio.adapter import get_audio_adapter from ..separator import Separator -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/commands/train.py b/spleeter/commands/train.py index 2a40c84..3bffaef 100644 --- a/spleeter/commands/train.py +++ b/spleeter/commands/train.py @@ -19,7 +19,7 @@ from ..model import model_fn from ..model.provider import ModelProvider from ..utils.logging import get_logger -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/dataset.py b/spleeter/dataset.py index fe2d349..d6d5d03 100644 --- a/spleeter/dataset.py +++ b/spleeter/dataset.py @@ -38,7 +38,7 @@ from .utils.tensor import ( set_tensor_shape, sync_apply) -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/model/__init__.py b/spleeter/model/__init__.py index 531f84c..2097e9c 100644 --- a/spleeter/model/__init__.py +++ b/spleeter/model/__init__.py @@ -13,7 +13,7 @@ from tensorflow.contrib.signal import stft, inverse_stft, hann_window from ..utils.tensor import pad_and_partition, pad_and_reshape -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/model/functions/__init__.py b/spleeter/model/functions/__init__.py index abe52e9..684f923 100644 --- a/spleeter/model/functions/__init__.py +++ b/spleeter/model/functions/__init__.py @@ -3,7 +3,7 @@ """ This package provide model functions. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/model/functions/blstm.py b/spleeter/model/functions/blstm.py index ff7ce02..b81122b 100644 --- a/spleeter/model/functions/blstm.py +++ b/spleeter/model/functions/blstm.py @@ -33,7 +33,7 @@ from tensorflow.keras.layers import ( from . import apply -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/model/functions/unet.py b/spleeter/model/functions/unet.py index 60dea1e..7f9dbea 100644 --- a/spleeter/model/functions/unet.py +++ b/spleeter/model/functions/unet.py @@ -32,7 +32,7 @@ from tensorflow.compat.v1.keras.initializers import he_uniform from . import apply -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/model/provider/__init__.py b/spleeter/model/provider/__init__.py index 3aa3d8d..3921907 100644 --- a/spleeter/model/provider/__init__.py +++ b/spleeter/model/provider/__init__.py @@ -15,7 +15,7 @@ from abc import ABC, abstractmethod from os import environ, makedirs from os.path import exists, isabs, join, sep -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/model/provider/github.py b/spleeter/model/provider/github.py index cd4d904..65a10b4 100644 --- a/spleeter/model/provider/github.py +++ b/spleeter/model/provider/github.py @@ -25,7 +25,7 @@ import requests from . import ModelProvider from ...utils.logging import get_logger -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/resources/__init__.py b/spleeter/resources/__init__.py index 41d2a65..01939fb 100644 --- a/spleeter/resources/__init__.py +++ b/spleeter/resources/__init__.py @@ -3,6 +3,6 @@ """ Packages that provides static resources file for the library. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/separator.py b/spleeter/separator.py index e769c28..35d8ae2 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -31,7 +31,7 @@ from .utils.estimator import create_estimator, to_predictor, get_default_model_d from .model import EstimatorSpecBuilder, InputProviderFactory -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/utils/__init__.py b/spleeter/utils/__init__.py index a4ccb5b..8828652 100644 --- a/spleeter/utils/__init__.py +++ b/spleeter/utils/__init__.py @@ -3,6 +3,6 @@ """ This package provides utility function and classes. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/utils/configuration.py b/spleeter/utils/configuration.py index d1fb167..36f1043 100644 --- a/spleeter/utils/configuration.py +++ b/spleeter/utils/configuration.py @@ -16,7 +16,7 @@ from os.path import exists from .. import resources, SpleeterError -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/utils/logging.py b/spleeter/utils/logging.py index 15431d0..6fee540 100644 --- a/spleeter/utils/logging.py +++ b/spleeter/utils/logging.py @@ -7,7 +7,7 @@ import logging from os import environ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/spleeter/utils/tensor.py b/spleeter/utils/tensor.py index 402548c..cc5e7e8 100644 --- a/spleeter/utils/tensor.py +++ b/spleeter/utils/tensor.py @@ -8,7 +8,7 @@ import tensorflow as tf import pandas as pd # pylint: enable=import-error -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/tests/__init__.py b/tests/__init__.py index f584f49..a58c36b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,6 +3,6 @@ """ Unit testing package. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/tests/test_eval.py b/tests/test_eval.py index 6a3634b..21229b1 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -3,7 +3,7 @@ """ Unit testing for Separator class. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' @@ -71,7 +71,7 @@ def generate_fake_eval_dataset(path): def test_evaluate(path="FAKE_MUSDB_DIR"): generate_fake_eval_dataset(path) p = create_argument_parser() - arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path]) + arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path, '--verbose']) params = load_configuration(arguments.configuration) metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): diff --git a/tests/test_ffmpeg_adapter.py b/tests/test_ffmpeg_adapter.py index 195b737..8eb284a 100644 --- a/tests/test_ffmpeg_adapter.py +++ b/tests/test_ffmpeg_adapter.py @@ -3,7 +3,7 @@ """ Unit testing for audio adapter. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' diff --git a/tests/test_separator.py b/tests/test_separator.py index 691f112..7213af6 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -3,7 +3,7 @@ """ Unit testing for Separator class. """ -__email__ = 'research@deezer.com' +__email__ = 'spleeter@deezer.com' __author__ = 'Deezer Research' __license__ = 'MIT License' From 137fca6d105a7d5b8c822067a75ed515a18ef1f6 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 17 Jul 2020 13:43:43 +0200 Subject: [PATCH 38/57] remove debugging residuals --- spleeter/commands/evaluate.py | 5 ----- tests/test_eval.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/spleeter/commands/evaluate.py b/spleeter/commands/evaluate.py index 64dbf09..3ddfb74 100644 --- a/spleeter/commands/evaluate.py +++ b/spleeter/commands/evaluate.py @@ -148,16 +148,11 @@ def entrypoint(arguments, params): if not exists(musdb_root_directory): raise IOError(f'musdb directory {musdb_root_directory} not found') # Separate musdb sources. - verbose = arguments.verbose - if verbose: - print("Separating audio files") audio_output_directory = _separate_evaluation_dataset( arguments, musdb_root_directory, params) # Compute metrics with musdb. - if verbose: - print("Compute Metrics with musdb") metrics_output_directory = _compute_musdb_metrics( arguments, musdb_root_directory, diff --git a/tests/test_eval.py b/tests/test_eval.py index 21229b1..bb9edef 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -71,7 +71,7 @@ def generate_fake_eval_dataset(path): def test_evaluate(path="FAKE_MUSDB_DIR"): generate_fake_eval_dataset(path) p = create_argument_parser() - arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path, '--verbose']) + arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path]) params = load_configuration(arguments.configuration) metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): From 4a3074e55beb4f1f13917673283501f333e99d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 17 Jul 2020 13:54:43 +0200 Subject: [PATCH 39/57] =?UTF-8?q?=F0=9F=90=9B=20remove=20concurrent=20test?= =?UTF-8?q?=20execution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c2b40ba..d667361 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ build-gpu: clean python3 setup.py sdist test: - $(PYTEST_CMD) -n 2 tests/ + $(PYTEST_CMD) tests/ deploy: pip install twine From e66937594cef61b72a039e633c80fe479f226559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 17 Jul 2020 16:04:17 +0200 Subject: [PATCH 40/57] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2465cfa..22b6ec3 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ __license__ = 'MIT License' # Default project values. project_name = 'spleeter' -project_version = '1.5.3' +project_version = '1.5.4' tensorflow_dependency = 'tensorflow' tensorflow_version = '1.15.2' here = path.abspath(path.dirname(__file__)) From 5cb5c63916ebd032a91174ed70c848eec085ca40 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 24 Jul 2020 12:23:47 +0200 Subject: [PATCH 41/57] missing parameter --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 38831c5..5718fdf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,6 +29,7 @@ jobs: run: | docker build \ --build-arg BASE=python:${{ matrix.distribution }} \ + --build-arg SPLEETER_PACKAGE=spleeter-gpu \ -t cuda:${{ matrix.distribution }} \ -f docker/cuda-10-0.dockerfile . echo "::set-env name=base::cuda:${{ matrix.distribution }}" From 4744ffbc847741c451714083fecb9b2a585537d8 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 24 Jul 2020 13:00:35 +0200 Subject: [PATCH 42/57] use env var for package in dowker workflow --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5718fdf..c9363fd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,6 +19,7 @@ jobs: echo "::set-env name=base::python:${{ matrix.distribution }}" echo "::set-env name=tag::${{ matrix.distribution }}" echo "::set-env name=file::spleeter" + echo "::set-env name=package::spleeter" - if: ${{ matrix.distribution == 'conda' }} name: Build Conda base image run: | @@ -29,17 +30,18 @@ jobs: run: | docker build \ --build-arg BASE=python:${{ matrix.distribution }} \ - --build-arg SPLEETER_PACKAGE=spleeter-gpu \ -t cuda:${{ matrix.distribution }} \ -f docker/cuda-10-0.dockerfile . echo "::set-env name=base::cuda:${{ matrix.distribution }}" echo "::set-env name=tag::${{ matrix.distribution }}-gpu" + echo "::set-env name=package::spleeter-gpu" # ---------------------------------------------------------------------- # Note: image building. - name: Build researchdeezer/spleeter:${{ env.tag }} image run: | docker build \ --build-arg BASE=${{ env.base }} \ + --build-arg SPLEETER_PACKAGE=${{ env.package }} \ -t researchdeezer/spleeter:${{ env.tag }} \ -f docker/${{ env.file }}.dockerfile . echo "::set-env name=modelargs::" From b03ef93be974bfb702af2b10fe9ffd1dd6704974 Mon Sep 17 00:00:00 2001 From: romi1502 Date: Fri, 24 Jul 2020 15:01:40 +0200 Subject: [PATCH 43/57] Added backend option for eval --- spleeter/commands/__init__.py | 1 + spleeter/commands/evaluate.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/spleeter/commands/__init__.py b/spleeter/commands/__init__.py index 995e866..ae5f184 100644 --- a/spleeter/commands/__init__.py +++ b/spleeter/commands/__init__.py @@ -170,6 +170,7 @@ def _create_evaluate_parser(parser_factory): parser.add_argument('-o', '--output_path', **OPT_OUTPUT) parser.add_argument('--mus_dir', **OPT_MUSDB) parser.add_argument('-m', '--mwf', **OPT_MWF) + parser.add_argument('-B', '--stft-backend', **OPT_STFT_BACKEND) return parser diff --git a/spleeter/commands/evaluate.py b/spleeter/commands/evaluate.py index 1e1e3f1..35c7181 100644 --- a/spleeter/commands/evaluate.py +++ b/spleeter/commands/evaluate.py @@ -77,7 +77,7 @@ def _separate_evaluation_dataset(arguments, musdb_root_directory, params): bitrate='128k', MWF=arguments.MWF, verbose=arguments.verbose, - stft_backend="auto"), + stft_backend=arguments.stft_backend), params) return audio_output_directory From 3fcc4ea28f024db77e0be2c8481372ca9c609ddc Mon Sep 17 00:00:00 2001 From: romi1502 Date: Fri, 24 Jul 2020 15:02:34 +0200 Subject: [PATCH 44/57] Added eval test for both backends --- tests/test_eval.py | 101 +++++++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index 6a3634b..82e0034 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -25,33 +25,64 @@ from spleeter.commands import evaluate from spleeter.utils.configuration import load_configuration -res_4stems = { "vocals": { - "SDR": -0.007, - "SAR": -19.231, - "SIR": -4.528, - "ISR": 0.000 +BACKENDS = ["tensorflow", "librosa"] +TEST_CONFIGURATIONS = {el:el for el in BACKENDS} + +res_4stems = { + "librosa": { + "vocals": { + "SDR": -0.007, + "SAR": -19.231, + "SIR": -4.528, + "ISR": 0.000 + }, + "drums": { + "SDR": -0.071, + "SAR": -14.496, + "SIR": -4.987, + "ISR": 0.001 + }, + "bass":{ + "SDR": -0.001, + "SAR": -12.426, + "SIR": -7.198, + "ISR": -0.001 + }, + "other":{ + "SDR": -1.453, + "SAR": -14.899, + "SIR": -4.678, + "ISR": -0.015 + } }, - "drums": { - "SDR": -0.071, - "SAR": -14.496, - "SIR": -4.987, - "ISR": 0.001 - }, - "bass":{ - "SDR": -0.001, - "SAR": -12.426, - "SIR": -7.198, - "ISR": -0.001 - }, - "other":{ - "SDR": -1.453, - "SAR": -14.899, - "SIR": -4.678, - "ISR": -0.015 + "tensorflow": { + "vocals": { + "SDR": 3.25e-05, + "SAR": -11.153575, + "SIR": -1.3849, + "ISR": 2.75e-05 + }, + "drums": { + "SDR": -0.079505, + "SAR": -15.7073575, + "SIR": -4.972755, + "ISR": 0.0013575 + }, + "bass":{ + "SDR": 2.5e-06, + "SAR": -10.3520575, + "SIR": -4.272325, + "ISR": 2.5e-06 + }, + "other":{ + "SDR": -1.359175, + "SAR": -14.7076775, + "SIR": -4.761505, + "ISR": -0.01528 + } } } - def generate_fake_eval_dataset(path): aa = get_default_audio_adapter() n_songs = 2 @@ -68,12 +99,18 @@ def generate_fake_eval_dataset(path): aa.save(filename, data, fs) -def test_evaluate(path="FAKE_MUSDB_DIR"): - generate_fake_eval_dataset(path) - p = create_argument_parser() - arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path]) - params = load_configuration(arguments.configuration) - metrics = evaluate.entrypoint(arguments, params) - for instrument, metric in metrics.items(): - for metric, value in metric.items(): - assert np.allclose(np.median(value), res_4stems[instrument][metric], atol=1e-3) \ No newline at end of file +@pytest.mark.parametrize('backend', TEST_CONFIGURATIONS) +def test_evaluate(backend): + with TemporaryDirectory() as directory: + + generate_fake_eval_dataset(directory) + p = create_argument_parser() + arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", directory, "-B", backend]) + params = load_configuration(arguments.configuration) + metrics = evaluate.entrypoint(arguments, params) + for instrument, metric in metrics.items(): + for metric, value in metric.items(): + assert np.allclose(np.median(value), res_4stems[backend][instrument][metric], atol=1e-3) + + +# test_evaluate("tensorflow") \ No newline at end of file From 27f52b71d5405af62345b7d819db196240239cb4 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 18:01:03 +0200 Subject: [PATCH 45/57] Fixing gltches issues with Istft --- spleeter/separator.py | 10 +++++++--- tests/test_eval.py | 26 +++++++++++--------------- tests/test_separator.py | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index 35d8ae2..ceceeca 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -122,16 +122,20 @@ class Separator(object): assert not (inverse and length is None) data = np.asfortranarray(data) N = self._params["frame_length"] + pad_edges = int(N/4) H = self._params["frame_step"] win = hann(N, sym=False) fstft = istft if inverse else stft - win_len_arg = {"win_length": None, "length": length} if inverse else {"n_fft": N} + win_len_arg = {"win_length": None, "length": length + 2*pad_edges} if inverse else {"n_fft": N} n_channels = data.shape[-1] out = [] - for c in range(n_channels): - d = data[:, :, c].T if inverse else data[:, c] + for c in range(n_channels): + d = data[:, :, c].T if inverse else np.concatenate((np.zeros(pad_edges,), data[:,c], np.zeros(pad_edges,))) s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg) + if inverse: + s = s[pad_edges:-pad_edges] s = np.expand_dims(s.T, 2-inverse) + out.append(s) if len(out) == 1: return out[0] diff --git a/tests/test_eval.py b/tests/test_eval.py index 298e421..e9bf762 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -99,18 +99,14 @@ def generate_fake_eval_dataset(path): aa.save(filename, data, fs) -@pytest.mark.parametrize('backend', TEST_CONFIGURATIONS) -def test_evaluate(backend): - with TemporaryDirectory() as directory: - - generate_fake_eval_dataset(directory) - p = create_argument_parser() - arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", directory, "-B", backend]) - params = load_configuration(arguments.configuration) - metrics = evaluate.entrypoint(arguments, params) - for instrument, metric in metrics.items(): - for metric, value in metric.items(): - assert np.allclose(np.median(value), res_4stems[backend][instrument][metric], atol=1e-3) - - -# test_evaluate("tensorflow") \ No newline at end of file +def test_evaluate(path="FAKE_MUSDB_DIR"): + generate_fake_eval_dataset(path) + p = create_argument_parser() + arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path]) + params = load_configuration(arguments.configuration) + metrics = evaluate.entrypoint(arguments, params) + for instrument, metric in metrics.items(): + print(instrument), print(metric) + for m, value in metric.items(): + print(np.median(value)), print(res_4stems[instrument][m]) + assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) diff --git a/tests/test_separator.py b/tests/test_separator.py index 7213af6..06f5676 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -38,6 +38,44 @@ TEST_CONFIGURATIONS = list(itertools.product(TEST_AUDIO_DESCRIPTORS, MODELS, BAC print("RUNNING TESTS WITH TF VERSION {}".format(tf.__version__)) +@pytest.mark.parametrize('test_file', TEST_AUDIO_DESCRIPTORS) +def test_separator_backends(test_file): + adapter = get_default_audio_adapter() + waveform, _ = adapter.load(test_file) + + separator_lib = Separator("spleeter:2stems", stft_backend="librosa") + separator_tf = Separator("spleeter:2stems", stft_backend="tensorflow") + + # Test the stft and inverse stft provides exact reconstruction + stft_matrix = separator_lib._stft(waveform) + reconstructed = separator_lib._stft(stft_matrix, inverse=True, length= waveform.shape[0]) + assert np.allclose(reconstructed, waveform, atol=1e-2) + + # # now also test that tensorflow and librosa STFT provide same results + from spleeter.audio.spectrogram import compute_spectrogram_tf + tf_waveform = tf.convert_to_tensor(waveform, tf.float32) + spectrogram_tf = compute_spectrogram_tf(tf_waveform, + separator_tf._params['frame_length'], + separator_tf._params['frame_step'],) + with tf.Session() as sess: + spectrogram_tf_eval = spectrogram_tf.eval() + + # check that stfts are equivalent up to the padding in the librosa case + assert stft_matrix.shape[0] == spectrogram_tf_eval.shape[0] + 2 + assert stft_matrix.shape[1:] == spectrogram_tf_eval.shape[1:] + assert np.allclose(np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) + + # compare both separation, it should be close + out_tf = separator_tf._separate_tensorflow(waveform, test_file) + out_lib = separator_lib._separate_librosa(waveform, test_file) + + for instrument in out_lib.keys(): + # test that both outputs are not null + assert np.sum(np.abs(out_tf[instrument])) > 1000 + assert np.sum(np.abs(out_lib[instrument])) > 1000 + max_diff = np.max(np.abs(out_tf[instrument] - out_lib[instrument])) + print(f"Max diff on {instrument} is {max_diff}") + assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate(test_file, configuration, backend): From 4b40685f9dbb9edd1baa65e0e9d3a243069d4119 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 18:09:34 +0200 Subject: [PATCH 46/57] remove useless prints --- tests/test_eval.py | 4 +--- tests/test_separator.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index e9bf762..fcd37a6 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -106,7 +106,5 @@ def test_evaluate(path="FAKE_MUSDB_DIR"): params = load_configuration(arguments.configuration) metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): - print(instrument), print(metric) for m, value in metric.items(): - print(np.median(value)), print(res_4stems[instrument][m]) - assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) + assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) \ No newline at end of file diff --git a/tests/test_separator.py b/tests/test_separator.py index 06f5676..f2a3f6d 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -73,8 +73,6 @@ def test_separator_backends(test_file): # test that both outputs are not null assert np.sum(np.abs(out_tf[instrument])) > 1000 assert np.sum(np.abs(out_lib[instrument])) > 1000 - max_diff = np.max(np.abs(out_tf[instrument] - out_lib[instrument])) - print(f"Max diff on {instrument} is {max_diff}") assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From 0ed145fa3efb6eaeedfa19473a7123b6f9924ea2 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 18:12:43 +0200 Subject: [PATCH 47/57] pep8 --- spleeter/separator.py | 9 +++++---- tests/test_separator.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index ceceeca..b0b0ff7 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -126,16 +126,17 @@ class Separator(object): H = self._params["frame_step"] win = hann(N, sym=False) fstft = istft if inverse else stft - win_len_arg = {"win_length": None, "length": length + 2*pad_edges} if inverse else {"n_fft": N} + win_len_arg = {"win_length": None, "length": length + + 2*pad_edges} if inverse else {"n_fft": N} n_channels = data.shape[-1] out = [] - for c in range(n_channels): - d = data[:, :, c].T if inverse else np.concatenate((np.zeros(pad_edges,), data[:,c], np.zeros(pad_edges,))) + for c in range(n_channels): + d = data[:, :, c].T if inverse else np.concatenate( + (np.zeros(pad_edges,), data[:, c], np.zeros(pad_edges,))) s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg) if inverse: s = s[pad_edges:-pad_edges] s = np.expand_dims(s.T, 2-inverse) - out.append(s) if len(out) == 1: return out[0] diff --git a/tests/test_separator.py b/tests/test_separator.py index f2a3f6d..ccfd086 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -38,6 +38,7 @@ TEST_CONFIGURATIONS = list(itertools.product(TEST_AUDIO_DESCRIPTORS, MODELS, BAC print("RUNNING TESTS WITH TF VERSION {}".format(tf.__version__)) + @pytest.mark.parametrize('test_file', TEST_AUDIO_DESCRIPTORS) def test_separator_backends(test_file): adapter = get_default_audio_adapter() @@ -48,22 +49,24 @@ def test_separator_backends(test_file): # Test the stft and inverse stft provides exact reconstruction stft_matrix = separator_lib._stft(waveform) - reconstructed = separator_lib._stft(stft_matrix, inverse=True, length= waveform.shape[0]) + reconstructed = separator_lib._stft( + stft_matrix, inverse=True, length=waveform.shape[0]) assert np.allclose(reconstructed, waveform, atol=1e-2) # # now also test that tensorflow and librosa STFT provide same results from spleeter.audio.spectrogram import compute_spectrogram_tf tf_waveform = tf.convert_to_tensor(waveform, tf.float32) spectrogram_tf = compute_spectrogram_tf(tf_waveform, - separator_tf._params['frame_length'], - separator_tf._params['frame_step'],) + separator_tf._params['frame_length'], + separator_tf._params['frame_step'],) with tf.Session() as sess: spectrogram_tf_eval = spectrogram_tf.eval() # check that stfts are equivalent up to the padding in the librosa case assert stft_matrix.shape[0] == spectrogram_tf_eval.shape[0] + 2 assert stft_matrix.shape[1:] == spectrogram_tf_eval.shape[1:] - assert np.allclose(np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) + assert np.allclose( + np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) # compare both separation, it should be close out_tf = separator_tf._separate_tensorflow(waveform, test_file) @@ -75,6 +78,7 @@ def test_separator_backends(test_file): assert np.sum(np.abs(out_lib[instrument])) > 1000 assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) + @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) def test_separate(test_file, configuration, backend): """ Test separation from raw data. """ From 94dedbe949f2d9771a490eed38b1fa049cd0d252 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Thu, 18 Jun 2020 20:17:03 +0200 Subject: [PATCH 48/57] investigate diff values --- spleeter/separator.py | 2 +- tests/test_separator.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index b0b0ff7..0776361 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -122,7 +122,7 @@ class Separator(object): assert not (inverse and length is None) data = np.asfortranarray(data) N = self._params["frame_length"] - pad_edges = int(N/4) + pad_edges = int(N/2) H = self._params["frame_step"] win = hann(N, sym=False) fstft = istft if inverse else stft diff --git a/tests/test_separator.py b/tests/test_separator.py index ccfd086..49b416c 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -74,9 +74,11 @@ def test_separator_backends(test_file): for instrument in out_lib.keys(): # test that both outputs are not null + print(np.sum(np.abs(out_tf[instrument]))) + print(np.sum(np.abs(out_lib[instrument]))) assert np.sum(np.abs(out_tf[instrument])) > 1000 assert np.sum(np.abs(out_lib[instrument])) > 1000 - assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.1) + assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.01) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From 97458816c9c4ff293f584c60e194ded3a31b48ff Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 19 Jun 2020 00:33:19 +0200 Subject: [PATCH 49/57] dont pad the signal, only the stft matrix before inversion --- spleeter/separator.py | 13 +++++++------ tests/test_separator.py | 12 ++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index 0776361..9318911 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -122,20 +122,21 @@ class Separator(object): assert not (inverse and length is None) data = np.asfortranarray(data) N = self._params["frame_length"] - pad_edges = int(N/2) H = self._params["frame_step"] + F = int(N/2) + 1 win = hann(N, sym=False) fstft = istft if inverse else stft - win_len_arg = {"win_length": None, "length": length + - 2*pad_edges} if inverse else {"n_fft": N} + win_len_arg = {"win_length": None, + "length": None} if inverse else {"n_fft": N} n_channels = data.shape[-1] out = [] for c in range(n_channels): - d = data[:, :, c].T if inverse else np.concatenate( - (np.zeros(pad_edges,), data[:, c], np.zeros(pad_edges,))) + d = np.concatenate((np.zeros((F, 1)), data[:, :, c].T, np.zeros( + (F, 1))), axis=1) if inverse else data[:, c] s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg) if inverse: - s = s[pad_edges:-pad_edges] + s = s[H:] + s = s[:length] s = np.expand_dims(s.T, 2-inverse) out.append(s) if len(out) == 1: diff --git a/tests/test_separator.py b/tests/test_separator.py index 49b416c..71d62ca 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -51,7 +51,7 @@ def test_separator_backends(test_file): stft_matrix = separator_lib._stft(waveform) reconstructed = separator_lib._stft( stft_matrix, inverse=True, length=waveform.shape[0]) - assert np.allclose(reconstructed, waveform, atol=1e-2) + assert np.allclose(reconstructed, waveform, atol=3e-2) # # now also test that tensorflow and librosa STFT provide same results from spleeter.audio.spectrogram import compute_spectrogram_tf @@ -62,11 +62,10 @@ def test_separator_backends(test_file): with tf.Session() as sess: spectrogram_tf_eval = spectrogram_tf.eval() - # check that stfts are equivalent up to the padding in the librosa case - assert stft_matrix.shape[0] == spectrogram_tf_eval.shape[0] + 2 - assert stft_matrix.shape[1:] == spectrogram_tf_eval.shape[1:] + # check that stfts are equivalent + assert stft_matrix.shape == spectrogram_tf_eval.shape assert np.allclose( - np.abs(stft_matrix[1:-1]), spectrogram_tf_eval, atol=1e-2) + np.abs(stft_matrix), spectrogram_tf_eval, atol=1e-2) # compare both separation, it should be close out_tf = separator_tf._separate_tensorflow(waveform, test_file) @@ -78,7 +77,8 @@ def test_separator_backends(test_file): print(np.sum(np.abs(out_lib[instrument]))) assert np.sum(np.abs(out_tf[instrument])) > 1000 assert np.sum(np.abs(out_lib[instrument])) > 1000 - assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.01) + print(np.max(out_tf[instrument]- out_lib[instrument])) + assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.025) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From bba1341bdf839f5bf06081ab0a04193e2589493e Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 19 Jun 2020 00:48:09 +0200 Subject: [PATCH 50/57] fix tests and more testing --- tests/test_eval.py | 4 +++- tests/test_separator.py | 9 +++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index fcd37a6..e9bf762 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -106,5 +106,7 @@ def test_evaluate(path="FAKE_MUSDB_DIR"): params = load_configuration(arguments.configuration) metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): + print(instrument), print(metric) for m, value in metric.items(): - assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) \ No newline at end of file + print(np.median(value)), print(res_4stems[instrument][m]) + assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) diff --git a/tests/test_separator.py b/tests/test_separator.py index 71d62ca..d850e97 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -72,13 +72,10 @@ def test_separator_backends(test_file): out_lib = separator_lib._separate_librosa(waveform, test_file) for instrument in out_lib.keys(): - # test that both outputs are not null - print(np.sum(np.abs(out_tf[instrument]))) - print(np.sum(np.abs(out_lib[instrument]))) - assert np.sum(np.abs(out_tf[instrument])) > 1000 - assert np.sum(np.abs(out_lib[instrument])) > 1000 - print(np.max(out_tf[instrument]- out_lib[instrument])) + # test that both outputs are close everywhere assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.025) + # it should be even more similar outside edges zones + assert np.allclose(out_tf[instrument][4096:-4096,:], out_lib[instrument][4096:-4096,:], atol=0.002) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From f98f16ed70f6a9c3a0551fb08e2242c2081987c7 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 24 Jul 2020 15:39:43 +0200 Subject: [PATCH 51/57] rebase --- tests/test_eval.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index e9bf762..88c627a 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -99,14 +99,14 @@ def generate_fake_eval_dataset(path): aa.save(filename, data, fs) -def test_evaluate(path="FAKE_MUSDB_DIR"): - generate_fake_eval_dataset(path) - p = create_argument_parser() - arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path]) - params = load_configuration(arguments.configuration) - metrics = evaluate.entrypoint(arguments, params) - for instrument, metric in metrics.items(): - print(instrument), print(metric) - for m, value in metric.items(): - print(np.median(value)), print(res_4stems[instrument][m]) - assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) +@pytest.mark.parametrize('backend', TEST_CONFIGURATIONS) +def test_evaluate(backend): + with TemporaryDirectory() as directory: + generate_fake_eval_dataset(directory) + p = create_argument_parser() + arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", directory, "-B", backend]) + params = load_configuration(arguments.configuration) + metrics = evaluate.entrypoint(arguments, params) + for instrument, metric in metrics.items(): + for m, value in metric.items(): + assert np.allclose(np.median(value), res_4stems[backend][instrument][m], atol=1e-3) From e7cd9f8aaa1c5966656454474b2b4b10bec55184 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 24 Jul 2020 15:42:02 +0200 Subject: [PATCH 52/57] rechange test values --- tests/test_eval.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index 88c627a..547d1e5 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -30,29 +30,29 @@ TEST_CONFIGURATIONS = {el:el for el in BACKENDS} res_4stems = { "librosa": { - "vocals": { - "SDR": -0.007, - "SAR": -19.231, - "SIR": -4.528, + "vocals": { + "SDR": 0.000, + "SAR": -16.212, + "SIR": -4.172, "ISR": 0.000 }, "drums": { - "SDR": -0.071, - "SAR": -14.496, - "SIR": -4.987, + "SDR": -0.077, + "SAR": -15.739, + "SIR": -5.045, "ISR": 0.001 }, "bass":{ - "SDR": -0.001, - "SAR": -12.426, - "SIR": -7.198, - "ISR": -0.001 + "SDR": -0.000, + "SAR": -10.665, + "SIR": -5.646, + "ISR": -0.000 }, "other":{ - "SDR": -1.453, - "SAR": -14.899, - "SIR": -4.678, - "ISR": -0.015 + "SDR": -1.309, + "SAR": -14.573, + "SIR": -4.705, + "ISR": -0.014 } }, "tensorflow": { From 0016e3519274ec2dc766c2eac40539d9a44c0185 Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 24 Jul 2020 16:32:32 +0200 Subject: [PATCH 53/57] align the padding in librosa to what is now done in tf backend --- spleeter/separator.py | 8 ++--- tests/test_eval.py | 74 +++++++++++++---------------------------- tests/test_separator.py | 18 +--------- 3 files changed, 27 insertions(+), 73 deletions(-) diff --git a/spleeter/separator.py b/spleeter/separator.py index 9318911..2c636b4 100644 --- a/spleeter/separator.py +++ b/spleeter/separator.py @@ -123,7 +123,7 @@ class Separator(object): data = np.asfortranarray(data) N = self._params["frame_length"] H = self._params["frame_step"] - F = int(N/2) + 1 + win = hann(N, sym=False) fstft = istft if inverse else stft win_len_arg = {"win_length": None, @@ -131,12 +131,10 @@ class Separator(object): n_channels = data.shape[-1] out = [] for c in range(n_channels): - d = np.concatenate((np.zeros((F, 1)), data[:, :, c].T, np.zeros( - (F, 1))), axis=1) if inverse else data[:, c] + d = np.concatenate((np.zeros((N, )), data[:, c], np.zeros((N, )))) if not inverse else data[:, :, c].T s = fstft(d, hop_length=H, window=win, center=False, **win_len_arg) if inverse: - s = s[H:] - s = s[:length] + s = s[N:N+length] s = np.expand_dims(s.T, 2-inverse) out.append(s) if len(out) == 1: diff --git a/tests/test_eval.py b/tests/test_eval.py index 547d1e5..97540a9 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -29,57 +29,29 @@ BACKENDS = ["tensorflow", "librosa"] TEST_CONFIGURATIONS = {el:el for el in BACKENDS} res_4stems = { - "librosa": { - "vocals": { - "SDR": 0.000, - "SAR": -16.212, - "SIR": -4.172, - "ISR": 0.000 - }, - "drums": { - "SDR": -0.077, - "SAR": -15.739, - "SIR": -5.045, - "ISR": 0.001 - }, - "bass":{ - "SDR": -0.000, - "SAR": -10.665, - "SIR": -5.646, - "ISR": -0.000 - }, - "other":{ - "SDR": -1.309, - "SAR": -14.573, - "SIR": -4.705, - "ISR": -0.014 - } + "vocals": { + "SDR": 3.25e-05, + "SAR": -11.153575, + "SIR": -1.3849, + "ISR": 2.75e-05 }, - "tensorflow": { - "vocals": { - "SDR": 3.25e-05, - "SAR": -11.153575, - "SIR": -1.3849, - "ISR": 2.75e-05 - }, - "drums": { - "SDR": -0.079505, - "SAR": -15.7073575, - "SIR": -4.972755, - "ISR": 0.0013575 - }, - "bass":{ - "SDR": 2.5e-06, - "SAR": -10.3520575, - "SIR": -4.272325, - "ISR": 2.5e-06 - }, - "other":{ - "SDR": -1.359175, - "SAR": -14.7076775, - "SIR": -4.761505, - "ISR": -0.01528 - } + "drums": { + "SDR": -0.079505, + "SAR": -15.7073575, + "SIR": -4.972755, + "ISR": 0.0013575 + }, + "bass":{ + "SDR": 2.5e-06, + "SAR": -10.3520575, + "SIR": -4.272325, + "ISR": 2.5e-06 + }, + "other":{ + "SDR": -1.359175, + "SAR": -14.7076775, + "SIR": -4.761505, + "ISR": -0.01528 } } @@ -109,4 +81,4 @@ def test_evaluate(backend): metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): for m, value in metric.items(): - assert np.allclose(np.median(value), res_4stems[backend][instrument][m], atol=1e-3) + assert np.allclose(np.median(value), res_4stems[instrument][m], atol=1e-3) diff --git a/tests/test_separator.py b/tests/test_separator.py index d850e97..3094900 100644 --- a/tests/test_separator.py +++ b/tests/test_separator.py @@ -53,29 +53,13 @@ def test_separator_backends(test_file): stft_matrix, inverse=True, length=waveform.shape[0]) assert np.allclose(reconstructed, waveform, atol=3e-2) - # # now also test that tensorflow and librosa STFT provide same results - from spleeter.audio.spectrogram import compute_spectrogram_tf - tf_waveform = tf.convert_to_tensor(waveform, tf.float32) - spectrogram_tf = compute_spectrogram_tf(tf_waveform, - separator_tf._params['frame_length'], - separator_tf._params['frame_step'],) - with tf.Session() as sess: - spectrogram_tf_eval = spectrogram_tf.eval() - - # check that stfts are equivalent - assert stft_matrix.shape == spectrogram_tf_eval.shape - assert np.allclose( - np.abs(stft_matrix), spectrogram_tf_eval, atol=1e-2) - # compare both separation, it should be close out_tf = separator_tf._separate_tensorflow(waveform, test_file) out_lib = separator_lib._separate_librosa(waveform, test_file) for instrument in out_lib.keys(): # test that both outputs are close everywhere - assert np.allclose(out_tf[instrument], out_lib[instrument], atol=0.025) - # it should be even more similar outside edges zones - assert np.allclose(out_tf[instrument][4096:-4096,:], out_lib[instrument][4096:-4096,:], atol=0.002) + assert np.allclose(out_tf[instrument], out_lib[instrument], atol=1e-5) @pytest.mark.parametrize('test_file, configuration, backend', TEST_CONFIGURATIONS) From c6cc51006975da0a3e4e0fe2e67c40b3cfdc57bf Mon Sep 17 00:00:00 2001 From: mmoussallam Date: Fri, 24 Jul 2020 17:04:34 +0200 Subject: [PATCH 54/57] update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9df7a2..a5f4162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog History +## 1.5.4 + +First release, July 24th 2020 + +Add some padding of the input waveform to avoid separation artefacts on the edges due to unstabilities in the inverse fourier transforms. +Also add tests to ensure both librosa and tensorflow backends have same outputs. + ## 1.5.2 First released, May 15th 2020 From b9d96e1697c178c6035a45c389e3c1b36071c5dd Mon Sep 17 00:00:00 2001 From: romi1502 Date: Fri, 4 Sep 2020 15:30:19 +0200 Subject: [PATCH 55/57] Removed mention to requirements.txt that no longer exists --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 605cf2d..89120cd 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ The following set of commands will clone this repository, create a virtual envir ```bash git clone https://github.com/Deezer/spleeter && cd spleeter python -m venv spleeterenv && source spleeterenv/bin/activate -pip install -r requirements.txt && pip install pytest pytest-xdist +pip install . && pip install pytest pytest-xdist make test ``` From d2283a1333a495f71fb44591af11fb71c1551f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 18 Sep 2020 16:32:40 +0200 Subject: [PATCH 56/57] =?UTF-8?q?=E2=9C=A8=20import=20cuda-10-1=20image?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/cuda-10-1.dockerfile | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docker/cuda-10-1.dockerfile diff --git a/docker/cuda-10-1.dockerfile b/docker/cuda-10-1.dockerfile new file mode 100644 index 0000000..19909fa --- /dev/null +++ b/docker/cuda-10-1.dockerfile @@ -0,0 +1,45 @@ +ARG BASE=python:3.8 +FROM ${BASE} + +ENV CUDA_VERSION 10.1.243 +ENV CUDA_PKG_VERSION 10-1=$CUDA_VERSION-1 +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411 brand=tesla,driver>=418,driver<419" +ENV CUDNN_VERSION 7.6.5.32 +ENV NCCL_VERSION 2.7.8 + +LABEL com.nvidia.cuda.version="${CUDA_VERSION}" +LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}" +LABEL com.nvidia.volumes.needed="nvidia_driver" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + gnupg2 \ + curl \ + ca-certificates \ + && curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - \ + && echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/cuda.list \ + && echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list \ + && apt-get purge --autoremove -y curl \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + cuda-cudart-$CUDA_PKG_VERSION \ + cuda-compat-10-1 \ + && ln -s cuda-10.1 /usr/local/cuda \ + && echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \ + && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf \ + && apt-get install -y --no-install-recommends \ + cuda-libraries-$CUDA_PKG_VERSION \ + cuda-npp-$CUDA_PKG_VERSION \ + cuda-nvtx-$CUDA_PKG_VERSION \ + libcublas10=10.2.1.243-1 \ + libcudnn7=$CUDNN_VERSION-1+cuda10.1 \ + libnccl2=$NCCL_VERSION-1+cuda10.1 \ + && apt-mark hold libnccl2 \ + && apt-mark hold libcudnn7 \ + && apt-mark hold libcublas10 \ + && rm -rf /var/lib/apt/lists/* From e6aeaa117967409fdf05ba5bafe174ec62ea3f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 24 Sep 2020 12:35:09 +0200 Subject: [PATCH 57/57] =?UTF-8?q?=F0=9F=9A=80=20=20switch=20to=20deezer=20?= =?UTF-8?q?org?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c9363fd..3233489 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,45 +37,45 @@ jobs: echo "::set-env name=package::spleeter-gpu" # ---------------------------------------------------------------------- # Note: image building. - - name: Build researchdeezer/spleeter:${{ env.tag }} image + - name: Build deezer/spleeter:${{ env.tag }} image run: | docker build \ --build-arg BASE=${{ env.base }} \ --build-arg SPLEETER_PACKAGE=${{ env.package }} \ - -t researchdeezer/spleeter:${{ env.tag }} \ + -t deezer/spleeter:${{ env.tag }} \ -f docker/${{ env.file }}.dockerfile . echo "::set-env name=modelargs::" - if: ${{ matrix.model != 'modelless' }} - name: Build researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} image + name: Build deezer/spleeter:${{ env.tag }}-${{ matrix.model }} image run: | docker build \ - --build-arg BASE=researchdeezer/spleeter:${{ env.tag }} \ + --build-arg BASE=deezer/spleeter:${{ env.tag }} \ --build-arg MODEL=${{ matrix.model }} \ - -t researchdeezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ + -t deezer/spleeter:${{ env.tag }}-${{ matrix.model }} \ -f docker/spleeter-model.dockerfile . echo "::set-env name=tag::${{ env.tag }}-${{ matrix.model }}" echo "::set-env name=modelarg::-p spleeter:${{ matrix.model }}" # ---------------------------------------------------------------------- # Note: image testing. - - name: Test researchdeezer/spleeter:${{ env.tag }} image + - name: Test deezer/spleeter:${{ env.tag }} image run: | docker run \ -v $(pwd):/runtime \ - researchdeezer/spleeter:${{ env.tag }} \ + deezer/spleeter:${{ env.tag }} \ separate -i /runtime/audio_example.mp3 -o /tmp \${{ env.modelarg }} # ---------------------------------------------------------------------- # Note: image deploy. - name: Docker login run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - - name: Push researchdeezer/spleeter:${{ env.tag }} image - run: docker push researchdeezer/spleeter:${{ env.tag }} + - name: Push deezer/spleeter:${{ env.tag }} image + run: docker push deezer/spleeter:${{ env.tag }} - if: ${{ env.tag == 'spleeter:3.7' }} - name: Push researchdeezer/spleeter:latest image + name: Push deezer/spleeter:latest image run: | - docker tag researchdeezer/spleeter:3.7 researchdeezer/spleeter:latest - docker push researchdeezer/spleeter:latest + docker tag deezer/spleeter:3.7 deezer/spleeter:latest + docker push deezer/spleeter:latest - if: ${{ env.tag == 'spleeter:3.7-gpu' }} - name: Push researchdeezer/spleeter:gpu image + name: Push deezer/spleeter:gpu image run: | - docker tag researchdeezer/spleeter:3.7-gpu researchdeezer/spleeter:gpu - docker push researchdeezer/spleeter:gpu + docker tag deezer/spleeter:3.7-gpu deezer/spleeter:gpu + docker push deezer/spleeter:gpu