From e22c64a270aa03676377a28b4a882573431c315b Mon Sep 17 00:00:00 2001 From: romi1502 Date: Fri, 5 Jun 2020 12:27:06 +0200 Subject: [PATCH 1/4] Add test for evaluation --- .gitignore | 3 +- spleeter/commands/evaluate.py | 2 + tests/test_eval.py | 79 +++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 tests/test_eval.py diff --git a/.gitignore b/.gitignore index 3660fc0..a8ba96b 100644 --- a/.gitignore +++ b/.gitignore @@ -110,4 +110,5 @@ __pycache__ pretrained_models docs/build .vscode -spleeter-feedstock/ \ No newline at end of file +spleeter-feedstock/ +*FAKE_MUSDB_DIR \ No newline at end of file diff --git a/spleeter/commands/evaluate.py b/spleeter/commands/evaluate.py index 9de330c..1e1e3f1 100644 --- a/spleeter/commands/evaluate.py +++ b/spleeter/commands/evaluate.py @@ -163,3 +163,5 @@ def entrypoint(arguments, params): get_logger().info('%s:', instrument) for metric, value in metric.items(): get_logger().info('%s: %s', metric, f'{np.median(value):.3f}') + + return metrics diff --git a/tests/test_eval.py b/tests/test_eval.py new file mode 100644 index 0000000..257f75f --- /dev/null +++ b/tests/test_eval.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# coding: utf8 + +""" Unit testing for Separator class. """ + +__email__ = 'research@deezer.com' +__author__ = 'Deezer Research' +__license__ = 'MIT License' + +import filecmp +import itertools +from os import makedirs +from os.path import splitext, basename, exists, join +from tempfile import TemporaryDirectory + +import pytest +import numpy as np + +import tensorflow as tf + +from spleeter.audio.adapter import get_default_audio_adapter +from spleeter.commands import create_argument_parser + +from spleeter.commands import evaluate + +from spleeter.utils.configuration import load_configuration + +res_4stems = { "vocals": { + "SDR": -0.009, + "SAR": -19.044, + "SIR": -4.072, + "ISR": -0.000 + }, + "drums": { + "SDR": -0.066, + "SAR": -14.294, + "SIR": -4.908, + "ISR": 0.002 + }, + "bass":{ + "SDR": -0.000, + "SAR": -6.364, + "SIR": -9.396, + "ISR": -0.001 + }, + "other":{ + "SDR": -1.464, + "SAR": -14.893, + "SIR": -4.762, + "ISR": -0.027 + } + } + + +def generate_fake_eval_dataset(path): + aa = get_default_audio_adapter() + n_songs = 2 + fs = 44100 + duration = 3 + n_channels = 2 + for song in range(n_songs): + song_path = join(path, "test", f"song{song}") + makedirs(song_path, exist_ok=True) + rng = np.random.RandomState(seed=0) + for instr in ["mixture", "vocals", "bass", "drums", "other"]: + filename = join(song_path, f"{instr}.wav") + data = rng.rand(duration*fs, n_channels)-0.5 + aa.save(filename, data, fs) + + +def test_evaluate(path="FAKE_MUSDB_DIR"): + generate_fake_eval_dataset(path) + p = create_argument_parser() + arguments = p.parse_args(["evaluate", "-p", "spleeter:4stems", "--mus_dir", path]) + params = load_configuration(arguments.configuration) + metrics = evaluate.entrypoint(arguments, params) + for instrument, metric in metrics.items(): + for metric, value in metric.items(): + assert np.allclose(np.median(value), res_4stems[instrument][metric], atol=1e-3) From ff3ac620dca86f1a435ac1c2b9f38d900d5eefb2 Mon Sep 17 00:00:00 2001 From: romi1502 Date: Fri, 5 Jun 2020 13:42:52 +0200 Subject: [PATCH 2/4] Fix fake data generation --- tests/test_eval.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/test_eval.py b/tests/test_eval.py index 257f75f..6a3634b 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -26,28 +26,28 @@ from spleeter.commands import evaluate from spleeter.utils.configuration import load_configuration res_4stems = { "vocals": { - "SDR": -0.009, - "SAR": -19.044, - "SIR": -4.072, - "ISR": -0.000 + "SDR": -0.007, + "SAR": -19.231, + "SIR": -4.528, + "ISR": 0.000 }, "drums": { - "SDR": -0.066, - "SAR": -14.294, - "SIR": -4.908, - "ISR": 0.002 + "SDR": -0.071, + "SAR": -14.496, + "SIR": -4.987, + "ISR": 0.001 }, "bass":{ - "SDR": -0.000, - "SAR": -6.364, - "SIR": -9.396, + "SDR": -0.001, + "SAR": -12.426, + "SIR": -7.198, "ISR": -0.001 }, "other":{ - "SDR": -1.464, - "SAR": -14.893, - "SIR": -4.762, - "ISR": -0.027 + "SDR": -1.453, + "SAR": -14.899, + "SIR": -4.678, + "ISR": -0.015 } } @@ -58,10 +58,10 @@ def generate_fake_eval_dataset(path): fs = 44100 duration = 3 n_channels = 2 + rng = np.random.RandomState(seed=0) for song in range(n_songs): song_path = join(path, "test", f"song{song}") makedirs(song_path, exist_ok=True) - rng = np.random.RandomState(seed=0) for instr in ["mixture", "vocals", "bass", "drums", "other"]: filename = join(song_path, f"{instr}.wav") data = rng.rand(duration*fs, n_channels)-0.5 @@ -76,4 +76,4 @@ def test_evaluate(path="FAKE_MUSDB_DIR"): metrics = evaluate.entrypoint(arguments, params) for instrument, metric in metrics.items(): for metric, value in metric.items(): - assert np.allclose(np.median(value), res_4stems[instrument][metric], atol=1e-3) + assert np.allclose(np.median(value), res_4stems[instrument][metric], atol=1e-3) \ No newline at end of file From 582cb347a8238935418615de43bbefca396aa9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 5 Jun 2020 14:55:59 +0200 Subject: [PATCH 3/4] fix: add musdb test dev for CI --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1d8826..5eea9a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: - restore_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest pytest-xdist + - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb - run: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} @@ -30,7 +30,7 @@ jobs: - restore_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest pytest-xdist + - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb - run: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} From 9dcf220241b832faf36df7834ccdc789d965ba17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Fri, 5 Jun 2020 14:56:30 +0200 Subject: [PATCH 4/4] fix: add museval dep for CI testing --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5eea9a9..5d4acc1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ jobs: - restore_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb + - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb museval - run: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} @@ -30,7 +30,7 @@ jobs: - restore_cache: key: models-{{ checksum "spleeter/model/__init__.py" }} - run: apt-get update && apt-get install -y ffmpeg - - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb + - run: pip install -r requirements.txt && pip install pytest pytest-xdist musdb museval - run: make test - save_cache: key: models-{{ checksum "spleeter/model/__init__.py" }}