yuzu/spleeter
1
0
Fork 0
mirror of https://github.com/YuzuZensai/spleeter.git synced 2026-01-06 04:32:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: docker image layout

Browse Source
This commit is contained in:
Félix Voituret
2020-06-11 11:53:35 +02:00
parent 05aa98a06a
commit 9f61fa8908
10 changed files with 117 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
.circleci/config.yml
View File

@@ -91,16 +91,18 @@ jobs:
# =======================================================================================
# Docker build.
# =======================================================================================
docker-conda-cpu:
docker:
- image: docker:17.05.0-ce-git
steps:
- checkout
- setup_remote_docker
- run: docker build -t researchdeezer/spleeter:conda -f docker/conda.dockerfile .
- run: docker build --build-arg BASETAG=conda --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-2stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=conda --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-4stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=conda --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-5stems -f docker/embedded-model.dockerfile .
- run: docker build -t conda -f docker/conda.dockerfile .
- run: docker build --build-arg BASE=conda -t researchdeezer/spleeter:conda -f docker/spleeter-conda.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-5stems -f docker/spleeter-model.dockerfile .
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-2stems separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:conda-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp
@@ -116,10 +118,13 @@ jobs:
steps:
- checkout
- setup_remote_docker
- run: docker build -t researchdeezer/spleeter:conda-gpu -f docker/conda-gpu.dockerfile .
- run: docker build --build-arg BASETAG=conda-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-gpu-2stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=conda-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-gpu-4stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=conda-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-gpu-5stems -f docker/embedded-model.dockerfile .
- run: docker build -t conda -f docker/conda.dockerfile .
- run: docker build --build-arg BASE=conda -t cuda:conda -f docker/cuda-10-0.dockerfile .
- run: docker build --build-arg BASE=cuda:conda -t conda-gpu -f docker/conda.dockerfile .
- run: docker build --build-arg BASE=conda-gpu -t researchdeezer/spleeter:conda-gpu -f docker/spleeter-conda.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:conda-gpu-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:conda-gpu-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:conda-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:conda-gpu-5stems -f docker/spleeter-model.dockerfile .
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:conda-gpu
- run: docker push researchdeezer/spleeter:conda-gpu-2stems
@@ -133,10 +138,10 @@ jobs:
steps:
- checkout
- setup_remote_docker
- run: docker build -t researchdeezer/spleeter:3.6 -f docker/python-3.6.dockerfile .
- run: docker build --build-arg BASETAG=3.6 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-2stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.6 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-4stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.6 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-5stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASE=python:3.6 -t researchdeezer/spleeter:3.6 -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-5stems -f docker/spleeter-model.dockerfile .
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6 separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-2stems separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.6-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp
@@ -152,10 +157,11 @@ jobs:
steps:
- checkout
- setup_remote_docker
- run: docker build -t researchdeezer/spleeter:3.6-gpu -f docker/python-3.6-gpu.dockerfile .
- run: docker build --build-arg BASETAG=3.6-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-gpu-2stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.6-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-gpu-4stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.6-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-gpu-5stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASE=python:3.6 -t cuda:3.6 -f docker/cuda-10-0.dockerfile .
- run: docker build --build-arg BASE=cuda:3.6 -t researchdeezer/spleeter:3.6-gpu -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-gpu-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-gpu-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.6-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-gpu-5stems -f docker/spleeter-model.dockerfile .
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:3.6-gpu
- run: docker push researchdeezer/spleeter:3.6-gpu-2stems
@@ -167,10 +173,10 @@ jobs:
steps:
- checkout
- setup_remote_docker
- run: docker build -t researchdeezer/spleeter:3.7 -f docker/python-3.7.dockerfile .
- run: docker build --build-arg BASETAG=3.7 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-2stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.7 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-4stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.7 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-5stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASE=python:3.7 -t researchdeezer/spleeter:3.7 -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7 --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-5stems -f docker/spleeter-model.dockerfile .
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7 separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-2stems separate -i /runtime/audio_example.mp3 -o /tmp
- run: docker run -v $(pwd):/runtime researchdeezer/spleeter:3.7-4stems separate -i /runtime/audio_example.mp3 -p spleeter:4stems -o /tmp
@@ -188,15 +194,17 @@ jobs:
steps:
- checkout
- setup_remote_docker
- run: docker build -t researchdeezer/spleeter:3.7-gpu -f docker/python-3.7-gpu.dockerfile .
- run: docker build --build-arg BASETAG=3.7-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-gpu-2stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.7-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-gpu-4stems -f docker/embedded-model.dockerfile .
- run: docker build --build-arg BASETAG=3.7-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-gpu-5stems -f docker/embedded-model.dockerfile .
- run: docker build -t cuda:3.7 -f docker/cuda-10-0.dockerfile .
- run: docker build --build-arg BASE=cuda:3.7 -t researchdeezer/spleeter:3.7-gpu -f docker/spleeter.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-gpu-2stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-gpu-4stems -f docker/spleeter-model.dockerfile .
- run: docker build --build-arg BASE=researchdeezer/spleeter:3.7-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-gpu-5stems -f docker/spleeter-model.dockerfile .
- run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
- run: docker push researchdeezer/spleeter:3.7-gpu
- run: docker push researchdeezer/spleeter:3.7-gpu-2stems
- run: docker push researchdeezer/spleeter:3.7-gpu-4stems
- run: docker push researchdeezer/spleeter:3.7-gpu-5stems
workflows:
version: 2
spleeter-release-pipeline:

23
docker/conda-gpu.dockerfile
View File

@@ -1,23 +0,0 @@
FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
RUN apt-get update --fix-missing \
&& apt-get install -y wget bzip2 ca-certificates curl git \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh \
&& /bin/bash ~/miniconda.sh -b -p /opt/conda \
&& rm ~/miniconda.sh \
&& /opt/conda/bin/conda clean -tipsy \
&& ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc \
&& echo "conda activate base" >> ~/.bashrc \
&& ln -s /opt/conda/bin/conda /usr/bin/conda
COPY audio_example.mp3 .
RUN conda install -y -c conda-forge musdb
# RUN conda install -y -c conda-forge museval
RUN conda install -y -c conda-forge spleeter-gpu=1.4.9
ENTRYPOINT ["spleeter"]

24
docker/conda.dockerfile
View File

@@ -1,11 +1,15 @@
FROM continuumio/miniconda3:4.7.10
ARG BASE=python:3.7
FROM ${BASE}
RUN mkdir -p /model
ENV MODEL_PATH /model
COPY audio_example.mp3 .
RUN conda install -y -c conda-forge musdb
# RUN conda install -y -c conda-forge museval
RUN conda install -y -c conda-forge spleeter=1.4.9
ENTRYPOINT ["spleeter"]
RUN apt-get update --fix-missing \
&& apt-get install -y wget bzip2 ca-certificates curl git \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh -O ~/miniconda.sh \
&& /bin/bash ~/miniconda.sh -b -p /opt/conda \
&& rm ~/miniconda.sh \
&& /opt/conda/bin/conda clean -tipsy \
&& ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh \
&& echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc \
&& echo "conda activate base" >> ~/.bashrc \
&& ln -s /opt/conda/bin/conda /usr/bin/conda

43
docker/cuda-10-0.dockerfile Normal file
View File

@@ -0,0 +1,43 @@
ARG BASE=python:3.7
FROM ${BASE}
ENV CUDA_VERSION 10.0.130
ENV CUDA_PKG_VERSION 10-0=$CUDA_VERSION-1
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
# Note: differ from original image, picked from old dzr_audio spec.
ENV NVIDIA_VISIBLE_DEVICES=3
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=10.0 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=410,driver<411"
ENV NCCL_VERSION 2.4.2
ENV CUDNN_VERSION 7.6.0.64
LABEL com.nvidia.cuda.version="${CUDA_VERSION}"
LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"
LABEL com.nvidia.volumes.needed="nvidia_driver"
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
gnupg2 \
curl \
ca-certificates \
&& curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - \
&& echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/cuda.list \
&& echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list \
&& apt-get purge --autoremove -y curl \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
cuda-cudart-$CUDA_PKG_VERSION \
cuda-compat-10-0 \
&& ln -s cuda-10.0 /usr/local/cuda \
&& echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
&& echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf \
&& apt-get install -y --no-install-recommends \
cuda-toolkit-10-0 \
cuda-libraries-$CUDA_PKG_VERSION \
cuda-nvtx-$CUDA_PKG_VERSION \
libnccl2=$NCCL_VERSION-1+cuda10.0 \
libcudnn7=$CUDNN_VERSION-1+cuda10.0 \
&& apt-mark hold libnccl2 \
&& apt-mark hold libcudnn7 \
&& rm -rf /var/lib/apt/lists/*

3
docker/python-3.7-gpu.dockerfile → docker/cuda-9.2.dockerfile
View File

@@ -1,4 +1,5 @@
FROM python:3.7
ARG BASE=python:3.7
FROM ${BASE}
# FROM 9.2-base-ubuntu18.04
# https://gitlab.com/nvidia/container-images/cuda/blob/ubuntu18.04/9.2/base/Dockerfile

57
docker/python-3.6-gpu.dockerfile
View File

@@ -1,57 +0,0 @@
FROM python:3.6
# FROM 9.2-base-ubuntu18.04
# https://gitlab.com/nvidia/container-images/cuda/blob/ubuntu18.04/9.2/base/Dockerfile
RUN apt-get update && apt-get install -y --no-install-recommends gnupg2 curl ca-certificates \
&& curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1710/x86_64/7fa2af80.pub | apt-key add - \
&& echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1710/x86_64 /" > /etc/apt/sources.list.d/cuda.list \
&& echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list \
&& apt-get purge --autoremove -y curl \
&& rm -rf /var/lib/apt/lists/*
ENV CUDA_VERSION 9.2.148
ENV CUDA_PKG_VERSION 9-2=$CUDA_VERSION-1
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cuda-cudart-$CUDA_PKG_VERSION \
&& ln -s cuda-9.2 /usr/local/cuda \
&& rm -rf /var/lib/apt/lists/*
LABEL com.nvidia.volumes.needed="nvidia_driver"
LABEL com.nvidia.cuda.version="${CUDA_VERSION}"
RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \
&& echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=9.2"
# FROM 9.2-runtime-ubuntu18.04
# https://gitlab.com/nvidia/container-images/cuda/blob/ubuntu18.04/9.2/runtime/Dockerfile
ENV NCCL_VERSION 2.3.7
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
cuda-libraries-$CUDA_PKG_VERSION \
cuda-nvtx-$CUDA_PKG_VERSION \
libnccl2=$NCCL_VERSION-1+cuda9.2 \
&& apt-mark hold libnccl2 \
&& rm -rf /var/lib/apt/lists/*
# FROM 9.2-runtime-cudnn7-ubuntu18.04
# https://gitlab.com/nvidia/container-images/cuda/blob/ubuntu18.04/9.2/runtime/cudnn7/Dockerfile
ENV CUDNN_VERSION 7.5.0.56
LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"
RUN apt-get update \
&& apt-get install -y --no-install-recommends libcudnn7=$CUDNN_VERSION-1+cuda9.2 \
&& apt-mark hold libcudnn7 \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir -p /model
ENV MODEL_PATH /model
COPY audio_example.mp3 .
# Spleeter installation.
RUN apt-get update && apt-get install -y ffmpeg libsndfile1
RUN pip install musdb museval
RUN pip install spleeter-gpu==1.4.9
ENTRYPOINT ["spleeter"]

11
docker/python-3.6.dockerfile
View File

@@ -1,11 +0,0 @@
FROM python:3.6
RUN mkdir -p /model
ENV MODEL_PATH /model
COPY audio_example.mp3 .
RUN apt-get update && apt-get install -y ffmpeg libsndfile1
RUN pip install musdb museval
RUN pip install spleeter==1.4.9
ENTRYPOINT ["spleeter"]

15
docker/spleeter-conda.dockerfile Normal file
View File

@@ -0,0 +1,15 @@
ARG BASE=conda
ARG SPLEETER_PACKAGE=spleeter
ARG SPLEETER_VERSION=1.4.9
FROM ${BASE}
ENV MODEL_PATH /model
RUN mkdir -p /model
COPY audio_example.mp3 .
RUN conda install -y -c conda-forge musdb
RUN conda install -y -c conda-forge ${SPLEETER_PACKAGE}==${SPLEETER_VERSION}
ENTRYPOINT ["spleeter"]

7
docker/embedded-model.dockerfile → docker/spleeter-model.dockerfile
View File

@@ -1,8 +1,7 @@
ARG BASETAG
ARG BASE=researchdeezer/spleeter
ARG MODEL=2-stems
FROM researchdeezer/spleeter:${BASETAG}
ARG MODEL
FROM ${BASE}
RUN mkdir -p /model/$MODEL \
&& wget -O /tmp/$MODEL.tar.gz https://github.com/deezer/spleeter/releases/download/v1.4.0/$MODEL.tar.gz \

11
docker/python-3.7.dockerfile → docker/spleeter.dockerfile
View File

@@ -1,11 +1,16 @@
FROM python:3.7
ARG BASE=python:3.6
ARG SPLEETER_PACKAGE=spleeter
ARG SPLEETER_VERSION=1.4.9
FROM ${BASE}
ENV MODEL_PATH /model
RUN mkdir -p /model
ENV MODEL_PATH /model
COPY audio_example.mp3 .
RUN apt-get update && apt-get install -y ffmpeg libsndfile1
RUN pip install musdb museval
RUN pip install spleeter==1.4.9
RUN pip install ${SPLEETER_PACKAGE}==${SPLEETER_VERSION}
ENTRYPOINT ["spleeter"]