From f7e97a1515dcbd1fc048636eb46a8975d5001ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Voituret?= Date: Thu, 21 Nov 2019 18:15:54 +0100 Subject: [PATCH] feat: add gpu python image --- .circleci/config.yml | 44 ++++++++++++++++++++++++++++++++ docker/python-3.6-gpu.dockerfile | 36 ++++++++++++++++++++++++-- docker/python-3.7-gpu.dockerfile | 41 +++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 docker/python-3.7-gpu.dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 6d8d2a9..2fc1e5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,6 +146,21 @@ jobs: - run: docker push researchdeezer/spleeter:3.6-2stems - run: docker push researchdeezer/spleeter:3.6-4stems - run: docker push researchdeezer/spleeter:3.6-5stems + docker-3.6-gpu: + docker: + - image: docker:17.05.0-ce-git + steps: + - checkout + - setup_remote_docker + - run: docker build -t researchdeezer/spleeter:3.6-gpu -f docker/python-3.6-gpu.dockerfile . + - run: docker build --build-arg BASETAG=3.6-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.6-gpu-2stems -f docker/embedded-model.dockerfile . + - run: docker build --build-arg BASETAG=3.6-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.6-gpu-4stems -f docker/embedded-model.dockerfile . + - run: docker build --build-arg BASETAG=3.6-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.6-gpu-5stems -f docker/embedded-model.dockerfile . + - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + - run: docker push researchdeezer/spleeter:3.6-gpu + - run: docker push researchdeezer/spleeter:3.6-gpu-2stems + - run: docker push researchdeezer/spleeter:3.6-gpu-4stems + - run: docker push researchdeezer/spleeter:3.6-gpu-5stems docker-3.7-cpu: docker: - image: docker:17.05.0-ce-git @@ -167,6 +182,21 @@ jobs: - run: docker push researchdeezer/spleeter:3.7-2stems - run: docker push researchdeezer/spleeter:3.7-4stems - run: docker push researchdeezer/spleeter:3.7-5stems + docker-3.7-gpu: + docker: + - image: docker:17.05.0-ce-git + steps: + - checkout + - setup_remote_docker + - run: docker build -t researchdeezer/spleeter:3.7-gpu -f docker/python-3.7-gpu.dockerfile . + - run: docker build --build-arg BASETAG=3.7-gpu --build-arg MODEL=2stems -t researchdeezer/spleeter:3.7-gpu-2stems -f docker/embedded-model.dockerfile . + - run: docker build --build-arg BASETAG=3.7-gpu --build-arg MODEL=4stems -t researchdeezer/spleeter:3.7-gpu-4stems -f docker/embedded-model.dockerfile . + - run: docker build --build-arg BASETAG=3.7-gpu --build-arg MODEL=5stems -t researchdeezer/spleeter:3.7-gpu-5stems -f docker/embedded-model.dockerfile . + - run: docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + - run: docker push researchdeezer/spleeter:3.7-gpu + - run: docker push researchdeezer/spleeter:3.7-gpu-2stems + - run: docker push researchdeezer/spleeter:3.7-gpu-4stems + - run: docker push researchdeezer/spleeter:3.7-gpu-5stems workflows: version: 2 spleeter-release-pipeline: @@ -225,9 +255,23 @@ workflows: branches: only: - master + - docker-3.6-gpu: + requires: + - pypi-deploy-gpu + filters: + branches: + only: + - master - docker-3.7-cpu: requires: - pypi-deploy + filters: + branches: + only: + - master + - docker-3.7-gpu: + requires: + - pypi-deploy-gpu filters: branches: only: diff --git a/docker/python-3.6-gpu.dockerfile b/docker/python-3.6-gpu.dockerfile index d42f3db..e66f378 100644 --- a/docker/python-3.6-gpu.dockerfile +++ b/docker/python-3.6-gpu.dockerfile @@ -1,6 +1,38 @@ -FROM nvidia/cuda:10.1-cudnn7-runtime-ubuntu18.04 +FROM python:3.6 -# TODO: Install cuda ! +# Cuda installation. +# Note : original cuda base file from https://gitlab.com/nvidia/container-images/cuda/blob/master/dist/ubuntu18.04/10.1/base/Dockerfile +RUN apt-get update \ + && apt-get install -y --no-install-recommends gnupg2 curl ca-certificates \ + && curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - \ + && echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/cuda.list \ + && echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list \ + && apt-get purge --autoremove -y curl \ + && rm -rf /var/lib/apt/lists/* + +ENV CUDA_VERSION 10.1.243 +ENV CUDA_PKG_VERSION 10-1=$CUDA_VERSION-1 + +# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a +RUN apt-get update \ + && apt-get install -y --no-install-recommends cuda-cudart-$CUDA_PKG_VERSION \ + && cuda-compat-10-1 \ + && ln -s cuda-10.1 /usr/local/cuda \ + && rm -rf /var/lib/apt/lists/* + +# Required for nvidia-docker v1 +RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \ + && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf + +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +# nvidia-container-runtime +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411" + +# Spleeter installation. RUN apt-get update && apt-get install -y ffmpeg libsndfile1 RUN pip install musdb museval RUN pip install spleeter-gpu==1.4.4 diff --git a/docker/python-3.7-gpu.dockerfile b/docker/python-3.7-gpu.dockerfile new file mode 100644 index 0000000..30150ad --- /dev/null +++ b/docker/python-3.7-gpu.dockerfile @@ -0,0 +1,41 @@ +FROM python:3.7 + +# Cuda installation. +# Note : original cuda base file from https://gitlab.com/nvidia/container-images/cuda/blob/master/dist/ubuntu18.04/10.1/base/Dockerfile +RUN apt-get update \ + && apt-get install -y --no-install-recommends gnupg2 curl ca-certificates \ + && curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - \ + && echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/cuda.list \ + && echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list \ + && apt-get purge --autoremove -y curl \ + && rm -rf /var/lib/apt/lists/* + +ENV CUDA_VERSION 10.1.243 +ENV CUDA_PKG_VERSION 10-1=$CUDA_VERSION-1 + +# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a +RUN apt-get update \ + && apt-get install -y --no-install-recommends cuda-cudart-$CUDA_PKG_VERSION \ + && cuda-compat-10-1 \ + && ln -s cuda-10.1 /usr/local/cuda \ + && rm -rf /var/lib/apt/lists/* + +# Required for nvidia-docker v1 +RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf \ + && echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf + +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} +ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64 + +# nvidia-container-runtime +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility +ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411" + +# Spleeter installation. +RUN apt-get update && apt-get install -y ffmpeg libsndfile1 +RUN pip install musdb museval +RUN pip install spleeter-gpu==1.4.4 +RUN mkdir -p /model +ENV MODEL_PATH /model +ENTRYPOINT ["spleeter"] \ No newline at end of file