From b62e4558332567de0753cfed827d5eee7fd7e30e Mon Sep 17 00:00:00 2001 From: Steve Potter Date: Mon, 29 Apr 2024 16:45:30 -0400 Subject: [PATCH 1/2] Add docker-compose directory that includes alternative approach to using Docker that could replace the existing one --- .dockerignore | 158 +++++++++++++++++++++++++++++++++ docker-compose/compose.yaml | 34 +++++++ docker-compose/dockerfile | 79 +++++++++++++++++ docker-compose/dockerfile.prod | 77 ++++++++++++++++ docker-compose/readme.md | 9 ++ 5 files changed, 357 insertions(+) create mode 100644 .dockerignore create mode 100644 docker-compose/compose.yaml create mode 100644 docker-compose/dockerfile create mode 100644 docker-compose/dockerfile.prod create mode 100644 docker-compose/readme.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..cc59bdd93 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,158 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +.history +exp/ +*.py[cod] +*$py.class +# C extensions +*.so +BundleTrack/build +# Distribution / packaging +.Python +foundationpose/ +build/ +develop-eggs/ +*.zip +demo_data/* +dist/ +# *.png +# *.jpg +debug/ +weights/* +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +.vscode/ +wheels/ +wandb/ +*.blend1 +*.tar.* +*.zip +*.svg +sacred/ +artifacts/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +demo_data/ +debug/ +coverage.xml +*.cover +demo_data/ +debug/ +weights/ +*.py,cover +.hypothesis/ +.pytest_cache/ +*.pth +weights/ +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache +cosypose/experiments + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Datasets +data/ +logs/ +!BundleTrack/XMem/inference/data +run_used.yml \ No newline at end of file diff --git a/docker-compose/compose.yaml b/docker-compose/compose.yaml new file mode 100644 index 000000000..60a44b352 --- /dev/null +++ b/docker-compose/compose.yaml @@ -0,0 +1,34 @@ +name: foundationpose +services: + foundationpose: + build: + context: ../ + dockerfile: dockerfile + pull_policy: build + image: foundationpose:latest + container_name: foundationpose + command: bash + volumes: + - /tmp/.X11-unix:/tmp/.X11-unix + - /tmp:/tmp + - /mnt:/mnt + - /home:/home + - ../:/foundation-pose/ + environment: + - DISPLAY=${DISPLAY} + - NVIDIA_DISABLE_REQUIRE=1 + network_mode: host + ipc: host + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + tty: true + stdin_open: true + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] \ No newline at end of file diff --git a/docker-compose/dockerfile b/docker-compose/dockerfile new file mode 100644 index 000000000..64c434998 --- /dev/null +++ b/docker-compose/dockerfile @@ -0,0 +1,79 @@ +FROM nvidia/cudagl:11.3.0-devel-ubuntu20.04 + +ENV TZ=US/Pacific +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update --fix-missing && \ + apt-get install -y libgtk2.0-dev && \ + apt-get install -y wget bzip2 ca-certificates curl git vim tmux g++ gcc build-essential cmake checkinstall gfortran libjpeg8-dev libtiff5-dev pkg-config yasm libavcodec-dev libavformat-dev libswscale-dev libdc1394-22-dev libxine2-dev libv4l-dev qt5-default libgtk2.0-dev libtbb-dev libatlas-base-dev libfaac-dev libmp3lame-dev libtheora-dev libvorbis-dev libxvidcore-dev libopencore-amrnb-dev libopencore-amrwb-dev x264 v4l-utils libprotobuf-dev protobuf-compiler libgoogle-glog-dev libgflags-dev libgphoto2-dev libhdf5-dev doxygen libflann-dev libboost-all-dev proj-data libproj-dev libyaml-cpp-dev cmake-curses-gui libzmq3-dev freeglut3-dev + + +RUN cd / && git clone https://github.com/pybind/pybind11 &&\ + cd pybind11 && git checkout v2.10.0 &&\ + mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DPYBIND11_INSTALL=ON -DPYBIND11_TEST=OFF &&\ + make -j6 && make install + + +RUN cd / && wget https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz &&\ + tar xvzf ./eigen-3.4.0.tar.gz &&\ + cd eigen-3.4.0 &&\ + mkdir build &&\ + cd build &&\ + cmake .. &&\ + make install + +SHELL ["/bin/bash", "--login", "-c"] + +RUN cd / && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /miniconda.sh && \ + /bin/bash /miniconda.sh -b -p /opt/conda &&\ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh &&\ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc &&\ + /bin/bash -c "source ~/.bashrc" && \ + /opt/conda/bin/conda update -n base -c defaults conda -y &&\ + /opt/conda/bin/conda create -n my python=3.8 + + +ENV PATH $PATH:/opt/conda/envs/my/bin + +RUN conda init bash &&\ + echo "conda activate my" >> ~/.bashrc &&\ + conda activate my &&\ + pip install torch==2.0.0+cu118 torchvision==0.15.1+cu118 torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cu118 &&\ + pip install "git+https://github.com/facebookresearch/pytorch3d.git@stable" &&\ + pip install scipy joblib scikit-learn ruamel.yaml trimesh pyyaml opencv-python imageio open3d transformations warp-lang einops kornia pyrender + +RUN cd / && git clone --recursive https://github.com/NVIDIAGameWorks/kaolin +RUN conda activate my && cd /kaolin &&\ + FORCE_CUDA=1 python setup.py develop + +RUN cd / && git clone https://github.com/NVlabs/nvdiffrast &&\ + conda activate my && cd /nvdiffrast && pip install . + +ENV OPENCV_IO_ENABLE_OPENEXR=1 + +RUN conda activate my &&\ + pip install scikit-image meshcat webdataset omegaconf pypng roma seaborn opencv-contrib-python openpyxl wandb imgaug Ninja xlsxwriter timm albumentations xatlas rtree nodejs jupyterlab objaverse g4f ultralytics==8.0.120 pycocotools videoio numba &&\ + conda install -y -c anaconda h5py + +# Note: /foundationpose will only exist if using docker compose as it mounts the local directory to the container + +# Building C++ code in the mycpp directory +RUN cd /foundationpose/mycpp/ && \ + mkdir -p build && \ + cd build && \ + cmake .. && \ + make -j11 + +# Clean and install Python package in /kaolin +RUN cd /kaolin && \ + rm -rf build *egg* && \ + pip install -e . + +# Clean and install Python package in mycuda directory of /foundationpose/bundlesdf +RUN cd /foundationpose/bundlesdf/mycuda && \ + rm -rf build *egg* && \ + pip install -e . + +# force system to use bash +ENV SHELL=/bin/bash +RUN ln -sf /bin/bash /bin/sh diff --git a/docker-compose/dockerfile.prod b/docker-compose/dockerfile.prod new file mode 100644 index 000000000..7d5589351 --- /dev/null +++ b/docker-compose/dockerfile.prod @@ -0,0 +1,77 @@ +FROM nvidia/cudagl:11.3.0-devel-ubuntu20.04 + +ENV TZ=US/Pacific +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update --fix-missing && \ + apt-get install -y libgtk2.0-dev && \ + apt-get install -y wget bzip2 ca-certificates curl git vim tmux g++ gcc build-essential cmake checkinstall gfortran libjpeg8-dev libtiff5-dev pkg-config yasm libavcodec-dev libavformat-dev libswscale-dev libdc1394-22-dev libxine2-dev libv4l-dev qt5-default libgtk2.0-dev libtbb-dev libatlas-base-dev libfaac-dev libmp3lame-dev libtheora-dev libvorbis-dev libxvidcore-dev libopencore-amrnb-dev libopencore-amrwb-dev x264 v4l-utils libprotobuf-dev protobuf-compiler libgoogle-glog-dev libgflags-dev libgphoto2-dev libhdf5-dev doxygen libflann-dev libboost-all-dev proj-data libproj-dev libyaml-cpp-dev cmake-curses-gui libzmq3-dev freeglut3-dev + + +RUN cd / && git clone https://github.com/pybind/pybind11 &&\ + cd pybind11 && git checkout v2.10.0 &&\ + mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DPYBIND11_INSTALL=ON -DPYBIND11_TEST=OFF &&\ + make -j6 && make install + + +RUN cd / && wget https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz &&\ + tar xvzf ./eigen-3.4.0.tar.gz &&\ + cd eigen-3.4.0 &&\ + mkdir build &&\ + cd build &&\ + cmake .. &&\ + make install + +SHELL ["/bin/bash", "--login", "-c"] + +RUN cd / && wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /miniconda.sh && \ + /bin/bash /miniconda.sh -b -p /opt/conda &&\ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh &&\ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc &&\ + /bin/bash -c "source ~/.bashrc" && \ + /opt/conda/bin/conda update -n base -c defaults conda -y &&\ + /opt/conda/bin/conda create -n my python=3.8 + + +ENV PATH $PATH:/opt/conda/envs/my/bin + +RUN conda init bash &&\ + echo "conda activate my" >> ~/.bashrc &&\ + conda activate my &&\ + pip install torch==2.0.0+cu118 torchvision==0.15.1+cu118 torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cu118 &&\ + pip install "git+https://github.com/facebookresearch/pytorch3d.git@stable" &&\ + pip install scipy joblib scikit-learn ruamel.yaml trimesh pyyaml opencv-python imageio open3d transformations warp-lang einops kornia pyrender + +RUN cd / && git clone --recursive https://github.com/NVIDIAGameWorks/kaolin +RUN conda activate my && cd /kaolin &&\ + FORCE_CUDA=1 python setup.py develop + +RUN cd / && git clone https://github.com/NVlabs/nvdiffrast &&\ + conda activate my && cd /nvdiffrast && pip install . + +ENV OPENCV_IO_ENABLE_OPENEXR=1 + +RUN conda activate my &&\ + pip install scikit-image meshcat webdataset omegaconf pypng roma seaborn opencv-contrib-python openpyxl wandb imgaug Ninja xlsxwriter timm albumentations xatlas rtree nodejs jupyterlab objaverse g4f ultralytics==8.0.120 pycocotools videoio numba &&\ + conda install -y -c anaconda h5py + + +COPY . /foundationpose + +# Building C++ code in the mycpp directory +RUN cd /foundationpose/mycpp/ && \ + mkdir -p build && \ + cd build && \ + cmake .. && \ + make -j11 + +# Clean and install Python package in /kaolin +# todo: can this go above the copy? +RUN cd /kaolin && \ + rm -rf build *egg* && \ + pip install -e . + +# Clean and install Python package in mycuda directory of /foundationpose/bundlesdf +RUN cd /foundationpose/bundlesdf/mycuda && \ + rm -rf build *egg* && \ + pip install -e . diff --git a/docker-compose/readme.md b/docker-compose/readme.md new file mode 100644 index 000000000..c16680387 --- /dev/null +++ b/docker-compose/readme.md @@ -0,0 +1,9 @@ +# Alternative (new?) Docker Support + +This directory provides an experimental alternative to the current recommended approach. It uses Docker Compose for development. Instead of using `run_container.sh`, to start the container: + +``` +docker compose up +``` + +There is also a dockerfile.prod that can be used to build foundationpose containers that are ready to use and don't require executing build_all.sh \ No newline at end of file From 34c1f62d295a4cb0e8b179f73b91cbc2953e6a81 Mon Sep 17 00:00:00 2001 From: Steve Potter Date: Thu, 6 Jun 2024 18:07:51 +0000 Subject: [PATCH 2/2] Fix some path mistakes in compose --- docker-compose/compose.yaml | 8 ++++---- docker-compose/readme.md | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker-compose/compose.yaml b/docker-compose/compose.yaml index 60a44b352..b720a2db7 100644 --- a/docker-compose/compose.yaml +++ b/docker-compose/compose.yaml @@ -1,11 +1,11 @@ name: foundationpose services: foundationpose: + image: foundationpose:latest build: context: ../ - dockerfile: dockerfile + dockerfile: ./docker-compose/dockerfile pull_policy: build - image: foundationpose:latest container_name: foundationpose command: bash volumes: @@ -13,7 +13,7 @@ services: - /tmp:/tmp - /mnt:/mnt - /home:/home - - ../:/foundation-pose/ + - ../:/foundationpose/ environment: - DISPLAY=${DISPLAY} - NVIDIA_DISABLE_REQUIRE=1 @@ -31,4 +31,4 @@ services: devices: - driver: nvidia count: 1 - capabilities: [gpu] \ No newline at end of file + capabilities: [gpu] \ No newline at end of file diff --git a/docker-compose/readme.md b/docker-compose/readme.md index c16680387..11e0162f7 100644 --- a/docker-compose/readme.md +++ b/docker-compose/readme.md @@ -1,8 +1,9 @@ # Alternative (new?) Docker Support -This directory provides an experimental alternative to the current recommended approach. It uses Docker Compose for development. Instead of using `run_container.sh`, to start the container: +This directory provides an experimental alternative to the current recommended approach. It uses Docker Compose for development. Instead of using `run_container.sh` as before, to start the container: ``` +cd docker-compose docker compose up ```