diff --git a/.gitignore b/.gitignore index 09e33e055dd71dcd5eca6dfe9c9891c4e8a45fc5..1f80847c12e867561b882b4bd53c1e87b6554476 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ outputs/* *.pyc +tools/Kitti2Coco/labels2coco/* +tools/Kitti2Coco/trainvalno5k-kitti.txt +*.pkl +dino/ \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..046abd3f2f2393981153afb368e538a6ebb5c4b3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,161 @@ +FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +ARG CMAKE_VERSION=3.16 +ARG CMAKE_BUILD=5 +ARG PYTHON_VERSION=3.6 +ARG TORCH_VERSION=1.8.1 +ARG TORCHVISION_VERSION=0.9.1 + +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --recursive" && \ + + rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/cuda.list \ + /etc/apt/sources.list.d/nvidia-ml.list && \ + + apt-get update && \ + +# ================================================================== +# tools +# ------------------------------------------------------------------ + + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + build-essential \ + apt-utils \ + ca-certificates \ + wget \ + git \ + vim \ + libssl-dev \ + curl \ + unzip \ + unrar \ + ffmpeg \ + libsm6 \ + libxext6 \ + && \ + + wget -O ~/cmake.tar.gz \ + https://cmake.org/files/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.${CMAKE_BUILD}.tar.gz && \ + tar -C ~/ -xzf ~/cmake.tar.gz && \ + cd ~/cmake-${CMAKE_VERSION}.${CMAKE_BUILD} && \ + ./bootstrap && \ + make -j$(nproc) install + +# ================================================================== +# python +# ------------------------------------------------------------------ +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --recursive" && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + software-properties-common \ + && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python3-distutils-extra \ + && \ + wget -O ~/get-pip.py \ + https://bootstrap.pypa.io/get-pip.py && \ + python${PYTHON_VERSION} ~/get-pip.py && \ + ln -s /usr/bin/python${PYTHON_VERSION} /usr/local/bin/python3 && \ + ln -s /usr/bin/python${PYTHON_VERSION} /usr/local/bin/python + +# ================================================================== +# dependencies +# ------------------------------------------------------------------ +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --recursive" && \ + $PIP_INSTALL \ + cachetools \ + easydict \ + enum34 \ + future \ + motmetrics \ + numba \ + numpy \ + opencv-python \ + pillow \ + protobuf \ + psutil \ + pybind11 \ + pyquaternion \ + pyyaml \ + seaborn \ + scikit-learn \ + scikit-image \ + scipy \ + setuptools \ + shapely \ + tensorboardX \ + tqdm \ + typing \ + requests \ + && \ + +# ================================================================== +# pytorch +# ------------------------------------------------------------------ + + $PIP_INSTALL \ + torch==$TORCH_VERSION \ + torchvision==$TORCHVISION_VERSION + +# ================================================================== +# spconv +# ------------------------------------------------------------------ + +RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ + PIP_INSTALL="python -m pip --no-cache-dir install --upgrade" && \ + GIT_CLONE="git clone --recursive" && \ + DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ + libsm6 libxext6 libxrender-dev \ + libboost-all-dev && \ + +# ================================================================== +# config & cleanup +# ------------------------------------------------------------------ + + cd && \ + ldconfig && \ + apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* \ + /tmp/* \ + ~/* + +# ================================================================== +# environment setup +# ------------------------------------------------------------------ + +ARG LOST_ROOT=/root/lost +ENV LOST_ROOT=$LOST_ROOT + +ARG NUSC_ROOT=/root/nusc +ENV NUSC_ROOT=$NUSC_ROOT + +ARG CADC_ROOT=/root/cadc +ENV CADC_ROOT=$CADC_ROOT + +ARG KITTI_ROOT=/root/kitti +ENV KITTI_ROOT=$KITTI_ROOT + +ARG LOGDIR=/root/logdir +ENV LOGDIR=$LOGDIR + +ENV PYTHONPATH=$PYTHONPATH:$LOST_ROOT +ENV PYTHONPATH=$PYTHONPATH:$NUSC_ROOT/nuscenes-devkit/python-sdk + +# TODO: Create dataset symlinks for datasets + +VOLUME ["$LOST_ROOT", "$CADC_ROOT", "$NUSC_ROOT", "$KITTI_ROOT", "$LOGDIR"] +WORKDIR /root/ + +ENTRYPOINT ["bash"] diff --git a/docker/README.md b/docker/README.md new file mode 100755 index 0000000000000000000000000000000000000000..ae4ede480526bce4d8b77bfcc35a90d9b8673655 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,47 @@ +# LOST Docker +The following docker files and scripts were adapted from PCDet +# Docker Support +## Preparation +Before building the docker image, make sure to check the +configurations in `config.sh`. +You may need to change the `HOST_*` variables +to the **absolute** paths of the corresponding directories. +```bash +# In config.sh +HOST_PCDET_ROOT=... # PCDet location, i.e. the repo itself +HOST_NUSC_ROOT=... # NuScenes dataset that contains nuscenes-devkit + # and data directory (e.g. v1.0-mini) +HOST_CADC_ROOT=... # CADC dataset directory +HOST_LOGDIR=... # Output location + # can be used to store model checkpoints etc. +``` +Also, you need to make sure that the file structure of the dataset +match the outline in `docs/INSTALL.md`. + +## Build Docker Image +Use the provided script to build the docker image. +The name of the image is by default `pcdet-standalone`. +```bash +bash build.sh +``` +This process should take a few minutes. + +## Run Docker Container +Use the provided script to run the docker container. +The name of the container is by default `pcdet-standalone.$(whoami).$RANDOM`. +```bash +bash run.sh +``` +Additional arguments for `docker run` can be passed directly. +For example +```bash +bash run.sh --cpuset-cpus=0,1 +``` + +The following directories from host will be mounted to the container's workspace: +```bash +HOST_PCDET_ROOT => PCDET_ROOT # `/root/pcdet` by default +HOST_NUSC_ROOT => NUSC_ROOT # `/root/nusc` by default +HOST_CADC_ROOT => CADC_ROOT # `/root/cadc` by default +HOST_LOGDIR => LOGDIR # `/root/logdir` by default +``` \ No newline at end of file diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 0000000000000000000000000000000000000000..559f3840211972bfba9a38e233a9085870c6add0 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,15 @@ +#! /usr/bin/env bash +source config.sh + +docker build . \ + --build-arg CMAKE_VERSION=${CMAKE_VERSION} \ + --build-arg CMAKE_BUILD=${CMAKE_BUILD} \ + --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ + --build-arg TORCH_VERSION=${TORCH_VERSION} \ + --build-arg TORCHVISION_VERSION=${TORCHVISION_VERSION} \ + --build-arg LOST_ROOT=${LOST_ROOT} \ + --build-arg NUSC_ROOT=${NUSC_ROOT} \ + --build-arg CADC_ROOT=${CADC_ROOT} \ + --build-arg KITTI_ROOT=${KITTI_ROOT} \ + --build-arg LOGDIR=${LOGDIR} \ + -t lost-ece613 diff --git a/docker/config.sh b/docker/config.sh new file mode 100755 index 0000000000000000000000000000000000000000..7e43809eeaedde31b2fb8699dbf43d7a630a0dae --- /dev/null +++ b/docker/config.sh @@ -0,0 +1,24 @@ +#! /usr/bin/env bash + +DATASET_ROOT_PATH= + +# Build component versions +CMAKE_VERSION=3.16 +CMAKE_BUILD=5 +PYTHON_VERSION=3.7 +TORCH_VERSION=1.8.1 +TORCHVISION_VERSION=0.9.1 + +# Workspace structure in docker +LOST_ROOT=/root/lost +NUSC_ROOT=/root/nusc +CADC_ROOT=/root/cadc +KITTI_ROOT=/root/kitti +LOGDIR=/root/logdir + +# Workspace structure on host machine +HOST_LOST_ROOT=/home/sdelcore/wiselab/ece613-lost +HOST_NUSC_ROOT=/home/sdelcore/datasets/nuscenes +HOST_CADC_ROOT=/home/sdelcore/datasets/cadc +HOST_KITTI_ROOT=/home/sdelcore/datasets/kitti +HOST_LOGDIR=/home/sdelcore/datasets/logdir \ No newline at end of file diff --git a/docker/install_cmake.sh b/docker/install_cmake.sh new file mode 100644 index 0000000000000000000000000000000000000000..80c26a0e9dcdc41fd6a3f32507e139b44ebf3112 --- /dev/null +++ b/docker/install_cmake.sh @@ -0,0 +1,20 @@ +#! /usr/bin/env bash + +# Remove current version +sudo apt purge --auto-remove cmake +# https://askubuntu.com/questions/355565/how-do-i-install-the-latest-version-of-cmake-from-the-command-line +version=3.16 +build=5 +mkdir ~/temp +cd ~/temp +wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz +tar -xzvf cmake-$version.$build.tar.gz +cd cmake-$version.$build/ +# install +cd ~/temp/cmake-$version.$build +./bootstrap +make -j$(nproc) +sudo make install +# Check the version +cmake --version + diff --git a/docker/install_spconv.sh b/docker/install_spconv.sh new file mode 100644 index 0000000000000000000000000000000000000000..f21f83460b673d51dfc03b968134349e30d1ac77 --- /dev/null +++ b/docker/install_spconv.sh @@ -0,0 +1,28 @@ +#! /usr/bin/env bash + +# Ubuntu 18.04 +# Step 0: Clone repo +echo "Enter FULL path of where to clone spconv" +read spconv_path +cd $spconv_path +git clone git@github.com:traveller59/spconv.git --recursive +# checkout older spconv 1.0 +cd $spconv_path/spconv/ +git checkout 8da6f967fb9a054d8870c3515b1b44eca2103634 +# Step 1: Install boost headers +sudo apt-get install libboost-all-dev +# Step 2: +cmake --version +echo "if the cmake version above is < 3.13.2 then run update_cmake.bash!" +echo "press enter" +read tmp +# Step 3: Run set up +pip show torch +echo "torch should be > 1.0!" +echo "press enter" +read tmp +python setup.py bdist_wheel +# Step 4: pip install the generated whl file +cd $spconv_path/spconv/dist +pip install spconv*.whl + diff --git a/docker/run.sh b/docker/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..a5957f24df36de893e7e4782f2d317b43baecb0b --- /dev/null +++ b/docker/run.sh @@ -0,0 +1,23 @@ +#! /usr/bin/env bash +source config.sh + +docker run \ + --name="lost-ece613.$(whoami).$RANDOM" \ + --gpus=all \ + --rm \ + -it \ + -v "${HOST_LOST_ROOT}":"${LOST_ROOT}" \ + -v "${HOST_NUSC_ROOT}":"${NUSC_ROOT}" \ + -v "${HOST_CADC_ROOT}":"${CADC_ROOT}" \ + -v "${HOST_KITTI_ROOT}":"${KITTI_ROOT}" \ + -v "${HOST_LOGDIR}":"${LOGDIR}" \ + -e DISPLAY=$DISPLAY \ + -e PUID=1000 \ + -e PGID=1000 \ + $@ \ + lost-ece613 + + # -p 5000:8888 \ +# -p 5001:6006 \ +# -p 5002:8889 \ +# -p 5003:8890 \