Skip to content

Commit 1c5c3fc

Browse files
daniel-j-hdaniel-j-h
committed
Initial commit
0 parents  commit 1c5c3fc

36 files changed

+1584
-0
lines changed

‎.dockerignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.git
2+
.gitignore

‎.flake8

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[flake8]
2+
max-line-length = 120

‎.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
__pycache__
2+
*.py[cod]
3+
4+
*.pth
5+
*.pb

‎Dockerfile.cpu

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
FROM ubuntu:18.04
2+
3+
WORKDIR /usr/src/app
4+
5+
ENV LANG="C.UTF-8" LC_ALL="C.UTF-8" PATH="/opt/venv/bin:$PATH" PIP_NO_CACHE_DIR="false" CFLAGS="-mavx2" CXXFLAGS="-mavx2"
6+
7+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
8+
python3 python3-pip python3-venv \
9+
wget make g++ ffmpeg python3-dev libblas-dev liblapack-dev swig \
10+
cmake yasm zlib1g-dev && \
11+
rm -rf /var/lib/apt/lists/*
12+
13+
COPY requirements.txt .
14+
15+
RUN python3 -m venv /opt/venv && \
16+
python3 -m pip install pip==19.2.3 pip-tools==4.0.0
17+
18+
# For pytorch and torchvision we need platform specific (cpu vs. gpu) wheels from
19+
# https://download.pytorch.org/whl/cpu/torch_stable.html
20+
# To generate hashes run: python3 -m pip hash *.whl
21+
RUN echo "https://download.pytorch.org/whl/cpu/torch-1.2.0%2Bcpu-cp36-cp36m-manylinux1_x86_64.whl \
22+
--hash=sha256:7b9b943673d3acb446248ba0d6feed6926bf60ce719ace4707a6559c1f57ced7 \
23+
\n \
24+
https://download.pytorch.org/whl/cpu/torchvision-0.4.0%2Bcpu-cp36-cp36m-manylinux1_x86_64.whl \
25+
--hash=sha256:63f342b858b18839fcf3ff8ad857e44a4ff0fcb8cb8e2bdc2f4ed9afa7cec9e0 \
26+
\n" >> requirements.txt && cat requirements.txt
27+
28+
RUN python3 -m piptools sync
29+
30+
RUN python3 -c "from torchvision.models import resnet50; resnet50(pretrained=True, progress=False)" && \
31+
python3 -c "from torchvision.models.video import r2plus1d_18; r2plus1d_18(pretrained=True, progress=False)"
32+
33+
RUN wget -q https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.3.tar.gz -O libjpeg-turbo.tar.gz && \
34+
echo "a69598bf079463b34d45ca7268462a18b6507fdaa62bb1dfd212f02041499b5d libjpeg-turbo.tar.gz" | sha256sum -c && \
35+
tar xf libjpeg-turbo.tar.gz && \
36+
rm libjpeg-turbo.tar.gz && \
37+
cd libjpeg-turbo* && \
38+
mkdir build && \
39+
cd build && \
40+
cmake -DCMAKE_BUILD_TYPE=Release -DREQUIRE_SIMD=On -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
41+
make -j $(nproc) && \
42+
make install && \
43+
ldconfig && \
44+
cd ../../ && \
45+
rm -rf libjpeg-turbo*
46+
47+
RUN python3 -m pip uninstall -y pillow && \
48+
python3 -m pip install --no-binary :all: --compile pillow-simd==6.0.0.post0
49+
50+
RUN wget -q https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz -O faiss.tar.gz && \
51+
echo "b24d347b0285d01c2ed663ccc7596cd0ea95071f3dd5ebb573ccfc28f15f043b faiss.tar.gz" | sha256sum -c && \
52+
tar xf faiss.tar.gz && \
53+
rm faiss.tar.gz && \
54+
cd faiss* && \
55+
./configure --without-cuda && \
56+
make -j $(nproc) && \
57+
make -j $(nproc) -C python && \
58+
make install && \
59+
make -C python install && \
60+
cd .. && \
61+
rm -rf faiss*
62+
63+
COPY . .
64+
65+
EXPOSE 5000
66+
ENTRYPOINT ["/usr/src/app/bin/sfi"]
67+
CMD ["-h"]

‎Dockerfile.gpu

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
FROM nvidia/cuda:10.1-cudnn7-devel
2+
3+
WORKDIR /usr/src/app
4+
5+
ENV LANG="C.UTF-8" LC_ALL="C.UTF-8" PATH="/opt/venv/bin:$PATH" PIP_NO_CACHE_DIR="false" CFLAGS="-mavx2" CXXFLAGS="-mavx2"
6+
7+
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
8+
python3 python3-pip python3-venv \
9+
wget make g++ ffmpeg python3-dev libblas-dev liblapack-dev swig \
10+
cmake yasm zlib1g-dev && \
11+
rm -rf /var/lib/apt/lists/*
12+
13+
COPY requirements.txt .
14+
15+
RUN python3 -m venv /opt/venv && \
16+
python3 -m pip install pip==19.2.3 pip-tools==4.0.0
17+
18+
# For pytorch and torchvision we need platform specific (cpu vs. gpu) wheels from
19+
# https://download.pytorch.org/whl/cu100/torch_stable.html
20+
# To generate hashes run: python3 -m pip hash *.whl
21+
RUN echo "https://download.pytorch.org/whl/cu100/torch-1.2.0-cp36-cp36m-manylinux1_x86_64.whl \
22+
--hash=sha256:a13bf6f78a49d844b85c142b8cd62d2e1833a11ed21ea0bc6b1ac73d24c76415 \
23+
\n \
24+
https://download.pytorch.org/whl/cu100/torchvision-0.4.0-cp36-cp36m-manylinux1_x86_64.whl \
25+
--hash=sha256:2f67efdf6edd9ea7f9cd9a3917ae5c63d5684e3bdb5cc9c2b364c15bdfe4456b \
26+
\n" >> requirements.txt
27+
28+
RUN python3 -m piptools sync
29+
30+
RUN python3 -c "from torchvision.models import resnet50; resnet50(pretrained=True, progress=False)" && \
31+
python3 -c "from torchvision.models.video import r2plus1d_18; r2plus1d_18(pretrained=True, progress=False)"
32+
33+
RUN wget -q https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.3.tar.gz -O libjpeg-turbo.tar.gz && \
34+
echo "a69598bf079463b34d45ca7268462a18b6507fdaa62bb1dfd212f02041499b5d libjpeg-turbo.tar.gz" | sha256sum -c && \
35+
tar xf libjpeg-turbo.tar.gz && \
36+
rm libjpeg-turbo.tar.gz && \
37+
cd libjpeg-turbo* && \
38+
mkdir build && \
39+
cd build && \
40+
cmake -DCMAKE_BUILD_TYPE=Release -DREQUIRE_SIMD=On -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
41+
make -j $(nproc) && \
42+
make install && \
43+
ldconfig && \
44+
cd ../../ && \
45+
rm -rf libjpeg-turbo*
46+
47+
48+
RUN python3 -m pip uninstall -y pillow && \
49+
python3 -m pip install --no-binary :all: --compile pillow-simd==6.0.0.post0
50+
51+
RUN wget -q https://github.com/facebookresearch/faiss/archive/v1.5.3.tar.gz -O faiss.tar.gz && \
52+
echo "b24d347b0285d01c2ed663ccc7596cd0ea95071f3dd5ebb573ccfc28f15f043b faiss.tar.gz" | sha256sum -c && \
53+
tar xf faiss.tar.gz && \
54+
rm faiss.tar.gz && \
55+
cd faiss* && \
56+
./configure --with-cuda-arch="-gencode=arch=compute_37,code=compute_37 -gencode=arch=compute_70,code=compute_70" --with-cuda="/usr/local/cuda" && \
57+
make -j $(nproc) && \
58+
make -j $(nproc) -C python && \
59+
make install && \
60+
make -C python install && \
61+
cd .. && \
62+
rm -rf faiss*
63+
64+
COPY . .
65+
66+
EXPOSE 5000
67+
ENTRYPOINT ["/usr/src/app/bin/sfi"]
68+
CMD ["-h"]

‎LICENSE.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2019 MoabitCoin
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎Makefile

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
dockerimage ?= moabitcoin/sfi
2+
dockerfile ?= Dockerfile.cpu
3+
srcdir ?= $(shell pwd)
4+
datadir ?= $(shell pwd)
5+
6+
install:
7+
@docker build -t $(dockerimage) -f $(dockerfile) .
8+
9+
i: install
10+
11+
12+
update:
13+
@docker build -t $(dockerimage) -f $(dockerfile) . --pull --no-cache
14+
15+
u: update
16+
17+
18+
run:
19+
@docker run -it --rm --ipc="host" --network="host" -p 5000:5000 -v $(srcdir)/sfi:/usr/src/app/sfi -v $(datadir):/data --entrypoint=/bin/bash $(dockerimage)
20+
21+
r: run
22+
23+
24+
publish:
25+
@docker image save $(dockerimage) \
26+
| pv -N "Publish $(dockerimage) to $(sshopts)" -s $(shell docker image inspect $(dockerimage) --format "{{.Size}}") \
27+
| ssh $(sshopts) "docker image load"
28+
29+
p: publish
30+
31+
32+
.PHONY: install i run r update u publish p

‎README.md

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Semantic Frame Index
2+
3+
Fast and efficient queries on video frames by semantic similarity.
4+
5+
6+
## Use Case
7+
8+
We record tens of thousand hours of drive video data and need to be able to search for semantically similar scenarios.
9+
Simlarity could mean similar lighting conditions, similar vehicle types, similar traffic volumes, similar objects on the road, and so on.
10+
11+
12+
## Implementation Sketch
13+
14+
We
15+
- extract key frames using a neural net for frame similarity in feature space
16+
- extract a trained convolutional neural net's high level feature maps for all key frames
17+
- compute Maximum Activations of Convolution (MAC) features from the high-level feature maps
18+
- index the feature maps for approximate nearest neighbor searches based on L2 distance
19+
- query the indexed dataset for semantically similar scenarios
20+
21+
22+
## Usage
23+
24+
All tools can be invoked via
25+
26+
./bin/sfi <tool> <args>
27+
28+
./bin/sfi --help
29+
./bin/sfi <tool> --help
30+
31+
32+
### stream-index
33+
34+
Builds an index from a directory of images for fast and efficient approximate nearest neighbor queries based on L2 distance.
35+
The quantizer for the index needs to get trained on a small subset of the feature maps to approximate the dataset's centroids.
36+
We recommend runing this step on GPUs.
37+
38+
39+
### save-feature
40+
41+
Extracts high level feature maps and computes MACs for an image frames from a trained convolutional neural net.
42+
43+
44+
### save-frames
45+
46+
Extracts semantic key frames from videos based on a trained convolution net for feature similarity between frames.
47+
48+
49+
### query-server
50+
51+
Loads up the index (slow) and keeps it in memory to handle nearest neighbor queries (fast).
52+
Responds to queries by searching the index, aggregating results, and re-ranking them.
53+
54+
55+
### query-client
56+
57+
Sends nearest neighbor requests against the query server and reports results to the user.
58+
The query and results are based on the saved MAC features.
59+
60+
61+
### model-train
62+
63+
Trains a binary classification model on a dataset (potentially noisy and obtained from the index).
64+
We recommend runing this step on GPUs.
65+
66+
67+
### model-infer
68+
69+
Predicts binary classification labels on a dataset, using a trained model.
70+
71+
72+
## Development
73+
74+
Create a self-contained reproducible development environment
75+
76+
make i
77+
78+
Get into the development environment
79+
80+
make r
81+
82+
The Python source code directory is mounted into the container: if you modify it on the host it will get modified in the container.
83+
84+
To make data visible in the container set the datadir env var, e.g. to make your `/tmp` directory show up in `/data` inside the container run
85+
86+
make r datadir=/tmp
87+
88+
See the `Makefile` for options and more advanced targets.
89+
90+
91+
## References
92+
93+
- [Particular object retrieval with integral max-pooling of CNN activations](https://arxiv.org/abs/1511.05879)
94+
- Product Quantizer (PQ) [part 1](http://mccormickml.com/2017/10/13/product-quantizer-tutorial-part-1/), and [part 2](http://mccormickml.com/2017/10/22/product-quantizer-tutorial-part-2/)
95+
- [Product Quantization for Nearest Neighbor Search](https://hal.inria.fr/file/index/docid/514462/filename/paper_hal.pdf)
96+
- [Billion-scale similarity search with GPUs](https://arxiv.org/pdf/1702.08734.pdf)
97+
- [faiss wiki](https://github.com/facebookresearch/faiss/wiki)
98+
99+
100+
## License
101+
102+
Copyright © 2019 MoabitCoin
103+
104+
Distributed under the MIT License (MIT).

‎bin/sfi

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
python3 -m sfi.tools "$@"

‎requirements.in

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
numpy
2+
pillow
3+
tqdm
4+
flask
5+
requests
6+
einops
7+
scikit-video

0 commit comments

Comments
 (0)