From 5e31f2150f2e3e4f726046f2e956632ff4f36759 Mon Sep 17 00:00:00 2001
From: Benny <775410794@qq.com>
Date: Tue, 26 Nov 2019 18:45:05 +0800
Subject: [PATCH] update framework
---
README.md | 107 ++++----
data_utils/ModelNetDataLoader.py | 146 +++++++----
data_utils/S3DISDataLoader.py | 386 +++++++++++++++++++++++++----
data_utils/ShapeNetDataLoader.py | 58 ++---
model/pointnet.py | 284 ---------------------
models/pointnet.py | 141 +++++++++++
{model => models}/pointnet2.py | 8 +-
models/pointnet2_cls_msg.py | 51 ++++
models/pointnet2_cls_ssg.py | 50 ++++
models/pointnet2_part_seg_msg.py | 59 +++++
models/pointnet2_part_seg_ssg.py | 59 +++++
models/pointnet2_sem_seg.py | 63 +++++
models/pointnet_cls.py | 40 +++
models/pointnet_part_seg.py | 86 +++++++
models/pointnet_sem_seg.py | 54 ++++
{model => models}/pointnet_util.py | 0
provider.py | 251 +++++++++++++++++++
test_cls.py | 105 ++++++++
test_partseg.py | 157 ++++++++++++
train_clf.py | 159 ------------
train_cls.py | 209 ++++++++++++++++
train_partseg.py | 325 ++++++++++++++++--------
train_semseg.py | 354 +++++++++++++++++---------
utils.py | 229 -----------------
visualizer/example.jpg | Bin 19459 -> 0 bytes
visualizer/pic2.png | Bin 0 -> 220561 bytes
visualizer/show3d_balls.py | 201 +++++++++------
27 files changed, 2420 insertions(+), 1162 deletions(-)
delete mode 100644 model/pointnet.py
create mode 100644 models/pointnet.py
rename {model => models}/pointnet2.py (95%)
create mode 100644 models/pointnet2_cls_msg.py
create mode 100644 models/pointnet2_cls_ssg.py
create mode 100644 models/pointnet2_part_seg_msg.py
create mode 100644 models/pointnet2_part_seg_ssg.py
create mode 100644 models/pointnet2_sem_seg.py
create mode 100644 models/pointnet_cls.py
create mode 100644 models/pointnet_part_seg.py
create mode 100644 models/pointnet_sem_seg.py
rename {model => models}/pointnet_util.py (100%)
create mode 100644 provider.py
create mode 100644 test_cls.py
create mode 100644 test_partseg.py
delete mode 100644 train_clf.py
create mode 100644 train_cls.py
delete mode 100644 utils.py
delete mode 100644 visualizer/example.jpg
create mode 100644 visualizer/pic2.png
diff --git a/README.md b/README.md
index 72abc7042..0a545ccd5 100644
--- a/README.md
+++ b/README.md
@@ -2,72 +2,81 @@
This repo is implementation for [PointNet](http://openaccess.thecvf.com/content_cvpr_2017/papers/Qi_PointNet_Deep_Learning_CVPR_2017_paper.pdf) and [PointNet++](http://papers.nips.cc/paper/7095-pointnet-deep-hierarchical-feature-learning-on-point-sets-in-a-metric-space.pdf) in pytorch.
-## Data Preparation
-* Download **ModelNet** [here](http://modelnet.cs.princeton.edu/ModelNet40.zip) for classification and **ShapeNet** [here](https://shapenet.cs.stanford.edu/media/shapenetcore_partanno_segmentation_benchmark_v0_normal.zip) for part segmentation. Uncompress the downloaded data in this directory. `./data/ModelNet` and `./data/ShapeNet`.
-* Run `download_data.sh` and download prepared **S3DIS** dataset for sematic segmantation and save it in `./data/indoor3d_sem_seg_hdf5_data/`
+## Update
+* 2019/11/26: (1). Fixed some errors in previous codes and added data augmentation tricks. Now classification can achieve 92.5\%! (2). Added testing codes, including classification and segmentation, and semantic segmentation with visualization. (3). Organized all models into `./models` files for easy use.
+
## Classification
-### PointNet
-* python train_clf.py --model_name pointnet
-### PointNet++
-* python train_clf.py --model_name pointnet2
+### Data Preparation
+Download alignment **ModelNet** [here](https://shapenet.cs.stanford.edu/media/modelnet40_normal_resampled.zip) and save in `data/modelnet40_normal_resampled/`
+
+### Run
+```
+## Check model in ./models folder
+## E.g. pointnet2_msg
+python train_cls.py --model pointnet2_cls_msg --normal --log_dir pointnet2_cls_msg
+python test_cls.py --normal --log_dir pointnet2_cls_msg
+```
+
### Performance
| Model | Accuracy |
|--|--|
| PointNet (Official) | 89.2|
-| PointNet (Pytorch) | **89.4**|
-| PointNet++ (Official) | **91.9** |
-| PointNet++ (Pytorch) | 91.8 |
-
-* Training Pointnet with 0.001 learning rate in SGD, 24 batchsize and 141 epochs.
-* Training Pointnet++ with 0.001 learning rate in SGD, 12 batchsize and 45 epochs.
+| PointNet++ (Official) | 91.9 |
+|--|--|
+| PointNet (Pytorch without normal) | 90.6|
+| PointNet (Pytorch with normal) | 91.4|
+| PointNet2_ssg (Pytorch without normal) | 92.2|
+| PointNet2_ssg (Pytorch with normal) | **92.4**|
## Part Segmentation
-### PointNet
-* python train_partseg.py --model_name pointnet
-### PointNet++
-* python train_partseg.py --model_name pointnet2
+### Data Preparation
+Download alignment **ShapeNet** [here](https://shapenet.cs.stanford.edu/media/shapenetcore_partanno_segmentation_benchmark_v0_normal.zip) and save in `data/shapenetcore_partanno_segmentation_benchmark_v0_normal/`
+### Run
+```
+## Check model in ./models folder
+## E.g. pointnet2_msg
+python train_partseg.py --model pointnet2_part_seg_msg --normal
+python test_partseg.py --normal --log_dir pointnet2_part_seg_msg
+```
### Performance
-| Model | Inctance avg | Class avg |aero | bag | cap |car |chair |ear phone |guitar | knife | lamp |laptop | motor |mug | pistol |rocket | skate board | table |
-|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|
-|PointNet (Official) |**83.7**|**80.4** |83.4| 78.7| 82.5| 74.9| 89.6 |73| 91.5| 85.9 |80.8| 95.3| 65.2 |93| 81.2| 57.9| 72.8| 80.6|
-|PointNet (Pytorch)| 82.4 |78.4| 81.1 |77.8 |83.7 |74.3 |83.3| 65.7| 90.5 |85.1| 78.1 |94.5 |63.7 |91.7 |80.5|56.2 |73.7 |67.5|
-|PointNet++ (Official)|**85.1** |**81.9** |82.4|79 |87.7 |77.3| 90.8| 71.8| 91| 85.9| 83.7| 95.3 |71.6| 94.1 |81.3| 58.7| 76.4| 82.6|
-|PointNet++ (Pytorch)| 84.1| 81.6 |82.6| 85.7| 89.3 |78.1|86.8| 68.9 |91.6| 88.9| 83.9 |96.8 |70.1 |95.7 |82.8| 59.8 |76.3 |71.1|
-
-* Training both Pointnet and Pointnet++ with 0.001 learning rate in Adam, 16 batchsize, about 130 epochs and 0.5 learning rate decay every 20/30 epochs.
-* **Class avg** is the mean IoU averaged across all object categories, and **inctance avg** is the mean IoU across all objects.
-* In official version PointNet, author use 2048 point cloud in training and 3000 point cloud with norm in testing. In official version PointNet++, author use 2048 point cloud with its norm (Bx2048x6) in both training and testing.
-
+| Model | Inctance avg | Class avg
+|--|--|--|
+|PointNet (Official) |**83.7**|**80.4**
+|PointNet (Pytorch)| - |-|
+|PointNet++ (Official)|**85.1** |**81.9**
+|PointNet++ (Pytorch)| -| -
## Semantic Segmentation
-### PointNet
-* python train_semseg.py --model_name pointnet
-### PointNet++
-* python train_semseg.py --model_name pointnet2
+### Run
+```
+## Check model in ./models folder
+## E.g. pointnet2_ssg
+python train_semseg.py --model pointnet2_sem_seg --normal
+python test_semseg.py --normal --log_dir pointnet2_sem_seg
+```
### Performance (test on Area_5)
-|Model | Mean IOU | ceiling | floor | wall | beam | column | window | door | chair| tabel| bookcase| sofa | board | clutter |
-|--|--|--|--|--|--|--|--|--|--|--|--|--|--|--|
-| PointNet (Official) | 41.09|88.8|**97.33**|69.8|0.05|3.92|**46.26**|10.76|**52.61**|**58.93**|**40.28**|5.85|26.38|33.22|
-| PointNet (Pytorch) | **44.43**|**91.1**|96.8|**72.1**|**5.82**|**14.7**|36.03|**37.1**|49.36|50.17|35.99|**14.26**|**33.9**|**40.23**|
-| PointNet++ (Official) |N/A | | | | | | | |
-| PointNet++ (Pytorch) | **52.28**|91.7|95.9|74.6|0.1|18.9|43.3|31.1|73.1|65.8|51.1|27.5|43.8|53.8|
-* Training Pointnet with 0.001 learning rate in Adam, 24 batchsize and 84 epochs.
-* Training Pointnet++ with 0.001 learning rate in Adam, 12 batchsize and 67 epochs.
+|Model | Mean IOU |
+|--|--|
+| PointNet (Official) | 41.09|
+| PointNet (Pytorch) | -|
+| PointNet++ (Official) |N/A |
+| PointNet++ (Pytorch) | -|
+
+
## Visualization
### Using show3d_balls.py
-`cd visualizer`
-`bash build.sh #build C++ code for visualization`
+```
+cd visualizer
+bash build.sh #build C++ code for visualization
+## run one example
+python show3d_balls.py
+```
-### Using pc_utils.py
-
-
-## TODO
+### Using MeshLab
+
-- [x] PointNet and PointNet++
-- [x] Experiment
-- [x] Visualization Tool
## Reference By
[halimacc/pointnet3](https://github.com/halimacc/pointnet3)
diff --git a/data_utils/ModelNetDataLoader.py b/data_utils/ModelNetDataLoader.py
index ccbc19d2f..de67e3136 100644
--- a/data_utils/ModelNetDataLoader.py
+++ b/data_utils/ModelNetDataLoader.py
@@ -1,67 +1,103 @@
import numpy as np
import warnings
-import h5py
+import os
from torch.utils.data import Dataset
warnings.filterwarnings('ignore')
-def load_h5(h5_filename):
- f = h5py.File(h5_filename)
- data = f['data'][:]
- label = f['label'][:]
- seg = []
- return (data, label, seg)
-
-def load_data(dir,classification = False):
- data_train0, label_train0,Seglabel_train0 = load_h5(dir + 'ply_data_train0.h5')
- data_train1, label_train1,Seglabel_train1 = load_h5(dir + 'ply_data_train1.h5')
- data_train2, label_train2,Seglabel_train2 = load_h5(dir + 'ply_data_train2.h5')
- data_train3, label_train3,Seglabel_train3 = load_h5(dir + 'ply_data_train3.h5')
- data_train4, label_train4,Seglabel_train4 = load_h5(dir + 'ply_data_train4.h5')
- data_test0, label_test0,Seglabel_test0 = load_h5(dir + 'ply_data_test0.h5')
- data_test1, label_test1,Seglabel_test1 = load_h5(dir + 'ply_data_test1.h5')
- train_data = np.concatenate([data_train0,data_train1,data_train2,data_train3,data_train4])
- train_label = np.concatenate([label_train0,label_train1,label_train2,label_train3,label_train4])
- train_Seglabel = np.concatenate([Seglabel_train0,Seglabel_train1,Seglabel_train2,Seglabel_train3,Seglabel_train4])
- test_data = np.concatenate([data_test0,data_test1])
- test_label = np.concatenate([label_test0,label_test1])
- test_Seglabel = np.concatenate([Seglabel_test0,Seglabel_test1])
-
- if classification:
- return train_data, train_label, test_data, test_label
- else:
- return train_data, train_Seglabel, test_data, test_Seglabel
+
+
+def pc_normalize(pc):
+ centroid = np.mean(pc, axis=0)
+ pc = pc - centroid
+ m = np.max(np.sqrt(np.sum(pc**2, axis=1)))
+ pc = pc / m
+ return pc
+
+def farthest_point_sample(point, npoint):
+ """
+ Input:
+ xyz: pointcloud data, [N, D]
+ npoint: number of samples
+ Return:
+ centroids: sampled pointcloud index, [npoint, D]
+ """
+ N, D = point.shape
+ xyz = point[:,:3]
+ centroids = np.zeros((npoint,))
+ distance = np.ones((N,)) * 1e10
+ farthest = np.random.randint(0, N)
+ for i in range(npoint):
+ centroids[i] = farthest
+ centroid = xyz[farthest, :]
+ dist = np.sum((xyz - centroid) ** 2, -1)
+ mask = dist < distance
+ distance[mask] = dist[mask]
+ farthest = np.argmax(distance, -1)
+ point = point[centroids.astype(np.int32)]
+ return point
class ModelNetDataLoader(Dataset):
- def __init__(self, data, labels, rotation = None):
- self.data = data
- self.labels = labels
- self.rotation = rotation
+ def __init__(self, root, npoint=1024, split='train', uniform=False, normal_channel=True, cache_size=15000):
+ self.root = root
+ self.npoints = npoint
+ self.uniform = uniform
+ self.catfile = os.path.join(self.root, 'modelnet40_shape_names.txt')
+
+ self.cat = [line.rstrip() for line in open(self.catfile)]
+ self.classes = dict(zip(self.cat, range(len(self.cat))))
+ self.normal_channel = normal_channel
+
+ shape_ids = {}
+ shape_ids['train'] = [line.rstrip() for line in open(os.path.join(self.root, 'modelnet40_train.txt'))]
+ shape_ids['test'] = [line.rstrip() for line in open(os.path.join(self.root, 'modelnet40_test.txt'))]
+
+ assert (split == 'train' or split == 'test')
+ shape_names = ['_'.join(x.split('_')[0:-1]) for x in shape_ids[split]]
+ # list of (shape_name, shape_txt_file_path) tuple
+ self.datapath = [(shape_names[i], os.path.join(self.root, shape_names[i], shape_ids[split][i]) + '.txt') for i
+ in range(len(shape_ids[split]))]
+ print('The size of %s data is %d'%(split,len(self.datapath)))
+
+ self.cache_size = cache_size # how many data points to cache in memory
+ self.cache = {} # from index to (point_set, cls) tuple
def __len__(self):
- return len(self.data)
-
- def rotate_point_cloud_by_angle(self, data, rotation_angle):
- """
- Rotate the point cloud along up direction with certain angle.
- :param batch_data: Nx3 array, original batch of point clouds
- :param rotation_angle: range of rotation
- :return: Nx3 array, rotated batch of point clouds
- """
- cosval = np.cos(rotation_angle)
- sinval = np.sin(rotation_angle)
- rotation_matrix = np.array([[cosval, 0, sinval],
- [0, 1, 0],
- [-sinval, 0, cosval]])
- rotated_data = np.dot(data, rotation_matrix)
-
- return rotated_data
+ return len(self.datapath)
+
+ def _get_item(self, index):
+ if index in self.cache:
+ point_set, cls = self.cache[index]
+ else:
+ fn = self.datapath[index]
+ cls = self.classes[self.datapath[index][0]]
+ cls = np.array([cls]).astype(np.int32)
+ point_set = np.loadtxt(fn[1], delimiter=',').astype(np.float32)
+ if self.uniform:
+ point_set = farthest_point_sample(point_set, self.npoints)
+ else:
+ point_set = point_set[0:self.npoints,:]
+
+ point_set[:, 0:3] = pc_normalize(point_set[:, 0:3])
+
+ if not self.normal_channel:
+ point_set = point_set[:, 0:3]
+
+ if len(self.cache) < self.cache_size:
+ self.cache[index] = (point_set, cls)
+
+ return point_set, cls
def __getitem__(self, index):
- if self.rotation is not None:
- pointcloud = self.data[index]
- angle = np.random.randint(self.rotation[0], self.rotation[1]) * np.pi / 180
- pointcloud = self.rotate_point_cloud_by_angle(pointcloud, angle)
+ return self._get_item(index)
- return pointcloud, self.labels[index]
- else:
- return self.data[index], self.labels[index]
\ No newline at end of file
+
+
+
+if __name__ == '__main__':
+ import torch
+
+ data = ModelNetDataLoader('/data/modelnet40_normal_resampled/',split='train', uniform=False, normal_channel=True,)
+ DataLoader = torch.utils.data.DataLoader(data, batch_size=12, shuffle=True)
+ for point,label in DataLoader:
+ print(point.shape)
+ print(label.shape)
\ No newline at end of file
diff --git a/data_utils/S3DISDataLoader.py b/data_utils/S3DISDataLoader.py
index 2a0dc4647..06de12b28 100644
--- a/data_utils/S3DISDataLoader.py
+++ b/data_utils/S3DISDataLoader.py
@@ -1,61 +1,337 @@
-# *_*coding:utf-8 *_*
import os
-from torch.utils.data import Dataset
+import sys
import numpy as np
-import h5py
-
-classes = ['ceiling','floor','wall','beam','column','window','door','table','chair','sofa','bookcase','board','clutter']
-class2label = {cls: i for i,cls in enumerate(classes)}
-
-def getDataFiles(list_filename):
- return [line.rstrip() for line in open(list_filename)]
-
-def load_h5(h5_filename):
- f = h5py.File(h5_filename)
- data = f['data'][:]
- label = f['label'][:]
- return (data, label)
-
-def loadDataFile(filename):
- return load_h5(filename)
-
-def recognize_all_data(test_area = 5):
- ALL_FILES = getDataFiles('./data/indoor3d_sem_seg_hdf5_data/all_files.txt')
- room_filelist = [line.rstrip() for line in open('./data/indoor3d_sem_seg_hdf5_data/room_filelist.txt')]
- data_batch_list = []
- label_batch_list = []
- for h5_filename in ALL_FILES:
- data_batch, label_batch = loadDataFile('./data/' + h5_filename)
- data_batch_list.append(data_batch)
- label_batch_list.append(label_batch)
- data_batches = np.concatenate(data_batch_list, 0)
- label_batches = np.concatenate(label_batch_list, 0)
-
- test_area = 'Area_' + str(test_area)
- train_idxs = []
- test_idxs = []
- for i, room_name in enumerate(room_filelist):
- if test_area in room_name:
- test_idxs.append(i)
- else:
- train_idxs.append(i)
-
- train_data = data_batches[train_idxs, ...]
- train_label = label_batches[train_idxs]
- test_data = data_batches[test_idxs, ...]
- test_label = label_batches[test_idxs]
- print('train_data',train_data.shape,'train_label' ,train_label.shape)
- print('test_data',test_data.shape,'test_label', test_label.shape)
- return train_data,train_label,test_data,test_label
-
-
-class S3DISDataLoader(Dataset):
- def __init__(self, data, labels):
- self.data = data
- self.labels = labels
+from torch.utils.data import Dataset
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(BASE_DIR)
+sys.path.append(BASE_DIR)
+sys.path.append(ROOT_DIR)
+
+# root = '../data/S3DIS/stanford_indoor3d/'
+
+class S3DISDataset(Dataset):
+ def __init__(self, root, block_points=8192, split='train', test_area=5, with_rgb=True, use_weight=True, block_size=1.5, padding=0.001):
+ self.npoints = block_points
+ self.block_size = block_size
+ self.padding = padding
+ self.root = root
+ self.with_rgb = with_rgb
+ self.split = split
+ assert split in ['train','test']
+ if self.split == 'train':
+ self.file_list = [d for d in os.listdir(root) if d.find('Area_%d'%test_area) is -1]
+ else:
+ self.file_list = [d for d in os.listdir(root) if d.find('Area_%d'%test_area) is not -1]
+ self.scene_points_list = []
+ self.semantic_labels_list = []
+ for file in self.file_list:
+ data = np.load(root+file)
+ self.scene_points_list.append(data[:,:6])
+ self.semantic_labels_list.append(data[:,6])
+ assert len(self.scene_points_list)==len(self.semantic_labels_list)
+ print('Number of scene: ',len(self.scene_points_list))
+ if split=='train' and use_weight:
+ labelweights = np.zeros(13)
+ for seg in self.semantic_labels_list:
+ tmp,_ = np.histogram(seg,range(14))
+ labelweights += tmp
+ labelweights = labelweights.astype(np.float32)
+ labelweights = labelweights/np.sum(labelweights)
+ self.labelweights = np.power(np.amax(labelweights) / labelweights, 1/3.0)
+ else:
+ self.labelweights = np.ones(13)
+
+ print(self.labelweights)
+
+ def __getitem__(self, index):
+ if self.with_rgb:
+ point_set = self.scene_points_list[index]
+ point_set[:,3:] = 2 * point_set[:,3:] / 255.0 - 1
+ else:
+ point_set = self.scene_points_list[index][:, 0:3]
+ semantic_seg = self.semantic_labels_list[index].astype(np.int32)
+ coordmax = np.max(point_set[:, 0:3], axis=0)
+ coordmin = np.min(point_set[:, 0:3], axis=0)
+ for i in range(10):
+ curcenter = point_set[np.random.choice(len(semantic_seg), 1)[0], 0:3]
+ curmin = curcenter - [self.block_size/2, self.block_size/2, 1.5]
+ curmax = curcenter + [self.block_size/2, self.block_size/2, 1.5]
+ curmin[2] = coordmin[2]
+ curmax[2] = coordmax[2]
+ curchoice = np.sum((point_set[:, 0:3] >= (curmin - 0.2)) * (point_set[:, 0:3] <= (curmax + 0.2)),
+ axis=1) == 3
+ cur_point_set = point_set[curchoice, 0:3]
+ cur_point_full = point_set[curchoice, :]
+ cur_semantic_seg = semantic_seg[curchoice]
+ if len(cur_semantic_seg) == 0:
+ continue
+
+ mask = np.sum((cur_point_set >= (curmin - self.padding)) * (cur_point_set <= (curmax + self.padding)), axis=1) == 3
+ if sum(mask) / float(len(mask)) < 0.01:
+ continue
+ vidx = np.ceil((cur_point_set[mask, :] - curmin) / (curmax - curmin) * [31.0, 31.0, 62.0])
+ vidx = np.unique(vidx[:, 0] * 31.0 * 62.0 + vidx[:, 1] * 62.0 + vidx[:, 2])
+ isvalid = len(vidx) / 31.0 / 31.0 / 62.0 >= 0.02
+ if isvalid:
+ break
+ choice = np.random.choice(len(cur_semantic_seg), self.npoints, replace=True)
+ point_set = cur_point_full[choice, :]
+ semantic_seg = cur_semantic_seg[choice]
+ mask = mask[choice]
+ sample_weight = self.labelweights[semantic_seg]
+ sample_weight *= mask
+ return point_set, semantic_seg, sample_weight
+
+ def __len__(self):
+ return len(self.scene_points_list)
+
+
+class S3DISDatasetWholeScene():
+ def __init__(self, root, block_points=8192, split='val', test_area=5, with_rgb = True, use_weight = True, block_size=1.5, stride=1.5, padding=0.001):
+ self.npoints = block_points
+ self.block_size = block_size
+ self.padding = padding
+ self.stride = stride
+ self.root = root
+ self.with_rgb = with_rgb
+ self.split = split
+ assert split in ['train', 'test']
+ if self.split == 'train':
+ self.file_list = [d for d in os.listdir(root) if d.find('Area_%d' % test_area) is -1]
+ else:
+ self.file_list = [d for d in os.listdir(root) if d.find('Area_%d' % test_area) is not -1]
+ self.scene_points_list = []
+ self.semantic_labels_list = []
+ for file in self.file_list:
+ data = np.load(root + file)
+ self.scene_points_list.append(data[:, :6])
+ self.semantic_labels_list.append(data[:, 6])
+ assert len(self.scene_points_list) == len(self.semantic_labels_list)
+ print('Number of scene: ', len(self.scene_points_list))
+ if split == 'train' and use_weight:
+ labelweights = np.zeros(13)
+ for seg in self.semantic_labels_list:
+ tmp, _ = np.histogram(seg, range(14))
+ labelweights += tmp
+ labelweights = labelweights.astype(np.float32)
+ labelweights = labelweights / np.sum(labelweights)
+ self.labelweights = np.power(np.amax(labelweights) / labelweights, 1 / 3.0)
+ else:
+ self.labelweights = np.ones(13)
+
+ print(self.labelweights)
+
+ def __getitem__(self, index):
+ if self.with_rgb:
+ point_set_ini = self.scene_points_list[index]
+ point_set_ini[:, 3:] = 2 * point_set_ini[:, 3:] / 255.0 - 1
+ else:
+ point_set_ini = self.scene_points_list[index][:, 0:3]
+ semantic_seg_ini = self.semantic_labels_list[index].astype(np.int32)
+ coordmax = np.max(point_set_ini[:, 0:3],axis=0)
+ coordmin = np.min(point_set_ini[:, 0:3],axis=0)
+ nsubvolume_x = np.ceil((coordmax[0]-coordmin[0])/self.block_size).astype(np.int32)
+ nsubvolume_y = np.ceil((coordmax[1]-coordmin[1])/self.block_size).astype(np.int32)
+ point_sets = list()
+ semantic_segs = list()
+ sample_weights = list()
+ for i in range(nsubvolume_x):
+ for j in range(nsubvolume_y):
+ curmin = coordmin+[i*self.block_size,j*self.block_size,0]
+ curmax = coordmin+[(i+1)*self.block_size,(j+1)*self.block_size,coordmax[2]-coordmin[2]]
+ curchoice = np.sum((point_set_ini[:, 0:3]>=(curmin-0.2))*(point_set_ini[:, 0:3]<=(curmax+0.2)),axis=1)==3
+ cur_point_set = point_set_ini[curchoice,0:3]
+ cur_point_full = point_set_ini[curchoice,:]
+ cur_semantic_seg = semantic_seg_ini[curchoice]
+ if len(cur_semantic_seg)==0:
+ continue
+ mask = np.sum((cur_point_set >= (curmin - self.padding)) * (cur_point_set <= (curmax + self.padding)),
+ axis=1) == 3
+ choice = np.random.choice(len(cur_semantic_seg), self.npoints, replace=True)
+ point_set = cur_point_full[choice,:] # Nx3/6
+ semantic_seg = cur_semantic_seg[choice] # N
+ mask = mask[choice]
+
+ if sum(mask)/float(len(mask))<0.01:
+ continue
+
+ sample_weight = self.labelweights[semantic_seg]
+ sample_weight *= mask # N
+ point_sets.append(np.expand_dims(point_set,0)) # 1xNx3
+ semantic_segs.append(np.expand_dims(semantic_seg,0)) # 1xN
+ sample_weights.append(np.expand_dims(sample_weight,0)) # 1xN
+ point_sets = np.concatenate(tuple(point_sets),axis=0)
+ semantic_segs = np.concatenate(tuple(semantic_segs),axis=0)
+ sample_weights = np.concatenate(tuple(sample_weights),axis=0)
+ return point_sets, semantic_segs, sample_weights
def __len__(self):
- return len(self.data)
+ return len(self.scene_points_list)
+
+
+class ScannetDatasetWholeScene_evaluation():
+ # prepare to give prediction on each points
+ def __init__(self, root, block_points=8192, split='test', test_area=5, with_rgb = True, use_weight = True, stride=0.5, block_size=1.5, padding=0.001):
+ self.block_points = block_points
+ self.block_size = block_size
+ self.padding = padding
+ self.root = root
+ self.with_rgb = with_rgb
+ self.split = split
+ self.stride = stride
+ self.scene_points_num = []
+ assert split in ['train', 'test']
+ if self.split == 'train':
+ self.file_list = [d for d in os.listdir(root) if d.find('Area_%d' % test_area) is -1]
+ else:
+ self.file_list = [d for d in os.listdir(root) if d.find('Area_%d' % test_area) is not -1]
+ self.scene_points_list = []
+ self.semantic_labels_list = []
+ for file in self.file_list:
+ data = np.load(root + file)
+ self.scene_points_list.append(data[:, :6])
+ self.semantic_labels_list.append(data[:, 6])
+ assert len(self.scene_points_list) == len(self.semantic_labels_list)
+ print('Number of scene: ', len(self.scene_points_list))
+ if split == 'train' and use_weight:
+ labelweights = np.zeros(13)
+ for seg in self.semantic_labels_list:
+ tmp, _ = np.histogram(seg, range(14))
+ self.scene_points_num.append(seg.shape[0])
+ labelweights += tmp
+ labelweights = labelweights.astype(np.float32)
+ labelweights = labelweights / np.sum(labelweights)
+ self.labelweights = np.power(np.amax(labelweights) / labelweights, 1 / 3.0)
+ else:
+ self.labelweights = np.ones(13)
+ for seg in self.semantic_labels_list:
+ self.scene_points_num.append(seg.shape[0])
+
+ print(self.labelweights)
+
+ def chunks(self, l, n):
+ """Yield successive n-sized chunks from l."""
+ for i in range(0, len(l), n):
+ yield l[i:i + n]
+
+ def split_data(self, data, idx):
+ new_data = []
+ for i in range(len(idx)):
+ new_data += [np.expand_dims(data[idx[i]], axis=0)]
+ return new_data
+
+ def nearest_dist(self, block_center, block_center_list):
+ num_blocks = len(block_center_list)
+ dist = np.zeros(num_blocks)
+ for i in range(num_blocks):
+ dist[i] = np.linalg.norm(block_center_list[i] - block_center, ord=2) # i->j
+ return np.argsort(dist)[0]
def __getitem__(self, index):
- return self.data[index], self.labels[index]
+ delta = self.stride
+ if self.with_rgb:
+ point_set_ini = self.scene_points_list[index]
+ point_set_ini[:, 3:] = 2 * point_set_ini[:, 3:] / 255.0 - 1
+ else:
+ point_set_ini = self.scene_points_list[index][:, 0:3]
+ semantic_seg_ini = self.semantic_labels_list[index].astype(np.int32)
+ coordmax = np.max(point_set_ini[:, 0:3], axis=0)
+ coordmin = np.min(point_set_ini[:, 0:3], axis=0)
+ nsubvolume_x = np.ceil((coordmax[0] - coordmin[0]) / delta).astype(np.int32)
+ nsubvolume_y = np.ceil((coordmax[1] - coordmin[1]) / delta).astype(np.int32)
+ point_sets = []
+ semantic_segs = []
+ sample_weights = []
+ point_idxs = []
+ block_center = []
+ for i in range(nsubvolume_x):
+ for j in range(nsubvolume_y):
+ curmin = coordmin + [i * delta, j * delta, 0]
+ curmax = curmin + [self.block_size, self.block_size, coordmax[2] - coordmin[2]]
+ curchoice = np.sum(
+ (point_set_ini[:, 0:3] >= (curmin - 0.2)) * (point_set_ini[:, 0:3] <= (curmax + 0.2)), axis=1) == 3
+ curchoice_idx = np.where(curchoice)[0]
+ cur_point_set = point_set_ini[curchoice, :]
+ cur_semantic_seg = semantic_seg_ini[curchoice]
+ if len(cur_semantic_seg) == 0:
+ continue
+ mask = np.sum((cur_point_set[:, 0:3] >= (curmin - self.padding)) * (cur_point_set[:, 0:3] <= (curmax + self.padding)),
+ axis=1) == 3
+ sample_weight = self.labelweights[cur_semantic_seg]
+ sample_weight *= mask # N
+ point_sets.append(cur_point_set) # 1xNx3/6
+ semantic_segs.append(cur_semantic_seg) # 1xN
+ sample_weights.append(sample_weight) # 1xN
+ point_idxs.append(curchoice_idx) # 1xN
+ block_center.append((curmin[0:2] + curmax[0:2]) / 2.0)
+
+ # merge small blocks
+ num_blocks = len(point_sets)
+ block_idx = 0
+ while block_idx < num_blocks:
+ if point_sets[block_idx].shape[0] > self.block_points/2:
+ block_idx += 1
+ continue
+
+ small_block_data = point_sets[block_idx].copy()
+ small_block_seg = semantic_segs[block_idx].copy()
+ small_block_smpw = sample_weights[block_idx].copy()
+ small_block_idxs = point_idxs[block_idx].copy()
+ small_block_center = block_center[block_idx].copy()
+ point_sets.pop(block_idx)
+ semantic_segs.pop(block_idx)
+ sample_weights.pop(block_idx)
+ point_idxs.pop(block_idx)
+ block_center.pop(block_idx)
+ nearest_block_idx = self.nearest_dist(small_block_center, block_center)
+ point_sets[nearest_block_idx] = np.concatenate((point_sets[nearest_block_idx], small_block_data), axis=0)
+ semantic_segs[nearest_block_idx] = np.concatenate((semantic_segs[nearest_block_idx], small_block_seg),
+ axis=0)
+ sample_weights[nearest_block_idx] = np.concatenate((sample_weights[nearest_block_idx], small_block_smpw),
+ axis=0)
+ point_idxs[nearest_block_idx] = np.concatenate((point_idxs[nearest_block_idx], small_block_idxs), axis=0)
+ num_blocks = len(point_sets)
+
+ # divide large blocks
+ num_blocks = len(point_sets)
+ div_blocks = []
+ div_blocks_seg = []
+ div_blocks_smpw = []
+ div_blocks_idxs = []
+ div_blocks_center = []
+ for block_idx in range(num_blocks):
+ cur_num_pts = point_sets[block_idx].shape[0]
+
+ point_idx_block = np.array([x for x in range(cur_num_pts)])
+ if point_idx_block.shape[0] % self.block_points != 0:
+ makeup_num = self.block_points - point_idx_block.shape[0] % self.block_points
+ np.random.shuffle(point_idx_block)
+ point_idx_block = np.concatenate((point_idx_block, point_idx_block[0:makeup_num].copy()))
+
+ np.random.shuffle(point_idx_block)
+
+ sub_blocks = list(self.chunks(point_idx_block, self.block_points))
+
+ div_blocks += self.split_data(point_sets[block_idx], sub_blocks)
+ div_blocks_seg += self.split_data(semantic_segs[block_idx], sub_blocks)
+ div_blocks_smpw += self.split_data(sample_weights[block_idx], sub_blocks)
+ div_blocks_idxs += self.split_data(point_idxs[block_idx], sub_blocks)
+ div_blocks_center += [block_center[block_idx].copy() for i in range(len(sub_blocks))]
+ div_blocks = np.concatenate(tuple(div_blocks), axis=0)
+ div_blocks_seg = np.concatenate(tuple(div_blocks_seg), axis=0)
+ div_blocks_smpw = np.concatenate(tuple(div_blocks_smpw), axis=0)
+ div_blocks_idxs = np.concatenate(tuple(div_blocks_idxs), axis=0)
+ return div_blocks, div_blocks_seg, div_blocks_smpw, div_blocks_idxs
+
+ def __len__(self):
+ return len(self.scene_points_list)
+
+if __name__ == '__main__':
+ data = S3DISDatasetWholeScene(split='test')
+ for i in range(10):
+ point_set, semantic_seg, sample_weight= data[i]
+ print(point_set.shape)
+ print(semantic_seg.shape)
+ print(sample_weight.shape)
+
diff --git a/data_utils/ShapeNetDataLoader.py b/data_utils/ShapeNetDataLoader.py
index c11bb3ef8..d7457c4ec 100644
--- a/data_utils/ShapeNetDataLoader.py
+++ b/data_utils/ShapeNetDataLoader.py
@@ -13,33 +13,24 @@ def pc_normalize(pc):
pc = pc / m
return pc
-def jitter_point_cloud(batch_data, sigma=0.01, clip=0.05):
- """ Randomly jitter points. jittering is per point.
- Input:
- BxNx3 array, original batch of point clouds
- Return:
- BxNx3 array, jittered batch of point clouds
- """
- N, C = batch_data.shape
- assert(clip > 0)
- jittered_data = np.clip(sigma * np.random.randn(N, C), -1*clip, clip)
- jittered_data += batch_data
- return jittered_data
-
class PartNormalDataset(Dataset):
- def __init__(self, npoints=2500, split='train', normalize=True, jitter=False):
+ def __init__(self,root = './data/shapenetcore_partanno_segmentation_benchmark_v0_normal', npoints=2500, split='train', class_choice=None, normal_channel=False):
self.npoints = npoints
- self.root = './data/shapenetcore_partanno_segmentation_benchmark_v0_normal'
+ self.root = root
self.catfile = os.path.join(self.root, 'synsetoffset2category.txt')
self.cat = {}
- self.normalize = normalize
- self.jitter = jitter
+ self.normal_channel = normal_channel
+
with open(self.catfile, 'r') as f:
for line in f:
ls = line.strip().split()
self.cat[ls[0]] = ls[1]
self.cat = {k: v for k, v in self.cat.items()}
+ self.classes_original = dict(zip(self.cat, range(len(self.cat))))
+
+ if not class_choice is None:
+ self.cat = {k:v for k,v in self.cat.items() if k in class_choice}
# print(self.cat)
self.meta = {}
@@ -77,7 +68,10 @@ def __init__(self, npoints=2500, split='train', normalize=True, jitter=False):
for fn in self.meta[item]:
self.datapath.append((item, fn))
- self.classes = dict(zip(self.cat, range(len(self.cat))))
+ self.classes = {}
+ for i in self.cat.keys():
+ self.classes[i] = self.classes_original[i]
+
# Mapping from category ('Chair') to a list of int [10,11,12,13] as segmentation labels
self.seg_classes = {'Earphone': [16, 17, 18], 'Motorbike': [30, 31, 32, 33, 34, 35], 'Rocket': [41, 42, 43],
'Car': [8, 9, 10, 11], 'Laptop': [28, 29], 'Cap': [6, 7], 'Skateboard': [44, 45, 46],
@@ -85,40 +79,40 @@ def __init__(self, npoints=2500, split='train', normalize=True, jitter=False):
'Table': [47, 48, 49], 'Airplane': [0, 1, 2, 3], 'Pistol': [38, 39, 40],
'Chair': [12, 13, 14, 15], 'Knife': [22, 23]}
- for cat in sorted(self.seg_classes.keys()):
- print(cat, self.seg_classes[cat])
+ # for cat in sorted(self.seg_classes.keys()):
+ # print(cat, self.seg_classes[cat])
self.cache = {} # from index to (point_set, cls, seg) tuple
self.cache_size = 20000
+
def __getitem__(self, index):
if index in self.cache:
- point_set, normal, seg, cls = self.cache[index]
+ ppoint_set, cls, seg = self.cache[index]
else:
fn = self.datapath[index]
cat = self.datapath[index][0]
cls = self.classes[cat]
cls = np.array([cls]).astype(np.int32)
data = np.loadtxt(fn[1]).astype(np.float32)
- point_set = data[:, 0:3]
- normal = data[:, 3:6]
+ if not self.normal_channel:
+ point_set = data[:, 0:3]
+ else:
+ point_set = data[:, 0:6]
seg = data[:, -1].astype(np.int32)
if len(self.cache) < self.cache_size:
- self.cache[index] = (point_set, normal, seg, cls)
- if self.normalize:
- point_set = pc_normalize(point_set)
- if self.jitter:
- jitter_point_cloud(point_set)
+ self.cache[index] = (point_set, cls, seg)
+ point_set[:, 0:3] = pc_normalize(point_set[:, 0:3])
+
choice = np.random.choice(len(seg), self.npoints, replace=True)
# resample
point_set = point_set[choice, :]
seg = seg[choice]
- normal = normal[choice, :]
+ return point_set, cls, seg
+ def __len__(self):
+ return len(self.datapath)
- return point_set,cls, seg, normal
- def __len__(self):
- return len(self.datapath)
\ No newline at end of file
diff --git a/model/pointnet.py b/model/pointnet.py
deleted file mode 100644
index 6f13dff51..000000000
--- a/model/pointnet.py
+++ /dev/null
@@ -1,284 +0,0 @@
-import torch
-import torch.nn as nn
-import torch.nn.parallel
-import torch.utils.data
-from torch.autograd import Variable
-import numpy as np
-import torch.nn.functional as F
-
-
-class STN3d(nn.Module):
- def __init__(self):
- super(STN3d, self).__init__()
- self.conv1 = torch.nn.Conv1d(3, 64, 1)
- self.conv2 = torch.nn.Conv1d(64, 128, 1)
- self.conv3 = torch.nn.Conv1d(128, 1024, 1)
- self.fc1 = nn.Linear(1024, 512)
- self.fc2 = nn.Linear(512, 256)
- self.fc3 = nn.Linear(256, 9)
- self.relu = nn.ReLU()
-
- self.bn1 = nn.BatchNorm1d(64)
- self.bn2 = nn.BatchNorm1d(128)
- self.bn3 = nn.BatchNorm1d(1024)
- self.bn4 = nn.BatchNorm1d(512)
- self.bn5 = nn.BatchNorm1d(256)
-
- def forward(self, x):
- batchsize = x.size()[0]
- x = F.relu(self.bn1(self.conv1(x)))
- x = F.relu(self.bn2(self.conv2(x)))
- x = F.relu(self.bn3(self.conv3(x)))
- x = torch.max(x, 2, keepdim=True)[0]
- x = x.view(-1, 1024)
-
- x = F.relu(self.bn4(self.fc1(x)))
- x = F.relu(self.bn5(self.fc2(x)))
- x = self.fc3(x)
-
- iden = Variable(torch.from_numpy(np.array([1, 0, 0, 0, 1, 0, 0, 0, 1]).astype(np.float32))).view(1, 9).repeat(
- batchsize, 1)
- if x.is_cuda:
- iden = iden.cuda()
- x = x + iden
- x = x.view(-1, 3, 3)
- return x
-
-
-class STNkd(nn.Module):
- def __init__(self, k=64):
- super(STNkd, self).__init__()
- self.conv1 = torch.nn.Conv1d(k, 64, 1)
- self.conv2 = torch.nn.Conv1d(64, 128, 1)
- self.conv3 = torch.nn.Conv1d(128, 1024, 1)
- self.fc1 = nn.Linear(1024, 512)
- self.fc2 = nn.Linear(512, 256)
- self.fc3 = nn.Linear(256, k * k)
- self.relu = nn.ReLU()
-
- self.bn1 = nn.BatchNorm1d(64)
- self.bn2 = nn.BatchNorm1d(128)
- self.bn3 = nn.BatchNorm1d(1024)
- self.bn4 = nn.BatchNorm1d(512)
- self.bn5 = nn.BatchNorm1d(256)
-
- self.k = k
-
- def forward(self, x):
- batchsize = x.size()[0]
- x = F.relu(self.bn1(self.conv1(x)))
- x = F.relu(self.bn2(self.conv2(x)))
- x = F.relu(self.bn3(self.conv3(x)))
- x = torch.max(x, 2, keepdim=True)[0]
- x = x.view(-1, 1024)
-
- x = F.relu(self.bn4(self.fc1(x)))
- x = F.relu(self.bn5(self.fc2(x)))
- x = self.fc3(x)
-
- iden = Variable(torch.from_numpy(np.eye(self.k).flatten().astype(np.float32))).view(1, self.k * self.k).repeat(
- batchsize, 1)
- if x.is_cuda:
- iden = iden.cuda()
- x = x + iden
- x = x.view(-1, self.k, self.k)
- return x
-
-
-class PointNetEncoder(nn.Module):
- def __init__(self, global_feat=True, feature_transform=False, semseg = False):
- super(PointNetEncoder, self).__init__()
- self.stn = STN3d() if not semseg else STNkd(k=9)
- self.conv1 = torch.nn.Conv1d(3, 64, 1) if not semseg else torch.nn.Conv1d(9, 64, 1)
- self.conv2 = torch.nn.Conv1d(64, 128, 1)
- self.conv3 = torch.nn.Conv1d(128, 1024, 1)
- self.bn1 = nn.BatchNorm1d(64)
- self.bn2 = nn.BatchNorm1d(128)
- self.bn3 = nn.BatchNorm1d(1024)
- self.global_feat = global_feat
- self.feature_transform = feature_transform
- if self.feature_transform:
- self.fstn = STNkd(k=64)
-
- def forward(self, x):
- n_pts = x.size()[2]
- trans = self.stn(x)
- x = x.transpose(2, 1)
- x = torch.bmm(x, trans)
- x = x.transpose(2, 1)
- x = F.relu(self.bn1(self.conv1(x)))
-
- if self.feature_transform:
- trans_feat = self.fstn(x)
- x = x.transpose(2, 1)
- x = torch.bmm(x, trans_feat)
- x = x.transpose(2, 1)
- else:
- trans_feat = None
-
- pointfeat = x
- x = F.relu(self.bn2(self.conv2(x)))
- x = self.bn3(self.conv3(x))
- x = torch.max(x, 2, keepdim=True)[0]
- x = x.view(-1, 1024)
- if self.global_feat:
- return x, trans, trans_feat
- else:
- x = x.view(-1, 1024, 1).repeat(1, 1, n_pts)
- return torch.cat([x, pointfeat], 1), trans, trans_feat
-
-
-class PointNetCls(nn.Module):
- def __init__(self, k=2, feature_transform=False):
- super(PointNetCls, self).__init__()
- self.feature_transform = feature_transform
- self.feat = PointNetEncoder(global_feat=True, feature_transform=feature_transform)
- self.fc1 = nn.Linear(1024, 512)
- self.fc2 = nn.Linear(512, 256)
- self.fc3 = nn.Linear(256, k)
- self.dropout = nn.Dropout(p=0.3)
- self.bn1 = nn.BatchNorm1d(512)
- self.bn2 = nn.BatchNorm1d(256)
- self.relu = nn.ReLU()
-
- def forward(self, x):
- x, trans, trans_feat = self.feat(x)
- x = F.relu(self.bn1(self.fc1(x)))
- x = F.relu(self.bn2(self.dropout(self.fc2(x))))
- x = self.fc3(x)
- return F.log_softmax(x, dim=1), trans_feat
-
-
-class PointNetDenseCls(nn.Module):
- def __init__(self, cat_num=16,part_num=50):
- super(PointNetDenseCls, self).__init__()
- self.cat_num = cat_num
- self.part_num = part_num
- self.stn = STN3d()
- self.conv1 = torch.nn.Conv1d(3, 64, 1)
- self.conv2 = torch.nn.Conv1d(64, 128, 1)
- self.conv3 = torch.nn.Conv1d(128, 128, 1)
- self.conv4 = torch.nn.Conv1d(128, 512, 1)
- self.conv5 = torch.nn.Conv1d(512, 2048, 1)
- self.bn1 = nn.BatchNorm1d(64)
- self.bn2 = nn.BatchNorm1d(128)
- self.bn3 = nn.BatchNorm1d(128)
- self.bn4 = nn.BatchNorm1d(512)
- self.bn5 = nn.BatchNorm1d(2048)
- self.fstn = STNkd(k=128)
- # classification network
- self.fc1 = nn.Linear(2048, 256)
- self.fc2 = nn.Linear(256, 256)
- self.fc3 = nn.Linear(256, cat_num)
- self.dropout = nn.Dropout(p=0.3)
- self.bnc1 = nn.BatchNorm1d(256)
- self.bnc2 = nn.BatchNorm1d(256)
- # segmentation network
- self.convs1 = torch.nn.Conv1d(4944, 256, 1)
- self.convs2 = torch.nn.Conv1d(256, 256, 1)
- self.convs3 = torch.nn.Conv1d(256, 128, 1)
- self.convs4 = torch.nn.Conv1d(128, part_num, 1)
- self.bns1 = nn.BatchNorm1d(256)
- self.bns2 = nn.BatchNorm1d(256)
- self.bns3 = nn.BatchNorm1d(128)
-
- def forward(self, point_cloud,label):
- batchsize,_ , n_pts = point_cloud.size()
- # point_cloud_transformed
- trans = self.stn(point_cloud)
- point_cloud = point_cloud.transpose(2, 1)
- point_cloud_transformed = torch.bmm(point_cloud, trans)
- point_cloud_transformed = point_cloud_transformed.transpose(2, 1)
- # MLP
- out1 = F.relu(self.bn1(self.conv1(point_cloud_transformed)))
- out2 = F.relu(self.bn2(self.conv2(out1)))
- out3 = F.relu(self.bn3(self.conv3(out2)))
- # net_transformed
- trans_feat = self.fstn(out3)
- x = out3.transpose(2, 1)
- net_transformed = torch.bmm(x, trans_feat)
- net_transformed = net_transformed.transpose(2, 1)
- # MLP
- out4 = F.relu(self.bn4(self.conv4(net_transformed)))
- out5 = self.bn5(self.conv5(out4))
- out_max = torch.max(out5, 2, keepdim=True)[0]
- out_max = out_max.view(-1, 2048)
- # classification network
- net = F.relu(self.bnc1(self.fc1(out_max)))
- net = F.relu(self.bnc2(self.dropout(self.fc2(net))))
- net = self.fc3(net) # [B,16]
- # segmentation network
- out_max = torch.cat([out_max,label],1)
- expand = out_max.view(-1, 2048+16, 1).repeat(1, 1, n_pts)
- concat = torch.cat([expand, out1, out2, out3, out4, out5], 1)
- net2 = F.relu(self.bns1(self.convs1(concat)))
- net2 = F.relu(self.bns2(self.convs2(net2)))
- net2 = F.relu(self.bns3(self.convs3(net2)))
- net2 = self.convs4(net2)
- net2 = net2.transpose(2, 1).contiguous()
- net2 = F.log_softmax(net2.view(-1, self.part_num), dim=-1)
- net2 = net2.view(batchsize, n_pts, self.part_num) # [B, N 50]
-
- return net, net2, trans_feat
-
-
-def feature_transform_reguliarzer(trans):
- d = trans.size()[1]
- I = torch.eye(d)[None, :, :]
- if trans.is_cuda:
- I = I.cuda()
- loss = torch.mean(torch.norm(torch.bmm(trans, trans.transpose(2, 1) - I), dim=(1, 2)))
- return loss
-
-class PointNetLoss(torch.nn.Module):
- def __init__(self, weight=1,mat_diff_loss_scale=0.001):
- super(PointNetLoss, self).__init__()
- self.mat_diff_loss_scale = mat_diff_loss_scale
- self.weight = weight
-
- def forward(self, labels_pred, label, seg_pred,seg, trans_feat):
- seg_loss = F.nll_loss(seg_pred, seg)
- mat_diff_loss = feature_transform_reguliarzer(trans_feat)
- label_loss = F.nll_loss(labels_pred, label)
-
- loss = self.weight * seg_loss + (1-self.weight) * label_loss + mat_diff_loss * self.mat_diff_loss_scale
- return loss, seg_loss, label_loss
-
-
-class PointNetSeg(nn.Module):
- def __init__(self,num_class,feature_transform=False, semseg = False):
- super(PointNetSeg, self).__init__()
- self.k = num_class
- self.feat = PointNetEncoder(global_feat=False,feature_transform=feature_transform, semseg = semseg)
- self.conv1 = torch.nn.Conv1d(1088, 512, 1)
- self.conv2 = torch.nn.Conv1d(512, 256, 1)
- self.conv3 = torch.nn.Conv1d(256, 128, 1)
- self.conv4 = torch.nn.Conv1d(128, self.k, 1)
- self.bn1 = nn.BatchNorm1d(512)
- self.bn1_1 = nn.BatchNorm1d(1024)
- self.bn2 = nn.BatchNorm1d(256)
- self.bn3 = nn.BatchNorm1d(128)
-
- def forward(self, x):
- batchsize = x.size()[0]
- n_pts = x.size()[2]
- x, trans, trans_feat = self.feat(x)
- x = F.relu(self.bn1(self.conv1(x)))
- x = F.relu(self.bn2(self.conv2(x)))
- x = F.relu(self.bn3(self.conv3(x)))
- x = self.conv4(x)
- x = x.transpose(2,1).contiguous()
- x = F.log_softmax(x.view(-1,self.k), dim=-1)
- x = x.view(batchsize, n_pts, self.k)
- return x, trans_feat
-
-
-
-if __name__ == '__main__':
- point = torch.randn(8,3,1024)
- label = torch.randn(8,16)
- model = PointNetDenseCls()
- net, net2, trans_feat = model(point,label)
- print('net',net.shape)
- print('net2',net2.shape)
- print('trans_feat',trans_feat.shape)
diff --git a/models/pointnet.py b/models/pointnet.py
new file mode 100644
index 000000000..c5c36f786
--- /dev/null
+++ b/models/pointnet.py
@@ -0,0 +1,141 @@
+import torch
+import torch.nn as nn
+import torch.nn.parallel
+import torch.utils.data
+from torch.autograd import Variable
+import numpy as np
+import torch.nn.functional as F
+
+
+class STN3d(nn.Module):
+ def __init__(self, channel):
+ super(STN3d, self).__init__()
+ self.conv1 = torch.nn.Conv1d(channel, 64, 1)
+ self.conv2 = torch.nn.Conv1d(64, 128, 1)
+ self.conv3 = torch.nn.Conv1d(128, 1024, 1)
+ self.fc1 = nn.Linear(1024, 512)
+ self.fc2 = nn.Linear(512, 256)
+ self.fc3 = nn.Linear(256, 9)
+ self.relu = nn.ReLU()
+
+ self.bn1 = nn.BatchNorm1d(64)
+ self.bn2 = nn.BatchNorm1d(128)
+ self.bn3 = nn.BatchNorm1d(1024)
+ self.bn4 = nn.BatchNorm1d(512)
+ self.bn5 = nn.BatchNorm1d(256)
+
+ def forward(self, x):
+ batchsize = x.size()[0]
+ x = F.relu(self.bn1(self.conv1(x)))
+ x = F.relu(self.bn2(self.conv2(x)))
+ x = F.relu(self.bn3(self.conv3(x)))
+ x = torch.max(x, 2, keepdim=True)[0]
+ x = x.view(-1, 1024)
+
+ x = F.relu(self.bn4(self.fc1(x)))
+ x = F.relu(self.bn5(self.fc2(x)))
+ x = self.fc3(x)
+
+ iden = Variable(torch.from_numpy(np.array([1, 0, 0, 0, 1, 0, 0, 0, 1]).astype(np.float32))).view(1, 9).repeat(
+ batchsize, 1)
+ if x.is_cuda:
+ iden = iden.cuda()
+ x = x + iden
+ x = x.view(-1, 3, 3)
+ return x
+
+
+class STNkd(nn.Module):
+ def __init__(self, k=64):
+ super(STNkd, self).__init__()
+ self.conv1 = torch.nn.Conv1d(k, 64, 1)
+ self.conv2 = torch.nn.Conv1d(64, 128, 1)
+ self.conv3 = torch.nn.Conv1d(128, 1024, 1)
+ self.fc1 = nn.Linear(1024, 512)
+ self.fc2 = nn.Linear(512, 256)
+ self.fc3 = nn.Linear(256, k * k)
+ self.relu = nn.ReLU()
+
+ self.bn1 = nn.BatchNorm1d(64)
+ self.bn2 = nn.BatchNorm1d(128)
+ self.bn3 = nn.BatchNorm1d(1024)
+ self.bn4 = nn.BatchNorm1d(512)
+ self.bn5 = nn.BatchNorm1d(256)
+
+ self.k = k
+
+ def forward(self, x):
+ batchsize = x.size()[0]
+ x = F.relu(self.bn1(self.conv1(x)))
+ x = F.relu(self.bn2(self.conv2(x)))
+ x = F.relu(self.bn3(self.conv3(x)))
+ x = torch.max(x, 2, keepdim=True)[0]
+ x = x.view(-1, 1024)
+
+ x = F.relu(self.bn4(self.fc1(x)))
+ x = F.relu(self.bn5(self.fc2(x)))
+ x = self.fc3(x)
+
+ iden = Variable(torch.from_numpy(np.eye(self.k).flatten().astype(np.float32))).view(1, self.k * self.k).repeat(
+ batchsize, 1)
+ if x.is_cuda:
+ iden = iden.cuda()
+ x = x + iden
+ x = x.view(-1, self.k, self.k)
+ return x
+
+
+class PointNetEncoder(nn.Module):
+ def __init__(self, global_feat=True, feature_transform=False, channel=3):
+ super(PointNetEncoder, self).__init__()
+ self.stn = STN3d(channel)
+ self.conv1 = torch.nn.Conv1d(channel, 64, 1)
+ self.conv2 = torch.nn.Conv1d(64, 128, 1)
+ self.conv3 = torch.nn.Conv1d(128, 1024, 1)
+ self.bn1 = nn.BatchNorm1d(64)
+ self.bn2 = nn.BatchNorm1d(128)
+ self.bn3 = nn.BatchNorm1d(1024)
+ self.global_feat = global_feat
+ self.feature_transform = feature_transform
+ if self.feature_transform:
+ self.fstn = STNkd(k=64)
+
+ def forward(self, x):
+ B, D, N = x.size()
+ trans = self.stn(x)
+ x = x.transpose(2, 1)
+ if D >3 :
+ x, feature = x.split(3,dim=2)
+ x = torch.bmm(x, trans)
+ if D > 3:
+ x = torch.cat([x,feature],dim=2)
+ x = x.transpose(2, 1)
+ x = F.relu(self.bn1(self.conv1(x)))
+
+ if self.feature_transform:
+ trans_feat = self.fstn(x)
+ x = x.transpose(2, 1)
+ x = torch.bmm(x, trans_feat)
+ x = x.transpose(2, 1)
+ else:
+ trans_feat = None
+
+ pointfeat = x
+ x = F.relu(self.bn2(self.conv2(x)))
+ x = self.bn3(self.conv3(x))
+ x = torch.max(x, 2, keepdim=True)[0]
+ x = x.view(-1, 1024)
+ if self.global_feat:
+ return x, trans, trans_feat
+ else:
+ x = x.view(-1, 1024, 1).repeat(1, 1, N)
+ return torch.cat([x, pointfeat], 1), trans, trans_feat
+
+
+def feature_transform_reguliarzer(trans):
+ d = trans.size()[1]
+ I = torch.eye(d)[None, :, :]
+ if trans.is_cuda:
+ I = I.cuda()
+ loss = torch.mean(torch.norm(torch.bmm(trans, trans.transpose(2, 1) - I), dim=(1, 2)))
+ return loss
diff --git a/model/pointnet2.py b/models/pointnet2.py
similarity index 95%
rename from model/pointnet2.py
rename to models/pointnet2.py
index 21f268cb2..9efcfec21 100644
--- a/model/pointnet2.py
+++ b/models/pointnet2.py
@@ -2,16 +2,14 @@
import torch
import numpy as np
import torch.nn.functional as F
-from model.pointnet_util import PointNetSetAbstractionMsg,PointNetSetAbstraction,PointNetFeaturePropagation
+from pointnet_util import PointNetSetAbstractionMsg, PointNetSetAbstraction,PointNetFeaturePropagation
class PointNet2ClsMsg(nn.Module):
def __init__(self):
super(PointNet2ClsMsg, self).__init__()
- self.sa1 = PointNetSetAbstractionMsg(512, [0.1, 0.2, 0.4], [16, 32, 128], 0,
- [[32, 32, 64], [64, 64, 128], [64, 96, 128]])
- self.sa2 = PointNetSetAbstractionMsg(128, [0.2, 0.4, 0.8], [32, 64, 128], 320,
- [[64, 64, 128], [128, 128, 256], [128, 128, 256]])
+ self.sa1 = PointNetSetAbstractionMsg(512, [0.1, 0.2, 0.4], [16, 32, 128], 0,[[32, 32, 64], [64, 64, 128], [64, 96, 128]])
+ self.sa2 = PointNetSetAbstractionMsg(128, [0.2, 0.4, 0.8], [32, 64, 128], 320,[[64, 64, 128], [128, 128, 256], [128, 128, 256]])
self.sa3 = PointNetSetAbstraction(None, None, None, 640 + 3, [256, 512, 1024], True)
self.fc1 = nn.Linear(1024, 512)
self.bn1 = nn.BatchNorm1d(512)
diff --git a/models/pointnet2_cls_msg.py b/models/pointnet2_cls_msg.py
new file mode 100644
index 000000000..d1032350d
--- /dev/null
+++ b/models/pointnet2_cls_msg.py
@@ -0,0 +1,51 @@
+import torch.nn as nn
+import torch.nn.functional as F
+from pointnet_util import PointNetSetAbstractionMsg, PointNetSetAbstraction
+
+
+class get_model(nn.Module):
+ def __init__(self,num_class,normal_channel=True):
+ super(get_model, self).__init__()
+ in_channel = 3 if normal_channel else 0
+ self.normal_channel = normal_channel
+ self.sa1 = PointNetSetAbstractionMsg(512, [0.1, 0.2, 0.4], [16, 32, 128], in_channel,[[32, 32, 64], [64, 64, 128], [64, 96, 128]])
+ self.sa2 = PointNetSetAbstractionMsg(128, [0.2, 0.4, 0.8], [32, 64, 128], 320,[[64, 64, 128], [128, 128, 256], [128, 128, 256]])
+ self.sa3 = PointNetSetAbstraction(None, None, None, 640 + 3, [256, 512, 1024], True)
+ self.fc1 = nn.Linear(1024, 512)
+ self.bn1 = nn.BatchNorm1d(512)
+ self.drop1 = nn.Dropout(0.4)
+ self.fc2 = nn.Linear(512, 256)
+ self.bn2 = nn.BatchNorm1d(256)
+ self.drop2 = nn.Dropout(0.5)
+ self.fc3 = nn.Linear(256, num_class)
+
+ def forward(self, xyz):
+ B, _, _ = xyz.shape
+ if self.normal_channel:
+ norm = xyz[:, 3:, :]
+ xyz = xyz[:, :3, :]
+ else:
+ norm = None
+ l1_xyz, l1_points = self.sa1(xyz, norm)
+ l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)
+ l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)
+ x = l3_points.view(B, 1024)
+ x = self.drop1(F.relu(self.bn1(self.fc1(x))))
+ x = self.drop2(F.relu(self.bn2(self.fc2(x))))
+ x = self.fc3(x)
+ x = F.log_softmax(x, -1)
+
+
+ return x,l3_points
+
+
+class get_loss(nn.Module):
+ def __init__(self):
+ super(get_loss, self).__init__()
+
+ def forward(self, pred, target, trans_feat):
+ total_loss = F.nll_loss(pred, target)
+
+ return total_loss
+
+
diff --git a/models/pointnet2_cls_ssg.py b/models/pointnet2_cls_ssg.py
new file mode 100644
index 000000000..f344312ea
--- /dev/null
+++ b/models/pointnet2_cls_ssg.py
@@ -0,0 +1,50 @@
+import torch.nn as nn
+import torch.nn.functional as F
+from pointnet_util import PointNetSetAbstraction
+
+
+class get_model(nn.Module):
+ def __init__(self,num_class,normal_channel=True):
+ super(get_model, self).__init__()
+ in_channel = 6 if normal_channel else 3
+ self.normal_channel = normal_channel
+ self.sa1 = PointNetSetAbstraction(npoint=512, radius=0.2, nsample=32, in_channel=in_channel, mlp=[64, 64, 128], group_all=False)
+ self.sa2 = PointNetSetAbstraction(npoint=128, radius=0.4, nsample=64, in_channel=128 + 3, mlp=[128, 128, 256], group_all=False)
+ self.sa3 = PointNetSetAbstraction(npoint=None, radius=None, nsample=None, in_channel=256 + 3, mlp=[256, 512, 1024], group_all=True)
+ self.fc1 = nn.Linear(1024, 512)
+ self.bn1 = nn.BatchNorm1d(512)
+ self.drop1 = nn.Dropout(0.4)
+ self.fc2 = nn.Linear(512, 256)
+ self.bn2 = nn.BatchNorm1d(256)
+ self.drop2 = nn.Dropout(0.4)
+ self.fc3 = nn.Linear(256, num_class)
+
+ def forward(self, xyz):
+ B, _, _ = xyz.shape
+ if self.normal_channel:
+ norm = xyz[:, 3:, :]
+ xyz = xyz[:, :3, :]
+ else:
+ norm = None
+ l1_xyz, l1_points = self.sa1(xyz, norm)
+ l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)
+ l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)
+ x = l3_points.view(B, 1024)
+ x = self.drop1(F.relu(self.bn1(self.fc1(x))))
+ x = self.drop2(F.relu(self.bn2(self.fc2(x))))
+ x = self.fc3(x)
+ x = F.log_softmax(x, -1)
+
+
+ return x, l3_points
+
+
+
+class get_loss(nn.Module):
+ def __init__(self):
+ super(get_loss, self).__init__()
+
+ def forward(self, pred, target, trans_feat):
+ total_loss = F.nll_loss(pred, target)
+
+ return total_loss
diff --git a/models/pointnet2_part_seg_msg.py b/models/pointnet2_part_seg_msg.py
new file mode 100644
index 000000000..15c229805
--- /dev/null
+++ b/models/pointnet2_part_seg_msg.py
@@ -0,0 +1,59 @@
+import torch.nn as nn
+import torch
+import torch.nn.functional as F
+from models.pointnet_util import PointNetSetAbstractionMsg,PointNetSetAbstraction,PointNetFeaturePropagation
+
+
+class get_model(nn.Module):
+ def __init__(self, num_classes, normal_channel=False):
+ super(get_model, self).__init__()
+ if normal_channel:
+ additional_channel = 3
+ else:
+ additional_channel = 0
+ self.normal_channel = normal_channel
+ self.sa1 = PointNetSetAbstractionMsg(512, [0.1, 0.2, 0.4], [32, 64, 128], 3+additional_channel, [[32, 32, 64], [64, 64, 128], [64, 96, 128]])
+ self.sa2 = PointNetSetAbstractionMsg(128, [0.4,0.8], [64, 128], 128+128+64, [[128, 128, 256], [128, 196, 256]])
+ self.sa3 = PointNetSetAbstraction(npoint=None, radius=None, nsample=None, in_channel=512 + 3, mlp=[256, 512, 1024], group_all=True)
+ self.fp3 = PointNetFeaturePropagation(in_channel=1536, mlp=[256, 256])
+ self.fp2 = PointNetFeaturePropagation(in_channel=576, mlp=[256, 128])
+ self.fp1 = PointNetFeaturePropagation(in_channel=150+additional_channel, mlp=[128, 128])
+ self.conv1 = nn.Conv1d(128, 128, 1)
+ self.bn1 = nn.BatchNorm1d(128)
+ self.drop1 = nn.Dropout(0.5)
+ self.conv2 = nn.Conv1d(128, num_classes, 1)
+
+ def forward(self, xyz, cls_label):
+ # Set Abstraction layers
+ B,C,N = xyz.shape
+ if self.normal_channel:
+ l0_points = xyz
+ l0_xyz = xyz[:,:3,:]
+ else:
+ l0_points = xyz
+ l0_xyz = xyz
+ l1_xyz, l1_points = self.sa1(l0_xyz, l0_points)
+ l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)
+ l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)
+ # Feature Propagation layers
+ l2_points = self.fp3(l2_xyz, l3_xyz, l2_points, l3_points)
+ l1_points = self.fp2(l1_xyz, l2_xyz, l1_points, l2_points)
+ cls_label_one_hot = cls_label.view(B,16,1).repeat(1,1,N)
+ l0_points = self.fp1(l0_xyz, l1_xyz, torch.cat([cls_label_one_hot,l0_xyz,l0_points],1), l1_points)
+ # FC layers
+ feat = F.relu(self.bn1(self.conv1(l0_points)))
+ x = self.drop1(feat)
+ x = self.conv2(x)
+ x = F.log_softmax(x, dim=1)
+ x = x.permute(0, 2, 1)
+ return x, l3_points
+
+
+class get_loss(nn.Module):
+ def __init__(self):
+ super(get_loss, self).__init__()
+
+ def forward(self, pred, target, trans_feat):
+ total_loss = F.nll_loss(pred, target)
+
+ return total_loss
\ No newline at end of file
diff --git a/models/pointnet2_part_seg_ssg.py b/models/pointnet2_part_seg_ssg.py
new file mode 100644
index 000000000..ba7cb7707
--- /dev/null
+++ b/models/pointnet2_part_seg_ssg.py
@@ -0,0 +1,59 @@
+import torch.nn as nn
+import torch
+import torch.nn.functional as F
+from models.pointnet_util import PointNetSetAbstraction,PointNetFeaturePropagation
+
+
+class get_model(nn.Module):
+ def __init__(self, num_classes, normal_channel=False):
+ super(get_model, self).__init__()
+ if normal_channel:
+ additional_channel = 3
+ else:
+ additional_channel = 0
+ self.normal_channel = normal_channel
+ self.sa1 = PointNetSetAbstraction(npoint=512, radius=0.2, nsample=32, in_channel=6+additional_channel, mlp=[64, 64, 128], group_all=False)
+ self.sa2 = PointNetSetAbstraction(npoint=128, radius=0.4, nsample=64, in_channel=128 + 3, mlp=[128, 128, 256], group_all=False)
+ self.sa3 = PointNetSetAbstraction(npoint=None, radius=None, nsample=None, in_channel=256 + 3, mlp=[256, 512, 1024], group_all=True)
+ self.fp3 = PointNetFeaturePropagation(in_channel=1280, mlp=[256, 256])
+ self.fp2 = PointNetFeaturePropagation(in_channel=384, mlp=[256, 128])
+ self.fp1 = PointNetFeaturePropagation(in_channel=128+16+6+additional_channel, mlp=[128, 128, 128])
+ self.conv1 = nn.Conv1d(128, 128, 1)
+ self.bn1 = nn.BatchNorm1d(128)
+ self.drop1 = nn.Dropout(0.5)
+ self.conv2 = nn.Conv1d(128, num_classes, 1)
+
+ def forward(self, xyz, cls_label):
+ # Set Abstraction layers
+ B,C,N = xyz.shape
+ if self.normal_channel:
+ l0_points = xyz
+ l0_xyz = xyz[:,:3,:]
+ else:
+ l0_points = xyz
+ l0_xyz = xyz
+ l1_xyz, l1_points = self.sa1(l0_xyz, l0_points)
+ l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)
+ l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)
+ # Feature Propagation layers
+ l2_points = self.fp3(l2_xyz, l3_xyz, l2_points, l3_points)
+ l1_points = self.fp2(l1_xyz, l2_xyz, l1_points, l2_points)
+ cls_label_one_hot = cls_label.view(B,16,1).repeat(1,1,N)
+ l0_points = self.fp1(l0_xyz, l1_xyz, torch.cat([cls_label_one_hot,l0_xyz,l0_points],1), l1_points)
+ # FC layers
+ feat = F.relu(self.bn1(self.conv1(l0_points)))
+ x = self.drop1(feat)
+ x = self.conv2(x)
+ x = F.log_softmax(x, dim=1)
+ x = x.permute(0, 2, 1)
+ return x, l3_points
+
+
+class get_loss(nn.Module):
+ def __init__(self):
+ super(get_loss, self).__init__()
+
+ def forward(self, pred, target, trans_feat):
+ total_loss = F.nll_loss(pred, target)
+
+ return total_loss
\ No newline at end of file
diff --git a/models/pointnet2_sem_seg.py b/models/pointnet2_sem_seg.py
new file mode 100644
index 000000000..f4719c0e8
--- /dev/null
+++ b/models/pointnet2_sem_seg.py
@@ -0,0 +1,63 @@
+import torch.nn as nn
+import torch.nn.functional as F
+from models.pointnet_util import PointNetSetAbstraction,PointNetFeaturePropagation
+
+
+class get_model(nn.Module):
+ def __init__(self, num_classes, with_rgb=True):
+ super(get_model, self).__init__()
+ self.with_rgb = with_rgb
+ if with_rgb:
+ additional_channel = 3
+ else:
+ additional_channel = 0
+ self.sa1 = PointNetSetAbstraction(1024, 0.1, 32, 6 + additional_channel, [32, 32, 64], False)
+ self.sa2 = PointNetSetAbstraction(256, 0.2, 32, 64 + 3, [64, 64, 128], False)
+ self.sa3 = PointNetSetAbstraction(64, 0.4, 32, 128 + 3, [128, 128, 256], False)
+ self.sa4 = PointNetSetAbstraction(16, 0.8, 32, 256 + 3, [256, 256, 512], False)
+ self.fp4 = PointNetFeaturePropagation(768, [256, 256])
+ self.fp3 = PointNetFeaturePropagation(384, [256, 256])
+ self.fp2 = PointNetFeaturePropagation(320, [256, 128])
+ self.fp1 = PointNetFeaturePropagation(128, [128, 128, 128])
+ self.conv1 = nn.Conv1d(128, 128, 1)
+ self.bn1 = nn.BatchNorm1d(128)
+ self.drop1 = nn.Dropout(0.5)
+ self.conv2 = nn.Conv1d(128, num_classes, 1)
+
+ def forward(self, xyz):
+ if self.with_rgb:
+ l0_points = xyz
+ l0_xyz = xyz[:,:3,:]
+ else:
+ l0_points = xyz
+ l0_xyz = xyz
+ l1_xyz, l1_points = self.sa1(l0_xyz, l0_points)
+ l2_xyz, l2_points = self.sa2(l1_xyz, l1_points)
+ l3_xyz, l3_points = self.sa3(l2_xyz, l2_points)
+ l4_xyz, l4_points = self.sa4(l3_xyz, l3_points)
+
+ l3_points = self.fp4(l3_xyz, l4_xyz, l3_points, l4_points)
+ l2_points = self.fp3(l2_xyz, l3_xyz, l2_points, l3_points)
+ l1_points = self.fp2(l1_xyz, l2_xyz, l1_points, l2_points)
+ l0_points = self.fp1(xyz, l1_xyz, None, l1_points)
+
+ x = self.drop1(F.relu(self.bn1(self.conv1(l0_points))))
+ x = self.conv2(x)
+ x = F.log_softmax(x, dim=1)
+ x = x.permute(0, 2, 1)
+ return x, l4_points
+
+
+class get_loss(nn.Module):
+ def __init__(self):
+ super(get_loss, self).__init__()
+ def forward(self, pred, target, trans_feat):
+ total_loss = F.nll_loss(pred, target)
+
+ return total_loss
+
+if __name__ == '__main__':
+ import torch
+ model = get_model(13, with_rgb=True)
+ xyz = torch.rand(12, 6, 2048)
+ (model(xyz))
\ No newline at end of file
diff --git a/models/pointnet_cls.py b/models/pointnet_cls.py
new file mode 100644
index 000000000..39232d1d8
--- /dev/null
+++ b/models/pointnet_cls.py
@@ -0,0 +1,40 @@
+import torch.nn as nn
+import torch.utils.data
+import torch.nn.functional as F
+from pointnet import PointNetEncoder, feature_transform_reguliarzer
+
+class get_model(nn.Module):
+ def __init__(self, k=40, normal_channel=True):
+ super(get_model, self).__init__()
+ if normal_channel:
+ channel = 6
+ else:
+ channel = 3
+ self.feat = PointNetEncoder(global_feat=True, feature_transform=True, channel=channel)
+ self.fc1 = nn.Linear(1024, 512)
+ self.fc2 = nn.Linear(512, 256)
+ self.fc3 = nn.Linear(256, k)
+ self.dropout = nn.Dropout(p=0.4)
+ self.bn1 = nn.BatchNorm1d(512)
+ self.bn2 = nn.BatchNorm1d(256)
+ self.relu = nn.ReLU()
+
+ def forward(self, x):
+ x, trans, trans_feat = self.feat(x)
+ x = F.relu(self.bn1(self.fc1(x)))
+ x = F.relu(self.bn2(self.dropout(self.fc2(x))))
+ x = self.fc3(x)
+ x = F.log_softmax(x, dim=1)
+ return x, trans_feat
+
+class get_loss(torch.nn.Module):
+ def __init__(self, mat_diff_loss_scale=0.001):
+ super(get_loss, self).__init__()
+ self.mat_diff_loss_scale = mat_diff_loss_scale
+
+ def forward(self, pred, target, trans_feat):
+ loss = F.nll_loss(pred, target)
+ mat_diff_loss = feature_transform_reguliarzer(trans_feat)
+
+ total_loss = loss + mat_diff_loss * self.mat_diff_loss_scale
+ return total_loss
diff --git a/models/pointnet_part_seg.py b/models/pointnet_part_seg.py
new file mode 100644
index 000000000..1ed657890
--- /dev/null
+++ b/models/pointnet_part_seg.py
@@ -0,0 +1,86 @@
+import torch
+import torch.nn as nn
+import torch.nn.parallel
+import torch.utils.data
+import torch.nn.functional as F
+from pointnet import STN3d, STNkd, feature_transform_reguliarzer
+
+
+class get_model(nn.Module):
+ def __init__(self, part_num=50, normal_channel=True):
+ super(get_model, self).__init__()
+ if normal_channel:
+ channel = 6
+ else:
+ channel = 3
+ self.part_num = part_num
+ self.stn = STN3d(channel)
+ self.conv1 = torch.nn.Conv1d(channel, 64, 1)
+ self.conv2 = torch.nn.Conv1d(64, 128, 1)
+ self.conv3 = torch.nn.Conv1d(128, 128, 1)
+ self.conv4 = torch.nn.Conv1d(128, 512, 1)
+ self.conv5 = torch.nn.Conv1d(512, 2048, 1)
+ self.bn1 = nn.BatchNorm1d(64)
+ self.bn2 = nn.BatchNorm1d(128)
+ self.bn3 = nn.BatchNorm1d(128)
+ self.bn4 = nn.BatchNorm1d(512)
+ self.bn5 = nn.BatchNorm1d(2048)
+ self.fstn = STNkd(k=128)
+ self.convs1 = torch.nn.Conv1d(4944, 256, 1)
+ self.convs2 = torch.nn.Conv1d(256, 256, 1)
+ self.convs3 = torch.nn.Conv1d(256, 128, 1)
+ self.convs4 = torch.nn.Conv1d(128, part_num, 1)
+ self.bns1 = nn.BatchNorm1d(256)
+ self.bns2 = nn.BatchNorm1d(256)
+ self.bns3 = nn.BatchNorm1d(128)
+
+ def forward(self, point_cloud, label):
+ B, D, N = point_cloud.size()
+ trans = self.stn(point_cloud)
+ point_cloud = point_cloud.transpose(2, 1)
+ if D > 3:
+ point_cloud, feature = point_cloud.split(3, dim=2)
+ point_cloud = torch.bmm(point_cloud, trans)
+ if D > 3:
+ point_cloud = torch.cat([point_cloud, feature], dim=2)
+
+ point_cloud = point_cloud.transpose(2, 1)
+
+ out1 = F.relu(self.bn1(self.conv1(point_cloud)))
+ out2 = F.relu(self.bn2(self.conv2(out1)))
+ out3 = F.relu(self.bn3(self.conv3(out2)))
+
+ trans_feat = self.fstn(out3)
+ x = out3.transpose(2, 1)
+ net_transformed = torch.bmm(x, trans_feat)
+ net_transformed = net_transformed.transpose(2, 1)
+
+ out4 = F.relu(self.bn4(self.conv4(net_transformed)))
+ out5 = self.bn5(self.conv5(out4))
+ out_max = torch.max(out5, 2, keepdim=True)[0]
+ out_max = out_max.view(-1, 2048)
+
+ out_max = torch.cat([out_max,label.squeeze(1)],1)
+ expand = out_max.view(-1, 2048+16, 1).repeat(1, 1, N)
+ concat = torch.cat([expand, out1, out2, out3, out4, out5], 1)
+ net = F.relu(self.bns1(self.convs1(concat)))
+ net = F.relu(self.bns2(self.convs2(net)))
+ net = F.relu(self.bns3(self.convs3(net)))
+ net = self.convs4(net)
+ net = net.transpose(2, 1).contiguous()
+ net = F.log_softmax(net.view(-1, self.part_num), dim=-1)
+ net = net.view(B, N, self.part_num) # [B, N, 50]
+
+ return net, trans_feat
+
+
+class get_loss(torch.nn.Module):
+ def __init__(self, mat_diff_loss_scale=0.001):
+ super(get_loss, self).__init__()
+ self.mat_diff_loss_scale = mat_diff_loss_scale
+
+ def forward(self, pred, target, trans_feat):
+ loss = F.nll_loss(pred, target)
+ mat_diff_loss = feature_transform_reguliarzer(trans_feat)
+ total_loss = loss + mat_diff_loss * self.mat_diff_loss_scale
+ return total_loss
\ No newline at end of file
diff --git a/models/pointnet_sem_seg.py b/models/pointnet_sem_seg.py
new file mode 100644
index 000000000..5a83c4cbf
--- /dev/null
+++ b/models/pointnet_sem_seg.py
@@ -0,0 +1,54 @@
+import torch
+import torch.nn as nn
+import torch.nn.parallel
+import torch.utils.data
+import torch.nn.functional as F
+from pointnet import PointNetEncoder, feature_transform_reguliarzer
+
+
+class get_model(nn.Module):
+ def __init__(self, num_class, with_rgb=True):
+ super(get_model, self).__init__()
+ if with_rgb:
+ channel = 6
+ else:
+ channel = 3
+ self.k = num_class
+ self.feat = PointNetEncoder(global_feat=False, feature_transform=True, channel=channel)
+ self.conv1 = torch.nn.Conv1d(1088, 512, 1)
+ self.conv2 = torch.nn.Conv1d(512, 256, 1)
+ self.conv3 = torch.nn.Conv1d(256, 128, 1)
+ self.conv4 = torch.nn.Conv1d(128, self.k, 1)
+ self.bn1 = nn.BatchNorm1d(512)
+ self.bn2 = nn.BatchNorm1d(256)
+ self.bn3 = nn.BatchNorm1d(128)
+
+ def forward(self, x):
+ batchsize = x.size()[0]
+ n_pts = x.size()[2]
+ x, trans, trans_feat = self.feat(x)
+ x = F.relu(self.bn1(self.conv1(x)))
+ x = F.relu(self.bn2(self.conv2(x)))
+ x = F.relu(self.bn3(self.conv3(x)))
+ x = self.conv4(x)
+ x = x.transpose(2,1).contiguous()
+ x = F.log_softmax(x.view(-1,self.k), dim=-1)
+ x = x.view(batchsize, n_pts, self.k)
+ return x, trans_feat
+
+class get_loss(torch.nn.Module):
+ def __init__(self, mat_diff_loss_scale=0.001):
+ super(get_loss, self).__init__()
+ self.mat_diff_loss_scale = mat_diff_loss_scale
+
+ def forward(self, pred, target, trans_feat):
+ loss = F.nll_loss(pred, target)
+ mat_diff_loss = feature_transform_reguliarzer(trans_feat)
+ total_loss = loss + mat_diff_loss * self.mat_diff_loss_scale
+ return total_loss
+
+
+if __name__ == '__main__':
+ model = get_model(13, with_rgb=False)
+ xyz = torch.rand(12, 3, 2048)
+ (model(xyz))
\ No newline at end of file
diff --git a/model/pointnet_util.py b/models/pointnet_util.py
similarity index 100%
rename from model/pointnet_util.py
rename to models/pointnet_util.py
diff --git a/provider.py b/provider.py
new file mode 100644
index 000000000..56046916e
--- /dev/null
+++ b/provider.py
@@ -0,0 +1,251 @@
+import numpy as np
+
+def normalize_data(batch_data):
+ """ Normalize the batch data, use coordinates of the block centered at origin,
+ Input:
+ BxNxC array
+ Output:
+ BxNxC array
+ """
+ B, N, C = batch_data.shape
+ normal_data = np.zeros((B, N, C))
+ for b in range(B):
+ pc = batch_data[b]
+ centroid = np.mean(pc, axis=0)
+ pc = pc - centroid
+ m = np.max(np.sqrt(np.sum(pc ** 2, axis=1)))
+ pc = pc / m
+ normal_data[b] = pc
+ return normal_data
+
+
+def shuffle_data(data, labels):
+ """ Shuffle data and labels.
+ Input:
+ data: B,N,... numpy array
+ label: B,... numpy array
+ Return:
+ shuffled data, label and shuffle indices
+ """
+ idx = np.arange(len(labels))
+ np.random.shuffle(idx)
+ return data[idx, ...], labels[idx], idx
+
+def shuffle_points(batch_data):
+ """ Shuffle orders of points in each point cloud -- changes FPS behavior.
+ Use the same shuffling idx for the entire batch.
+ Input:
+ BxNxC array
+ Output:
+ BxNxC array
+ """
+ idx = np.arange(batch_data.shape[1])
+ np.random.shuffle(idx)
+ return batch_data[:,idx,:]
+
+def rotate_point_cloud(batch_data):
+ """ Randomly rotate the point clouds to augument the dataset
+ rotation is per shape based along up direction
+ Input:
+ BxNx3 array, original batch of point clouds
+ Return:
+ BxNx3 array, rotated batch of point clouds
+ """
+ rotated_data = np.zeros(batch_data.shape, dtype=np.float32)
+ for k in range(batch_data.shape[0]):
+ rotation_angle = np.random.uniform() * 2 * np.pi
+ cosval = np.cos(rotation_angle)
+ sinval = np.sin(rotation_angle)
+ rotation_matrix = np.array([[cosval, 0, sinval],
+ [0, 1, 0],
+ [-sinval, 0, cosval]])
+ shape_pc = batch_data[k, ...]
+ rotated_data[k, ...] = np.dot(shape_pc.reshape((-1, 3)), rotation_matrix)
+ return rotated_data
+
+def rotate_point_cloud_z(batch_data):
+ """ Randomly rotate the point clouds to augument the dataset
+ rotation is per shape based along up direction
+ Input:
+ BxNx3 array, original batch of point clouds
+ Return:
+ BxNx3 array, rotated batch of point clouds
+ """
+ rotated_data = np.zeros(batch_data.shape, dtype=np.float32)
+ for k in range(batch_data.shape[0]):
+ rotation_angle = np.random.uniform() * 2 * np.pi
+ cosval = np.cos(rotation_angle)
+ sinval = np.sin(rotation_angle)
+ rotation_matrix = np.array([[cosval, sinval, 0],
+ [-sinval, cosval, 0],
+ [0, 0, 1]])
+ shape_pc = batch_data[k, ...]
+ rotated_data[k, ...] = np.dot(shape_pc.reshape((-1, 3)), rotation_matrix)
+ return rotated_data
+
+def rotate_point_cloud_with_normal(batch_xyz_normal):
+ ''' Randomly rotate XYZ, normal point cloud.
+ Input:
+ batch_xyz_normal: B,N,6, first three channels are XYZ, last 3 all normal
+ Output:
+ B,N,6, rotated XYZ, normal point cloud
+ '''
+ for k in range(batch_xyz_normal.shape[0]):
+ rotation_angle = np.random.uniform() * 2 * np.pi
+ cosval = np.cos(rotation_angle)
+ sinval = np.sin(rotation_angle)
+ rotation_matrix = np.array([[cosval, 0, sinval],
+ [0, 1, 0],
+ [-sinval, 0, cosval]])
+ shape_pc = batch_xyz_normal[k,:,0:3]
+ shape_normal = batch_xyz_normal[k,:,3:6]
+ batch_xyz_normal[k,:,0:3] = np.dot(shape_pc.reshape((-1, 3)), rotation_matrix)
+ batch_xyz_normal[k,:,3:6] = np.dot(shape_normal.reshape((-1, 3)), rotation_matrix)
+ return batch_xyz_normal
+
+def rotate_perturbation_point_cloud_with_normal(batch_data, angle_sigma=0.06, angle_clip=0.18):
+ """ Randomly perturb the point clouds by small rotations
+ Input:
+ BxNx6 array, original batch of point clouds and point normals
+ Return:
+ BxNx3 array, rotated batch of point clouds
+ """
+ rotated_data = np.zeros(batch_data.shape, dtype=np.float32)
+ for k in range(batch_data.shape[0]):
+ angles = np.clip(angle_sigma*np.random.randn(3), -angle_clip, angle_clip)
+ Rx = np.array([[1,0,0],
+ [0,np.cos(angles[0]),-np.sin(angles[0])],
+ [0,np.sin(angles[0]),np.cos(angles[0])]])
+ Ry = np.array([[np.cos(angles[1]),0,np.sin(angles[1])],
+ [0,1,0],
+ [-np.sin(angles[1]),0,np.cos(angles[1])]])
+ Rz = np.array([[np.cos(angles[2]),-np.sin(angles[2]),0],
+ [np.sin(angles[2]),np.cos(angles[2]),0],
+ [0,0,1]])
+ R = np.dot(Rz, np.dot(Ry,Rx))
+ shape_pc = batch_data[k,:,0:3]
+ shape_normal = batch_data[k,:,3:6]
+ rotated_data[k,:,0:3] = np.dot(shape_pc.reshape((-1, 3)), R)
+ rotated_data[k,:,3:6] = np.dot(shape_normal.reshape((-1, 3)), R)
+ return rotated_data
+
+
+def rotate_point_cloud_by_angle(batch_data, rotation_angle):
+ """ Rotate the point cloud along up direction with certain angle.
+ Input:
+ BxNx3 array, original batch of point clouds
+ Return:
+ BxNx3 array, rotated batch of point clouds
+ """
+ rotated_data = np.zeros(batch_data.shape, dtype=np.float32)
+ for k in range(batch_data.shape[0]):
+ #rotation_angle = np.random.uniform() * 2 * np.pi
+ cosval = np.cos(rotation_angle)
+ sinval = np.sin(rotation_angle)
+ rotation_matrix = np.array([[cosval, 0, sinval],
+ [0, 1, 0],
+ [-sinval, 0, cosval]])
+ shape_pc = batch_data[k,:,0:3]
+ rotated_data[k,:,0:3] = np.dot(shape_pc.reshape((-1, 3)), rotation_matrix)
+ return rotated_data
+
+def rotate_point_cloud_by_angle_with_normal(batch_data, rotation_angle):
+ """ Rotate the point cloud along up direction with certain angle.
+ Input:
+ BxNx6 array, original batch of point clouds with normal
+ scalar, angle of rotation
+ Return:
+ BxNx6 array, rotated batch of point clouds iwth normal
+ """
+ rotated_data = np.zeros(batch_data.shape, dtype=np.float32)
+ for k in range(batch_data.shape[0]):
+ #rotation_angle = np.random.uniform() * 2 * np.pi
+ cosval = np.cos(rotation_angle)
+ sinval = np.sin(rotation_angle)
+ rotation_matrix = np.array([[cosval, 0, sinval],
+ [0, 1, 0],
+ [-sinval, 0, cosval]])
+ shape_pc = batch_data[k,:,0:3]
+ shape_normal = batch_data[k,:,3:6]
+ rotated_data[k,:,0:3] = np.dot(shape_pc.reshape((-1, 3)), rotation_matrix)
+ rotated_data[k,:,3:6] = np.dot(shape_normal.reshape((-1,3)), rotation_matrix)
+ return rotated_data
+
+
+
+def rotate_perturbation_point_cloud(batch_data, angle_sigma=0.06, angle_clip=0.18):
+ """ Randomly perturb the point clouds by small rotations
+ Input:
+ BxNx3 array, original batch of point clouds
+ Return:
+ BxNx3 array, rotated batch of point clouds
+ """
+ rotated_data = np.zeros(batch_data.shape, dtype=np.float32)
+ for k in range(batch_data.shape[0]):
+ angles = np.clip(angle_sigma*np.random.randn(3), -angle_clip, angle_clip)
+ Rx = np.array([[1,0,0],
+ [0,np.cos(angles[0]),-np.sin(angles[0])],
+ [0,np.sin(angles[0]),np.cos(angles[0])]])
+ Ry = np.array([[np.cos(angles[1]),0,np.sin(angles[1])],
+ [0,1,0],
+ [-np.sin(angles[1]),0,np.cos(angles[1])]])
+ Rz = np.array([[np.cos(angles[2]),-np.sin(angles[2]),0],
+ [np.sin(angles[2]),np.cos(angles[2]),0],
+ [0,0,1]])
+ R = np.dot(Rz, np.dot(Ry,Rx))
+ shape_pc = batch_data[k, ...]
+ rotated_data[k, ...] = np.dot(shape_pc.reshape((-1, 3)), R)
+ return rotated_data
+
+
+def jitter_point_cloud(batch_data, sigma=0.01, clip=0.05):
+ """ Randomly jitter points. jittering is per point.
+ Input:
+ BxNx3 array, original batch of point clouds
+ Return:
+ BxNx3 array, jittered batch of point clouds
+ """
+ B, N, C = batch_data.shape
+ assert(clip > 0)
+ jittered_data = np.clip(sigma * np.random.randn(B, N, C), -1*clip, clip)
+ jittered_data += batch_data
+ return jittered_data
+
+def shift_point_cloud(batch_data, shift_range=0.1):
+ """ Randomly shift point cloud. Shift is per point cloud.
+ Input:
+ BxNx3 array, original batch of point clouds
+ Return:
+ BxNx3 array, shifted batch of point clouds
+ """
+ B, N, C = batch_data.shape
+ shifts = np.random.uniform(-shift_range, shift_range, (B,3))
+ for batch_index in range(B):
+ batch_data[batch_index,:,:] += shifts[batch_index,:]
+ return batch_data
+
+
+def random_scale_point_cloud(batch_data, scale_low=0.8, scale_high=1.25):
+ """ Randomly scale the point cloud. Scale is per point cloud.
+ Input:
+ BxNx3 array, original batch of point clouds
+ Return:
+ BxNx3 array, scaled batch of point clouds
+ """
+ B, N, C = batch_data.shape
+ scales = np.random.uniform(scale_low, scale_high, B)
+ for batch_index in range(B):
+ batch_data[batch_index,:,:] *= scales[batch_index]
+ return batch_data
+
+def random_point_dropout(batch_pc, max_dropout_ratio=0.875):
+ ''' batch_pc: BxNx3 '''
+ for b in range(batch_pc.shape[0]):
+ dropout_ratio = np.random.random()*max_dropout_ratio # 0~0.875
+ drop_idx = np.where(np.random.random((batch_pc.shape[1]))<=dropout_ratio)[0]
+ if len(drop_idx)>0:
+ batch_pc[b,drop_idx,:] = batch_pc[b,0,:] # set to the first point
+ return batch_pc
+
+
+
diff --git a/test_cls.py b/test_cls.py
new file mode 100644
index 000000000..ce64445ed
--- /dev/null
+++ b/test_cls.py
@@ -0,0 +1,105 @@
+"""
+Author: Benny
+Date: Nov 2019
+"""
+from data_utils.ModelNetDataLoader import ModelNetDataLoader
+import argparse
+import numpy as np
+import os
+import torch
+import logging
+from tqdm import tqdm
+import sys
+import importlib
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = BASE_DIR
+sys.path.append(os.path.join(ROOT_DIR, 'models'))
+
+
+def parse_args():
+ '''PARAMETERS'''
+ parser = argparse.ArgumentParser('PointNet')
+ parser.add_argument('--batch_size', type=int, default=24, help='batch size in training')
+ parser.add_argument('--gpu', type=str, default='0', help='specify gpu device')
+ parser.add_argument('--num_point', type=int, default=1024, help='Point Number [default: 1024]')
+ parser.add_argument('--log_dir', type=str, default='pointnet2_ssg_normal', help='Experiment root')
+ parser.add_argument('--normal', action='store_true', default=True, help='Whether to use normal information [default: False]')
+ parser.add_argument('--num_votes', type=int, default=3, help='Aggregate classification scores with voting [default: 3]')
+ return parser.parse_args()
+
+def test(model, loader, num_class=40, vote_num=1):
+ mean_correct = []
+ class_acc = np.zeros((num_class,3))
+ for j, data in tqdm(enumerate(loader), total=len(loader)):
+ points, target = data
+ target = target[:, 0]
+ points = points.transpose(2, 1)
+ points, target = points.cuda(), target.cuda()
+ classifier = model.eval()
+ vote_pool = torch.zeros(target.size()[0],num_class).cuda()
+ for _ in range(vote_num):
+ pred, _ = classifier(points)
+ vote_pool += pred
+ pred = vote_pool/vote_num
+ pred_choice = pred.data.max(1)[1]
+ for cat in np.unique(target.cpu()):
+ classacc = pred_choice[target==cat].eq(target[target==cat].long().data).cpu().sum()
+ class_acc[cat,0]+= classacc.item()/float(points[target==cat].size()[0])
+ class_acc[cat,1]+=1
+ correct = pred_choice.eq(target.long().data).cpu().sum()
+ mean_correct.append(correct.item()/float(points.size()[0]))
+ class_acc[:,2] = class_acc[:,0]/ class_acc[:,1]
+ class_acc = np.mean(class_acc[:,2])
+ instance_acc = np.mean(mean_correct)
+ return instance_acc, class_acc
+
+
+def main(args):
+ def log_string(str):
+ logger.info(str)
+ print(str)
+
+ '''HYPER PARAMETER'''
+ os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
+
+ '''CREATE DIR'''
+ experiment_dir = 'log/classification/' + args.log_dir
+
+ '''LOG'''
+ args = parse_args()
+ logger = logging.getLogger("Model")
+ logger.setLevel(logging.INFO)
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ file_handler = logging.FileHandler('%s/eval.txt' % experiment_dir)
+ file_handler.setLevel(logging.INFO)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ log_string('PARAMETER ...')
+ log_string(args)
+
+ '''DATA LOADING'''
+ log_string('Load dataset ...')
+ DATA_PATH = 'data/modelnet40_normal_resampled/'
+ TEST_DATASET = ModelNetDataLoader(root=DATA_PATH, npoint=args.num_point, split='test', normal_channel=args.normal)
+ testDataLoader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=args.batch_size, shuffle=False, num_workers=4)
+
+ '''MODEL LOADING'''
+ num_class = 40
+ model_name = os.listdir(experiment_dir+'/logs')[0].split('.')[0]
+ MODEL = importlib.import_module(model_name)
+
+ classifier = MODEL.get_model(num_class,normal_channel=args.normal).cuda()
+
+ checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
+ classifier.load_state_dict(checkpoint['model_state_dict'])
+
+ with torch.no_grad():
+ instance_acc, class_acc = test(classifier.eval(), testDataLoader, vote_num=args.num_votes)
+ log_string('Test Instance Accuracy: %f, Class Accuracy: %f' % (instance_acc, class_acc))
+
+
+
+if __name__ == '__main__':
+ args = parse_args()
+ main(args)
diff --git a/test_partseg.py b/test_partseg.py
new file mode 100644
index 000000000..8e14982fa
--- /dev/null
+++ b/test_partseg.py
@@ -0,0 +1,157 @@
+"""
+Author: Benny
+Date: Nov 2019
+"""
+import argparse
+import os
+from data_utils.ShapeNetDataLoader import PartNormalDataset
+import torch
+import logging
+import sys
+import importlib
+from tqdm import tqdm
+import numpy as np
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = BASE_DIR
+sys.path.append(os.path.join(ROOT_DIR, 'models'))
+
+seg_classes = {'Earphone': [16, 17, 18], 'Motorbike': [30, 31, 32, 33, 34, 35], 'Rocket': [41, 42, 43], 'Car': [8, 9, 10, 11], 'Laptop': [28, 29], 'Cap': [6, 7], 'Skateboard': [44, 45, 46], 'Mug': [36, 37], 'Guitar': [19, 20, 21], 'Bag': [4, 5], 'Lamp': [24, 25, 26, 27], 'Table': [47, 48, 49], 'Airplane': [0, 1, 2, 3], 'Pistol': [38, 39, 40], 'Chair': [12, 13, 14, 15], 'Knife': [22, 23]}
+seg_label_to_cat = {} # {0:Airplane, 1:Airplane, ...49:Table}
+for cat in seg_classes.keys():
+ for label in seg_classes[cat]:
+ seg_label_to_cat[label] = cat
+
+def to_categorical(y, num_classes):
+ """ 1-hot encodes a tensor """
+ new_y = torch.eye(num_classes)[y.cpu().data.numpy(),]
+ if (y.is_cuda):
+ return new_y.cuda()
+ return new_y
+
+
+def parse_args():
+ '''PARAMETERS'''
+ parser = argparse.ArgumentParser('PointNet')
+ parser.add_argument('--batch_size', type=int, default=24, help='batch size in testing')
+ parser.add_argument('--gpu', type=str, default='0', help='specify gpu device')
+ parser.add_argument('--num_point', type=int, default=2048, help='Point Number [default: 1024]')
+ parser.add_argument('--log_dir', type=str, default='pointnet2_part_seg_ssg', help='Experiment root')
+ parser.add_argument('--normal', action='store_true', default=True, help='Whether to use normal information [default: False]')
+ parser.add_argument('--num_votes', type=int, default=3, help='Aggregate segmentation scores with voting [default: 3]')
+ return parser.parse_args()
+
+def main(args):
+ def log_string(str):
+ logger.info(str)
+ print(str)
+
+ '''HYPER PARAMETER'''
+ os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
+ experiment_dir = 'log/part_seg/' + args.log_dir
+
+ '''LOG'''
+ args = parse_args()
+ logger = logging.getLogger("Model")
+ logger.setLevel(logging.INFO)
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ file_handler = logging.FileHandler('%s/eval.txt' % experiment_dir)
+ file_handler.setLevel(logging.INFO)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ log_string('PARAMETER ...')
+ log_string(args)
+
+ root = 'data/shapenetcore_partanno_segmentation_benchmark_v0_normal/'
+
+ TEST_DATASET = PartNormalDataset(root = root, npoints=args.num_point, split='test', normal_channel=args.normal)
+ testDataLoader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=args.batch_size,shuffle=False, num_workers=4)
+ log_string("The number of test data is: %d" % len(TEST_DATASET))
+ num_classes = 16
+ num_part = 50
+
+ '''MODEL LOADING'''
+ model_name = os.listdir(experiment_dir+'/logs')[0].split('.')[0]
+ MODEL = importlib.import_module(model_name)
+ classifier = MODEL.get_model(num_part, normal_channel=args.normal).cuda()
+ checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
+ classifier.load_state_dict(checkpoint['model_state_dict'])
+
+
+ with torch.no_grad():
+ test_metrics = {}
+ total_correct = 0
+ total_seen = 0
+ total_seen_class = [0 for _ in range(num_part)]
+ total_correct_class = [0 for _ in range(num_part)]
+ shape_ious = {cat: [] for cat in seg_classes.keys()}
+ seg_label_to_cat = {} # {0:Airplane, 1:Airplane, ...49:Table}
+ for cat in seg_classes.keys():
+ for label in seg_classes[cat]:
+ seg_label_to_cat[label] = cat
+
+ for batch_id, (points, label, target) in tqdm(enumerate(testDataLoader), total=len(testDataLoader), smoothing=0.9):
+ batchsize, num_point, _ = points.size()
+ cur_batch_size, NUM_POINT, _ = points.size()
+ points, label, target = points.float().cuda(), label.long().cuda(), target.long().cuda()
+ points = points.transpose(2, 1)
+ classifier = classifier.eval()
+ vote_pool = torch.zeros(target.size()[0], target.size()[1], num_part).cuda()
+ for _ in range(args.num_votes):
+ seg_pred, _ = classifier(points, to_categorical(label, num_classes))
+ vote_pool += seg_pred
+ seg_pred = vote_pool / args.num_votes
+ cur_pred_val = seg_pred.cpu().data.numpy()
+ cur_pred_val_logits = cur_pred_val
+ cur_pred_val = np.zeros((cur_batch_size, NUM_POINT)).astype(np.int32)
+ target = target.cpu().data.numpy()
+ for i in range(cur_batch_size):
+ cat = seg_label_to_cat[target[i, 0]]
+ logits = cur_pred_val_logits[i, :, :]
+ cur_pred_val[i, :] = np.argmax(logits[:, seg_classes[cat]], 1) + seg_classes[cat][0]
+ correct = np.sum(cur_pred_val == target)
+ total_correct += correct
+ total_seen += (cur_batch_size * NUM_POINT)
+
+ for l in range(num_part):
+ total_seen_class[l] += np.sum(target == l)
+ total_correct_class[l] += (np.sum((cur_pred_val == l) & (target == l)))
+
+ for i in range(cur_batch_size):
+ segp = cur_pred_val[i, :]
+ segl = target[i, :]
+ cat = seg_label_to_cat[segl[0]]
+ part_ious = [0.0 for _ in range(len(seg_classes[cat]))]
+ for l in seg_classes[cat]:
+ if (np.sum(segl == l) == 0) and (
+ np.sum(segp == l) == 0): # part is not present, no prediction as well
+ part_ious[l - seg_classes[cat][0]] = 1.0
+ else:
+ part_ious[l - seg_classes[cat][0]] = np.sum((segl == l) & (segp == l)) / float(
+ np.sum((segl == l) | (segp == l)))
+ shape_ious[cat].append(np.mean(part_ious))
+
+ all_shape_ious = []
+ for cat in shape_ious.keys():
+ for iou in shape_ious[cat]:
+ all_shape_ious.append(iou)
+ shape_ious[cat] = np.mean(shape_ious[cat])
+ mean_shape_ious = np.mean(list(shape_ious.values()))
+ test_metrics['accuracy'] = total_correct / float(total_seen)
+ test_metrics['class_avg_accuracy'] = np.mean(
+ np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float))
+ for cat in sorted(shape_ious.keys()):
+ log_string('eval mIoU of %s %f' % (cat + ' ' * (14 - len(cat)), shape_ious[cat]))
+ test_metrics['class_avg_iou'] = mean_shape_ious
+ test_metrics['inctance_avg_iou'] = np.mean(all_shape_ious)
+
+
+ log_string('Accuracy is: %.5f'%test_metrics['accuracy'])
+ log_string('Class avg accuracy is: %.5f'%test_metrics['class_avg_accuracy'])
+ log_string('Class avg mIOU is: %.5f'%test_metrics['class_avg_iou'])
+ log_string('Inctance avg mIOU is: %.5f'%test_metrics['inctance_avg_iou'])
+
+if __name__ == '__main__':
+ args = parse_args()
+ main(args)
+
diff --git a/train_clf.py b/train_clf.py
deleted file mode 100644
index 17d94125e..000000000
--- a/train_clf.py
+++ /dev/null
@@ -1,159 +0,0 @@
-import argparse
-import os
-import torch
-import torch.nn.parallel
-import torch.utils.data
-import torch.nn.functional as F
-from data_utils.ModelNetDataLoader import ModelNetDataLoader, load_data
-import datetime
-import logging
-from pathlib import Path
-from tqdm import tqdm
-from utils import test, save_checkpoint
-from model.pointnet2 import PointNet2ClsMsg
-from model.pointnet import PointNetCls, feature_transform_reguliarzer
-
-
-def parse_args():
- '''PARAMETERS'''
- parser = argparse.ArgumentParser('PointNet')
- parser.add_argument('--batchsize', type=int, default=24, help='batch size in training')
- parser.add_argument('--epoch', default=200, type=int, help='number of epoch in training')
- parser.add_argument('--learning_rate', default=0.001, type=float, help='learning rate in training')
- parser.add_argument('--gpu', type=str, default='0', help='specify gpu device')
- parser.add_argument('--train_metric', type=str, default=False, help='whether evaluate on training dataset')
- parser.add_argument('--optimizer', type=str, default='SGD', help='optimizer for training')
- parser.add_argument('--pretrain', type=str, default=None,help='whether use pretrain model')
- parser.add_argument('--decay_rate', type=float, default=1e-4, help='decay rate of learning rate')
- parser.add_argument('--rotation', default=None, help='range of training rotation')
- parser.add_argument('--model_name', default='pointnet2', help='range of training rotation')
- parser.add_argument('--feature_transform', default=False, help="use feature transform in pointnet")
- return parser.parse_args()
-
-def main(args):
- '''HYPER PARAMETER'''
- os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
- datapath = './data/ModelNet/'
-
- if args.rotation is not None:
- ROTATION = (int(args.rotation[0:2]),int(args.rotation[3:5]))
- else:
- ROTATION = None
-
- '''CREATE DIR'''
- experiment_dir = Path('./experiment/')
- experiment_dir.mkdir(exist_ok=True)
- checkpoints_dir = Path('./experiment/checkpoints/')
- checkpoints_dir.mkdir(exist_ok=True)
- log_dir = Path('./experiment/logs/')
- log_dir.mkdir(exist_ok=True)
-
- '''LOG'''
- args = parse_args()
- logger = logging.getLogger("PointNet2")
- logger.setLevel(logging.INFO)
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
- file_handler = logging.FileHandler('./experiment/logs/train_%s_'%args.model_name+ str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M'))+'.txt')
- file_handler.setLevel(logging.INFO)
- file_handler.setFormatter(formatter)
- logger.addHandler(file_handler)
- logger.info('---------------------------------------------------TRANING---------------------------------------------------')
- logger.info('PARAMETER ...')
- logger.info(args)
-
- '''DATA LOADING'''
- logger.info('Load dataset ...')
- train_data, train_label, test_data, test_label = load_data(datapath, classification=True)
- logger.info("The number of training data is: %d",train_data.shape[0])
- logger.info("The number of test data is: %d", test_data.shape[0])
- trainDataset = ModelNetDataLoader(train_data, train_label, rotation=ROTATION)
- if ROTATION is not None:
- print('The range of training rotation is',ROTATION)
- testDataset = ModelNetDataLoader(test_data, test_label, rotation=ROTATION)
- trainDataLoader = torch.utils.data.DataLoader(trainDataset, batch_size=args.batchsize, shuffle=True)
- testDataLoader = torch.utils.data.DataLoader(testDataset, batch_size=args.batchsize, shuffle=False)
-
- '''MODEL LOADING'''
- num_class = 40
- classifier = PointNetCls(num_class,args.feature_transform).cuda() if args.model_name == 'pointnet' else PointNet2ClsMsg().cuda()
- if args.pretrain is not None:
- print('Use pretrain model...')
- logger.info('Use pretrain model')
- checkpoint = torch.load(args.pretrain)
- start_epoch = checkpoint['epoch']
- classifier.load_state_dict(checkpoint['model_state_dict'])
- else:
- print('No existing model, starting training from scratch...')
- start_epoch = 0
-
-
- if args.optimizer == 'SGD':
- optimizer = torch.optim.SGD(classifier.parameters(), lr=0.01, momentum=0.9)
- elif args.optimizer == 'Adam':
- optimizer = torch.optim.Adam(
- classifier.parameters(),
- lr=args.learning_rate,
- betas=(0.9, 0.999),
- eps=1e-08,
- weight_decay=args.decay_rate
- )
- scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
- global_epoch = 0
- global_step = 0
- best_tst_accuracy = 0.0
- blue = lambda x: '\033[94m' + x + '\033[0m'
-
- '''TRANING'''
- logger.info('Start training...')
- for epoch in range(start_epoch,args.epoch):
- print('Epoch %d (%d/%s):' % (global_epoch + 1, epoch + 1, args.epoch))
- logger.info('Epoch %d (%d/%s):' ,global_epoch + 1, epoch + 1, args.epoch)
-
- scheduler.step()
- for batch_id, data in tqdm(enumerate(trainDataLoader, 0), total=len(trainDataLoader), smoothing=0.9):
- points, target = data
- target = target[:, 0]
- points = points.transpose(2, 1)
- points, target = points.cuda(), target.cuda()
- optimizer.zero_grad()
- classifier = classifier.train()
- pred, trans_feat = classifier(points)
- loss = F.nll_loss(pred, target.long())
- if args.feature_transform and args.model_name == 'pointnet':
- loss += feature_transform_reguliarzer(trans_feat) * 0.001
- loss.backward()
- optimizer.step()
- global_step += 1
-
- train_acc = test(classifier.eval(), trainDataLoader) if args.train_metric else None
- acc = test(classifier, testDataLoader)
-
-
- print('\r Loss: %f' % loss.data)
- logger.info('Loss: %.2f', loss.data)
- if args.train_metric:
- print('Train Accuracy: %f' % train_acc)
- logger.info('Train Accuracy: %f', (train_acc))
- print('\r Test %s: %f' % (blue('Accuracy'),acc))
- logger.info('Test Accuracy: %f', acc)
-
- if (acc >= best_tst_accuracy) and epoch > 5:
- best_tst_accuracy = acc
- logger.info('Save model...')
- save_checkpoint(
- global_epoch + 1,
- train_acc if args.train_metric else 0.0,
- acc,
- classifier,
- optimizer,
- str(checkpoints_dir),
- args.model_name)
- print('Saving model....')
- global_epoch += 1
- print('Best Accuracy: %f'%best_tst_accuracy)
-
- logger.info('End of training...')
-
-if __name__ == '__main__':
- args = parse_args()
- main(args)
diff --git a/train_cls.py b/train_cls.py
new file mode 100644
index 000000000..d9f9b8613
--- /dev/null
+++ b/train_cls.py
@@ -0,0 +1,209 @@
+"""
+Author: Benny
+Date: Nov 2019
+"""
+from data_utils.ModelNetDataLoader import ModelNetDataLoader
+import argparse
+import numpy as np
+import os
+import torch
+import datetime
+import logging
+from pathlib import Path
+from tqdm import tqdm
+import sys
+import provider
+import importlib
+import shutil
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = BASE_DIR
+sys.path.append(os.path.join(ROOT_DIR, 'models'))
+
+
+def parse_args():
+ '''PARAMETERS'''
+ parser = argparse.ArgumentParser('PointNet')
+ parser.add_argument('--batch_size', type=int, default=24, help='batch size in training [default: 24]')
+ parser.add_argument('--model', default='pointnet_cls', help='model name [default: pointnet_cls]')
+ parser.add_argument('--epoch', default=200, type=int, help='number of epoch in training [default: 200]')
+ parser.add_argument('--learning_rate', default=0.001, type=float, help='learning rate in training [default: 0.001]')
+ parser.add_argument('--gpu', type=str, default='0', help='specify gpu device [default: 0]')
+ parser.add_argument('--num_point', type=int, default=1024, help='Point Number [default: 1024]')
+ parser.add_argument('--optimizer', type=str, default='Adam', help='optimizer for training [default: Adam]')
+ parser.add_argument('--log_dir', type=str, default=None, help='experiment root')
+ parser.add_argument('--decay_rate', type=float, default=1e-4, help='decay rate [default: 1e-4]')
+ parser.add_argument('--normal', action='store_true', default=False, help='Whether to use normal information [default: False]')
+ return parser.parse_args()
+
+def test(model, loader, num_class=40):
+ mean_correct = []
+ class_acc = np.zeros((num_class,3))
+ for j, data in tqdm(enumerate(loader), total=len(loader)):
+ points, target = data
+ target = target[:, 0]
+ points = points.transpose(2, 1)
+ points, target = points.cuda(), target.cuda()
+ classifier = model.eval()
+ pred, _ = classifier(points)
+ pred_choice = pred.data.max(1)[1]
+ for cat in np.unique(target.cpu()):
+ classacc = pred_choice[target==cat].eq(target[target==cat].long().data).cpu().sum()
+ class_acc[cat,0]+= classacc.item()/float(points[target==cat].size()[0])
+ class_acc[cat,1]+=1
+ correct = pred_choice.eq(target.long().data).cpu().sum()
+ mean_correct.append(correct.item()/float(points.size()[0]))
+ class_acc[:,2] = class_acc[:,0]/ class_acc[:,1]
+ class_acc = np.mean(class_acc[:,2])
+ instance_acc = np.mean(mean_correct)
+ return instance_acc, class_acc
+
+
+def main(args):
+ def log_string(str):
+ logger.info(str)
+ print(str)
+
+ '''HYPER PARAMETER'''
+ os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
+
+ '''CREATE DIR'''
+ timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
+ experiment_dir = Path('./log/')
+ experiment_dir.mkdir(exist_ok=True)
+ experiment_dir = experiment_dir.joinpath('classification')
+ experiment_dir.mkdir(exist_ok=True)
+ if args.log_dir is None:
+ experiment_dir = experiment_dir.joinpath(timestr)
+ else:
+ experiment_dir = experiment_dir.joinpath(args.log_dir)
+ experiment_dir.mkdir(exist_ok=True)
+ checkpoints_dir = experiment_dir.joinpath('checkpoints/')
+ checkpoints_dir.mkdir(exist_ok=True)
+ log_dir = experiment_dir.joinpath('logs/')
+ log_dir.mkdir(exist_ok=True)
+
+ '''LOG'''
+ args = parse_args()
+ logger = logging.getLogger("Model")
+ logger.setLevel(logging.INFO)
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model))
+ file_handler.setLevel(logging.INFO)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+ log_string('PARAMETER ...')
+ log_string(args)
+
+ '''DATA LOADING'''
+ log_string('Load dataset ...')
+ DATA_PATH = 'data/modelnet40_normal_resampled/'
+
+ TRAIN_DATASET = ModelNetDataLoader(root=DATA_PATH, npoint=args.num_point, split='train',
+ normal_channel=args.normal)
+ TEST_DATASET = ModelNetDataLoader(root=DATA_PATH, npoint=args.num_point, split='test',
+ normal_channel=args.normal)
+ trainDataLoader = torch.utils.data.DataLoader(TRAIN_DATASET, batch_size=args.batch_size, shuffle=True, num_workers=4)
+ testDataLoader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=args.batch_size, shuffle=False, num_workers=4)
+
+ '''MODEL LOADING'''
+ num_class = 40
+ MODEL = importlib.import_module(args.model)
+ shutil.copy('./models/%s.py' % args.model, str(experiment_dir))
+ shutil.copy('./models/pointnet_util.py', str(experiment_dir))
+
+ classifier = MODEL.get_model(num_class,normal_channel=args.normal).cuda()
+ criterion = MODEL.get_loss().cuda()
+
+ try:
+ checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
+ start_epoch = checkpoint['epoch']
+ classifier.load_state_dict(checkpoint['model_state_dict'])
+ log_string('Use pretrain model')
+ except:
+ log_string('No existing model, starting training from scratch...')
+ start_epoch = 0
+
+
+ if args.optimizer == 'Adam':
+ optimizer = torch.optim.Adam(
+ classifier.parameters(),
+ lr=args.learning_rate,
+ betas=(0.9, 0.999),
+ eps=1e-08,
+ weight_decay=args.decay_rate
+ )
+ else:
+ optimizer = torch.optim.SGD(classifier.parameters(), lr=0.01, momentum=0.9)
+
+ scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.7)
+ global_epoch = 0
+ global_step = 0
+ best_instance_acc = 0.0
+ best_class_acc = 0.0
+ mean_correct = []
+
+ '''TRANING'''
+ logger.info('Start training...')
+ for epoch in range(start_epoch,args.epoch):
+ log_string('Epoch %d (%d/%s):' % (global_epoch + 1, epoch + 1, args.epoch))
+
+ scheduler.step()
+ for batch_id, data in tqdm(enumerate(trainDataLoader, 0), total=len(trainDataLoader), smoothing=0.9):
+ points, target = data
+ points = points.data.numpy()
+ points = provider.random_point_dropout(points)
+ points[:,:, 0:3] = provider.random_scale_point_cloud(points[:,:, 0:3])
+ points[:,:, 0:3] = provider.shift_point_cloud(points[:,:, 0:3])
+ points = torch.Tensor(points)
+ target = target[:, 0]
+
+ points = points.transpose(2, 1)
+ points, target = points.cuda(), target.cuda()
+ optimizer.zero_grad()
+
+ classifier = classifier.train()
+ pred, trans_feat = classifier(points)
+ loss = criterion(pred, target.long(), trans_feat)
+ pred_choice = pred.data.max(1)[1]
+ correct = pred_choice.eq(target.long().data).cpu().sum()
+ mean_correct.append(correct.item() / float(points.size()[0]))
+ loss.backward()
+ optimizer.step()
+ global_step += 1
+
+ train_instance_acc = np.mean(mean_correct)
+ log_string('Train Instance Accuracy: %f' % train_instance_acc)
+
+
+ with torch.no_grad():
+ instance_acc, class_acc = test(classifier.eval(), testDataLoader)
+
+ if (instance_acc >= best_instance_acc):
+ best_instance_acc = instance_acc
+ best_epoch = epoch + 1
+
+ if (class_acc >= best_class_acc):
+ best_class_acc = class_acc
+ log_string('Test Instance Accuracy: %f, Class Accuracy: %f'% (instance_acc, class_acc))
+ log_string('Best Instance Accuracy: %f, Class Accuracy: %f'% (best_instance_acc, best_class_acc))
+
+ if (instance_acc >= best_instance_acc):
+ logger.info('Save model...')
+ savepath = str(checkpoints_dir) + '/best_model.pth'
+ log_string('Saving at %s'% savepath)
+ state = {
+ 'epoch': best_epoch,
+ 'instance_acc': instance_acc,
+ 'class_acc': class_acc,
+ 'model_state_dict': classifier.state_dict(),
+ 'optimizer_state_dict': optimizer.state_dict(),
+ }
+ torch.save(state, savepath)
+ global_epoch += 1
+
+ logger.info('End of training...')
+
+if __name__ == '__main__':
+ args = parse_args()
+ main(args)
diff --git a/train_partseg.py b/train_partseg.py
index 34edbd56e..00ab242f4 100644
--- a/train_partseg.py
+++ b/train_partseg.py
@@ -1,20 +1,24 @@
+"""
+Author: Benny
+Date: Nov 2019
+"""
import argparse
import os
-import torch
-import torch.nn.parallel
-import torch.utils.data
-from utils import to_categorical
-from collections import defaultdict
-from torch.autograd import Variable
from data_utils.ShapeNetDataLoader import PartNormalDataset
-import torch.nn.functional as F
+import torch
import datetime
import logging
from pathlib import Path
-from utils import test_partseg
+import sys
+import importlib
+import shutil
from tqdm import tqdm
-from model.pointnet2 import PointNet2PartSeg_msg_one_hot
-from model.pointnet import PointNetDenseCls,PointNetLoss
+import provider
+import numpy as np
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = BASE_DIR
+sys.path.append(os.path.join(ROOT_DIR, 'models'))
seg_classes = {'Earphone': [16, 17, 18], 'Motorbike': [30, 31, 32, 33, 34, 35], 'Rocket': [41, 42, 43], 'Car': [8, 9, 10, 11], 'Laptop': [28, 29], 'Cap': [6, 7], 'Skateboard': [44, 45, 46], 'Mug': [36, 37], 'Guitar': [19, 20, 21], 'Bag': [4, 5], 'Lamp': [24, 25, 26, 27], 'Table': [47, 48, 49], 'Airplane': [0, 1, 2, 3], 'Pistol': [38, 39, 40], 'Chair': [12, 13, 14, 15], 'Knife': [22, 23]}
seg_label_to_cat = {} # {0:Airplane, 1:Airplane, ...49:Table}
@@ -22,155 +26,258 @@
for label in seg_classes[cat]:
seg_label_to_cat[label] = cat
+def to_categorical(y, num_classes):
+ """ 1-hot encodes a tensor """
+ new_y = torch.eye(num_classes)[y.cpu().data.numpy(),]
+ if (y.is_cuda):
+ return new_y.cuda()
+ return new_y
+
+
def parse_args():
- parser = argparse.ArgumentParser('PointNet2')
- parser.add_argument('--batchsize', type=int, default=32, help='input batch size')
- parser.add_argument('--workers', type=int, default=4, help='number of data loading workers')
- parser.add_argument('--epoch', type=int, default=201, help='number of epochs for training')
- parser.add_argument('--pretrain', type=str, default=None,help='whether use pretrain model')
- parser.add_argument('--gpu', type=str, default='0', help='specify gpu device')
- parser.add_argument('--model_name', type=str, default='pointnet2', help='Name of model')
- parser.add_argument('--learning_rate', type=float, default=0.001, help='learning rate for training')
- parser.add_argument('--decay_rate', type=float, default=1e-4, help='weight decay')
- parser.add_argument('--optimizer', type=str, default='Adam', help='type of optimizer')
- parser.add_argument('--multi_gpu', type=str, default=None, help='whether use multi gpu training')
- parser.add_argument('--jitter', default=False, help="randomly jitter point cloud")
- parser.add_argument('--step_size', type=int, default=20, help="randomly rotate point cloud")
+ parser = argparse.ArgumentParser('Model')
+ parser.add_argument('--model', type=str, default='pointnet2_part_seg_msg', help='model name [default: pointnet2_part_seg_msg]')
+ parser.add_argument('--batch_size', type=int, default=16, help='Batch Size during training [default: 16]')
+ parser.add_argument('--epoch', default=251, type=int, help='Epoch to run [default: 251]')
+ parser.add_argument('--learning_rate', default=0.001, type=float, help='Initial learning rate [default: 0.001]')
+ parser.add_argument('--gpu', type=str, default='0', help='GPU to use [default: GPU 0]')
+ parser.add_argument('--optimizer', type=str, default='Adam', help='Adam or SGD [default: Adam]')
+ parser.add_argument('--log_dir', type=str, default=None, help='Log path [default: None]')
+ parser.add_argument('--decay_rate', type=float, default=1e-4, help='weight decay [default: 1e-4]')
+ parser.add_argument('--npoint', type=int, default=2048, help='Point Number [default: 2048]')
+ parser.add_argument('--normal', action='store_true', default=False, help='Whether to use normal information [default: True]')
+ parser.add_argument('--step_size', type=int, default=20, help='Decay step for lr decay [default: every 20 epochs]')
+ parser.add_argument('--lr_decay', type=float, default=0.5, help='Decay rate for lr decay [default: 0.5]')
return parser.parse_args()
def main(args):
- os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if args.multi_gpu is None else '0,1,2,3'
+ def log_string(str):
+ logger.info(str)
+ print(str)
+
+ '''HYPER PARAMETER'''
+ os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
+
'''CREATE DIR'''
- experiment_dir = Path('./experiment/')
+ timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
+ experiment_dir = Path('./log/')
+ experiment_dir.mkdir(exist_ok=True)
+ experiment_dir = experiment_dir.joinpath('part_seg')
+ experiment_dir.mkdir(exist_ok=True)
+ if args.log_dir is None:
+ experiment_dir = experiment_dir.joinpath(timestr)
+ else:
+ experiment_dir = experiment_dir.joinpath(args.log_dir)
experiment_dir.mkdir(exist_ok=True)
- file_dir = Path(str(experiment_dir) +'/%sPartSeg-'%args.model_name + str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')))
- file_dir.mkdir(exist_ok=True)
- checkpoints_dir = file_dir.joinpath('checkpoints/')
+ checkpoints_dir = experiment_dir.joinpath('checkpoints/')
checkpoints_dir.mkdir(exist_ok=True)
- log_dir = file_dir.joinpath('logs/')
+ log_dir = experiment_dir.joinpath('logs/')
log_dir.mkdir(exist_ok=True)
'''LOG'''
args = parse_args()
- logger = logging.getLogger(args.model_name)
+ logger = logging.getLogger("Model")
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
- file_handler = logging.FileHandler(str(log_dir) + '/train_%s_partseg.txt'%args.model_name)
+ file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model))
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
- logger.info('---------------------------------------------------TRANING---------------------------------------------------')
- logger.info('PARAMETER ...')
- logger.info(args)
- norm = True if args.model_name == 'pointnet' else False
- TRAIN_DATASET = PartNormalDataset(npoints=2048, split='trainval',normalize=norm, jitter=args.jitter)
- dataloader = torch.utils.data.DataLoader(TRAIN_DATASET, batch_size=args.batchsize,shuffle=True, num_workers=int(args.workers))
- TEST_DATASET = PartNormalDataset(npoints=2048, split='test',normalize=norm,jitter=False)
- testdataloader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=10,shuffle=True, num_workers=int(args.workers))
- print("The number of training data is:",len(TRAIN_DATASET))
- logger.info("The number of training data is:%d",len(TRAIN_DATASET))
- print("The number of test data is:", len(TEST_DATASET))
- logger.info("The number of test data is:%d", len(TEST_DATASET))
+ log_string('PARAMETER ...')
+ log_string(args)
+
+ root = 'data/shapenetcore_partanno_segmentation_benchmark_v0_normal/'
+
+ TRAIN_DATASET = PartNormalDataset(root = root, npoints=args.npoint, split='trainval', normal_channel=args.normal)
+ trainDataLoader = torch.utils.data.DataLoader(TRAIN_DATASET, batch_size=args.batch_size,shuffle=True, num_workers=4)
+ TEST_DATASET = PartNormalDataset(root = root, npoints=args.npoint, split='test', normal_channel=args.normal)
+ testDataLoader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=args.batch_size,shuffle=False, num_workers=4)
+ log_string("The number of training data is: %d" % len(TRAIN_DATASET))
+ log_string("The number of test data is: %d" % len(TEST_DATASET))
num_classes = 16
num_part = 50
- blue = lambda x: '\033[94m' + x + '\033[0m'
- model = PointNet2PartSeg_msg_one_hot(num_part) if args.model_name == 'pointnet2'else PointNetDenseCls(cat_num=num_classes,part_num=num_part)
+ '''MODEL LOADING'''
+ MODEL = importlib.import_module(args.model)
+ shutil.copy('models/%s.py' % args.model, str(experiment_dir))
+ shutil.copy('models/pointnet_util.py', str(experiment_dir))
- if args.pretrain is not None:
- model.load_state_dict(torch.load(args.pretrain))
- print('load model %s'%args.pretrain)
- logger.info('load model %s'%args.pretrain)
- else:
- print('Training from scratch')
- logger.info('Training from scratch')
- pretrain = args.pretrain
- init_epoch = int(pretrain[-14:-11]) if args.pretrain is not None else 0
+ classifier = MODEL.get_model(num_part, normal_channel=args.normal).cuda()
+ criterion = MODEL.get_loss().cuda()
+
+
+ def weights_init(m):
+ classname = m.__class__.__name__
+ if classname.find('Conv2d') != -1:
+ torch.nn.init.xavier_normal_(m.weight.data)
+ torch.nn.init.constant_(m.bias.data, 0.0)
+ elif classname.find('Linear') != -1:
+ torch.nn.init.xavier_normal_(m.weight.data)
+ torch.nn.init.constant_(m.bias.data, 0.0)
+ try:
+ checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
+ start_epoch = checkpoint['epoch']
+ classifier.load_state_dict(checkpoint['model_state_dict'])
+ log_string('Use pretrain model')
+ except:
+ log_string('No existing model, starting training from scratch...')
+ start_epoch = 0
+ classifier = classifier.apply(weights_init)
- if args.optimizer == 'SGD':
- optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
- elif args.optimizer == 'Adam':
+ if args.optimizer == 'Adam':
optimizer = torch.optim.Adam(
- model.parameters(),
+ classifier.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=args.decay_rate
)
- scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=0.5)
-
- '''GPU selection and multi-GPU'''
- if args.multi_gpu is not None:
- device_ids = [int(x) for x in args.multi_gpu.split(',')]
- torch.backends.cudnn.benchmark = True
- model.cuda(device_ids[0])
- model = torch.nn.DataParallel(model, device_ids=device_ids)
else:
- model.cuda()
- criterion = PointNetLoss()
+ optimizer = torch.optim.SGD(classifier.parameters(), lr=args.learning_rate, momentum=0.9)
+
+ def bn_momentum_adjust(m, momentum):
+ if isinstance(m, torch.nn.BatchNorm2d) or isinstance(m, torch.nn.BatchNorm1d):
+ m.momentum = momentum
+
LEARNING_RATE_CLIP = 1e-5
+ MOMENTUM_ORIGINAL = 0.1
+ MOMENTUM_DECCAY = 0.5
+ MOMENTUM_DECCAY_STEP = args.step_size
- history = defaultdict(lambda: list())
best_acc = 0
+ global_epoch = 0
best_class_avg_iou = 0
best_inctance_avg_iou = 0
- for epoch in range(init_epoch,args.epoch):
- scheduler.step()
- lr = max(optimizer.param_groups[0]['lr'],LEARNING_RATE_CLIP)
- print('Learning rate:%f' % lr)
+ for epoch in range(start_epoch,args.epoch):
+ log_string('Epoch %d (%d/%s):' % (global_epoch + 1, epoch + 1, args.epoch))
+ '''Adjust learning rate and BN momentum'''
+ lr = max(args.learning_rate * (args.lr_decay ** (epoch // args.step_size)), LEARNING_RATE_CLIP)
+ log_string('Learning rate:%f' % lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
- for i, data in tqdm(enumerate(dataloader, 0),total=len(dataloader),smoothing=0.9):
- points, label, target, norm_plt = data
- points, label, target = Variable(points.float()),Variable(label.long()), Variable(target.long())
+ mean_correct = []
+ momentum = MOMENTUM_ORIGINAL * (MOMENTUM_DECCAY ** (epoch // MOMENTUM_DECCAY_STEP))
+ if momentum < 0.01:
+ momentum = 0.01
+ print('BN momentum updated to: %f' % momentum)
+ classifier = classifier.apply(lambda x: bn_momentum_adjust(x,momentum))
+
+ '''learning one epoch'''
+ for i, data in tqdm(enumerate(trainDataLoader), total=len(trainDataLoader), smoothing=0.9):
+ points, label, target = data
+ points = points.data.numpy()
+ points[:,:, 0:3] = provider.random_scale_point_cloud(points[:,:, 0:3])
+ points[:,:, 0:3] = provider.shift_point_cloud(points[:,:, 0:3])
+ points = torch.Tensor(points)
+ points, label, target = points.float().cuda(),label.long().cuda(), target.long().cuda()
points = points.transpose(2, 1)
- norm_plt = norm_plt.transpose(2, 1)
- points, label, target,norm_plt = points.cuda(),label.squeeze().cuda(), target.cuda(), norm_plt.cuda()
optimizer.zero_grad()
- model = model.train()
- if args.model_name == 'pointnet':
- labels_pred, seg_pred, trans_feat = model(points, to_categorical(label, 16))
- seg_pred = seg_pred.contiguous().view(-1, num_part)
- target = target.view(-1, 1)[:, 0]
- loss, seg_loss, label_loss = criterion(labels_pred, label, seg_pred, target, trans_feat)
- else:
- seg_pred = model(points, norm_plt, to_categorical(label, 16))
- seg_pred = seg_pred.contiguous().view(-1, num_part)
- target = target.view(-1, 1)[:, 0]
- loss = F.nll_loss(seg_pred, target)
-
- history['loss'].append(loss.cpu().data.numpy())
+ classifier = classifier.train()
+ seg_pred, trans_feat = classifier(points, to_categorical(label, num_classes))
+ seg_pred = seg_pred.contiguous().view(-1, num_part)
+ target = target.view(-1, 1)[:, 0]
+ pred_choice = seg_pred.data.max(1)[1]
+ correct = pred_choice.eq(target.data).cpu().sum()
+ mean_correct.append(correct.item() / (args.batch_size * args.npoint))
+ loss = criterion(seg_pred, target, trans_feat)
loss.backward()
optimizer.step()
+ train_instance_acc = np.mean(mean_correct)
+ log_string('Train accuracy is: %.5f' % train_instance_acc)
- forpointnet2 = args.model_name == 'pointnet2'
- test_metrics, test_hist_acc, cat_mean_iou = test_partseg(model.eval(), testdataloader, seg_label_to_cat,50,forpointnet2)
+ with torch.no_grad():
+ test_metrics = {}
+ total_correct = 0
+ total_seen = 0
+ total_seen_class = [0 for _ in range(num_part)]
+ total_correct_class = [0 for _ in range(num_part)]
+ shape_ious = {cat: [] for cat in seg_classes.keys()}
+ seg_label_to_cat = {} # {0:Airplane, 1:Airplane, ...49:Table}
+ for cat in seg_classes.keys():
+ for label in seg_classes[cat]:
+ seg_label_to_cat[label] = cat
- print('Epoch %d %s accuracy: %f Class avg mIOU: %f Inctance avg mIOU: %f' % (
- epoch, blue('test'), test_metrics['accuracy'],test_metrics['class_avg_iou'],test_metrics['inctance_avg_iou']))
+ for batch_id, (points, label, target) in tqdm(enumerate(testDataLoader), total=len(testDataLoader), smoothing=0.9):
+ cur_batch_size, NUM_POINT, _ = points.size()
+ points, label, target = points.float().cuda(), label.long().cuda(), target.long().cuda()
+ points = points.transpose(2, 1)
+ classifier = classifier.eval()
+ seg_pred, _ = classifier(points, to_categorical(label, num_classes))
+ cur_pred_val = seg_pred.cpu().data.numpy()
+ cur_pred_val_logits = cur_pred_val
+ cur_pred_val = np.zeros((cur_batch_size, NUM_POINT)).astype(np.int32)
+ target = target.cpu().data.numpy()
+ for i in range(cur_batch_size):
+ cat = seg_label_to_cat[target[i, 0]]
+ logits = cur_pred_val_logits[i, :, :]
+ cur_pred_val[i, :] = np.argmax(logits[:, seg_classes[cat]], 1) + seg_classes[cat][0]
+ correct = np.sum(cur_pred_val == target)
+ total_correct += correct
+ total_seen += (cur_batch_size * NUM_POINT)
+
+ for l in range(num_part):
+ total_seen_class[l] += np.sum(target == l)
+ total_correct_class[l] += (np.sum((cur_pred_val == l) & (target == l)))
+
+ for i in range(cur_batch_size):
+ segp = cur_pred_val[i, :]
+ segl = target[i, :]
+ cat = seg_label_to_cat[segl[0]]
+ part_ious = [0.0 for _ in range(len(seg_classes[cat]))]
+ for l in seg_classes[cat]:
+ if (np.sum(segl == l) == 0) and (
+ np.sum(segp == l) == 0): # part is not present, no prediction as well
+ part_ious[l - seg_classes[cat][0]] = 1.0
+ else:
+ part_ious[l - seg_classes[cat][0]] = np.sum((segl == l) & (segp == l)) / float(
+ np.sum((segl == l) | (segp == l)))
+ shape_ious[cat].append(np.mean(part_ious))
+
+ all_shape_ious = []
+ for cat in shape_ious.keys():
+ for iou in shape_ious[cat]:
+ all_shape_ious.append(iou)
+ shape_ious[cat] = np.mean(shape_ious[cat])
+ mean_shape_ious = np.mean(list(shape_ious.values()))
+ test_metrics['accuracy'] = total_correct / float(total_seen)
+ test_metrics['class_avg_accuracy'] = np.mean(
+ np.array(total_correct_class) / np.array(total_seen_class, dtype=np.float))
+ for cat in sorted(shape_ious.keys()):
+ log_string('eval mIoU of %s %f' % (cat + ' ' * (14 - len(cat)), shape_ious[cat]))
+ test_metrics['class_avg_iou'] = mean_shape_ious
+ test_metrics['inctance_avg_iou'] = np.mean(all_shape_ious)
+
+
+ log_string('Epoch %d test Accuracy: %f Class avg mIOU: %f Inctance avg mIOU: %f' % (
+ epoch+1, test_metrics['accuracy'],test_metrics['class_avg_iou'],test_metrics['inctance_avg_iou']))
+ if (test_metrics['inctance_avg_iou'] >= best_inctance_avg_iou):
+ logger.info('Save model...')
+ savepath = str(checkpoints_dir) + '/best_model.pth'
+ log_string('Saving at %s'% savepath)
+ state = {
+ 'epoch': epoch,
+ 'train_acc': train_instance_acc,
+ 'test_acc': test_metrics['accuracy'],
+ 'class_avg_iou': test_metrics['class_avg_iou'],
+ 'inctance_avg_iou': test_metrics['inctance_avg_iou'],
+ 'model_state_dict': classifier.state_dict(),
+ 'optimizer_state_dict': optimizer.state_dict(),
+ }
+ torch.save(state, savepath)
+ log_string('Saving model....')
- logger.info('Epoch %d %s Accuracy: %f Class avg mIOU: %f Inctance avg mIOU: %f' % (
- epoch, blue('test'), test_metrics['accuracy'],test_metrics['class_avg_iou'],test_metrics['inctance_avg_iou']))
if test_metrics['accuracy'] > best_acc:
best_acc = test_metrics['accuracy']
- torch.save(model.state_dict(), '%s/%s_%.3d_%.4f.pth' % (checkpoints_dir,args.model_name, epoch, best_acc))
- logger.info(cat_mean_iou)
- logger.info('Save model..')
- print('Save model..')
- print(cat_mean_iou)
if test_metrics['class_avg_iou'] > best_class_avg_iou:
best_class_avg_iou = test_metrics['class_avg_iou']
if test_metrics['inctance_avg_iou'] > best_inctance_avg_iou:
best_inctance_avg_iou = test_metrics['inctance_avg_iou']
- print('Best accuracy is: %.5f'%best_acc)
- logger.info('Best accuracy is: %.5f'%best_acc)
- print('Best class avg mIOU is: %.5f'%best_class_avg_iou)
- logger.info('Best class avg mIOU is: %.5f'%best_class_avg_iou)
- print('Best inctance avg mIOU is: %.5f'%best_inctance_avg_iou)
- logger.info('Best inctance avg mIOU is: %.5f'%best_inctance_avg_iou)
-
+ log_string('Best accuracy is: %.5f'%best_acc)
+ log_string('Best class avg mIOU is: %.5f'%best_class_avg_iou)
+ log_string('Best inctance avg mIOU is: %.5f'%best_inctance_avg_iou)
+ global_epoch+=1
if __name__ == '__main__':
args = parse_args()
diff --git a/train_semseg.py b/train_semseg.py
index 3b2a3b5df..777192053 100644
--- a/train_semseg.py
+++ b/train_semseg.py
@@ -1,162 +1,286 @@
+"""
+Author: Benny
+Date: Nov 2019
+"""
import argparse
import os
+from data_utils.S3DISDataLoader import S3DISDataset, S3DISDatasetWholeScene
import torch
-import numpy as np
-import torch.nn.parallel
-import torch.utils.data
-from collections import defaultdict
-from torch.autograd import Variable
-from data_utils.S3DISDataLoader import S3DISDataLoader, recognize_all_data,class2label
-import torch.nn.functional as F
import datetime
import logging
from pathlib import Path
-from utils import test_semseg
+import sys
+import importlib
+import shutil
from tqdm import tqdm
-from model.pointnet2 import PointNet2SemSeg
-from model.pointnet import PointNetSeg, feature_transform_reguliarzer
+import provider
+import numpy as np
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = BASE_DIR
+sys.path.append(os.path.join(ROOT_DIR, 'models'))
+
+seg_classes = {'Earphone': [16, 17, 18], 'Motorbike': [30, 31, 32, 33, 34, 35], 'Rocket': [41, 42, 43], 'Car': [8, 9, 10, 11], 'Laptop': [28, 29], 'Cap': [6, 7], 'Skateboard': [44, 45, 46], 'Mug': [36, 37], 'Guitar': [19, 20, 21], 'Bag': [4, 5], 'Lamp': [24, 25, 26, 27], 'Table': [47, 48, 49], 'Airplane': [0, 1, 2, 3], 'Pistol': [38, 39, 40], 'Chair': [12, 13, 14, 15], 'Knife': [22, 23]}
+seg_label_to_cat = {} # {0:Airplane, 1:Airplane, ...49:Table}
+for cat in seg_classes.keys():
+ for label in seg_classes[cat]:
+ seg_label_to_cat[label] = cat
-seg_classes = class2label
-seg_label_to_cat = {}
-for i,cat in enumerate(seg_classes.keys()):
- seg_label_to_cat[i] = cat
def parse_args():
- parser = argparse.ArgumentParser('PointNet')
- parser.add_argument('--batchsize', type=int, default=12, help='input batch size')
- parser.add_argument('--workers', type=int, default=4, help='number of data loading workers')
- parser.add_argument('--epoch', type=int, default=201, help='number of epochs for training')
- parser.add_argument('--pretrain', type=str, default=None,help='whether use pretrain model')
- parser.add_argument('--gpu', type=str, default='0', help='specify gpu device')
- parser.add_argument('--learning_rate', type=float, default=0.001, help='learning rate for training')
- parser.add_argument('--decay_rate', type=float, default=1e-4, help='weight decay')
- parser.add_argument('--optimizer', type=str, default='Adam', help='type of optimizer')
- parser.add_argument('--multi_gpu', type=str, default=None, help='whether use multi gpu training')
- parser.add_argument('--model_name', type=str, default='pointnet2', help='Name of model')
+ parser = argparse.ArgumentParser('Model')
+ parser.add_argument('--model', type=str, default='pointnet_sem_seg', help='model name [default: pointnet_sem_seg]')
+ parser.add_argument('--batch_size', type=int, default=12, help='Batch Size during training [default: 12]')
+ parser.add_argument('--epoch', default=1024, type=int, help='Epoch to run [default: 251]')
+ parser.add_argument('--learning_rate', default=0.001, type=float, help='Initial learning rate [default: 0.001]')
+ parser.add_argument('--gpu', type=str, default='0', help='GPU to use [default: GPU 0]')
+ parser.add_argument('--optimizer', type=str, default='Adam', help='Adam or SGD [default: Adam]')
+ parser.add_argument('--log_dir', type=str, default=None, help='Log path [default: None]')
+ parser.add_argument('--decay_rate', type=float, default=1e-4, help='weight decay [default: 1e-4]')
+ parser.add_argument('--npoint', type=int, default=8192, help='Point Number [default: 2048]')
+ parser.add_argument('--with_rgb', action='store_true', default=False, help='Whether to use RGB information [default: False]')
+ parser.add_argument('--step_size', type=int, default=200, help='Decay step for lr decay [default: every 200 epochs]')
+ parser.add_argument('--lr_decay', type=float, default=0.7, help='Decay rate for lr decay [default: 0.7]')
+ parser.add_argument('--test_area', type=int, default=5, help='Which area to use for test, option: 1-6 [default: 5]')
return parser.parse_args()
def main(args):
- os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu if args.multi_gpu is None else '0,1,2,3'
+ def log_string(str):
+ logger.info(str)
+ print(str)
+
+ '''HYPER PARAMETER'''
+ os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
+
'''CREATE DIR'''
- experiment_dir = Path('./experiment/')
+ timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
+ experiment_dir = Path('./log/')
+ experiment_dir.mkdir(exist_ok=True)
+ experiment_dir = experiment_dir.joinpath('sem_seg')
+ experiment_dir.mkdir(exist_ok=True)
+ if args.log_dir is None:
+ experiment_dir = experiment_dir.joinpath(timestr)
+ else:
+ experiment_dir = experiment_dir.joinpath(args.log_dir)
experiment_dir.mkdir(exist_ok=True)
- file_dir = Path(str(experiment_dir) +'/%sSemSeg-'%args.model_name+ str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M')))
- file_dir.mkdir(exist_ok=True)
- checkpoints_dir = file_dir.joinpath('checkpoints/')
+ checkpoints_dir = experiment_dir.joinpath('checkpoints/')
checkpoints_dir.mkdir(exist_ok=True)
- log_dir = file_dir.joinpath('logs/')
+ log_dir = experiment_dir.joinpath('logs/')
log_dir.mkdir(exist_ok=True)
'''LOG'''
args = parse_args()
- logger = logging.getLogger(args.model_name)
+ logger = logging.getLogger("Model")
logger.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
- file_handler = logging.FileHandler(str(log_dir) + '/train_%s_semseg.txt'%args.model_name)
+ file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model))
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
- logger.info('---------------------------------------------------TRANING---------------------------------------------------')
- logger.info('PARAMETER ...')
- logger.info(args)
- print('Load data...')
- train_data, train_label, test_data, test_label = recognize_all_data(test_area = 5)
-
- dataset = S3DISDataLoader(train_data,train_label)
- dataloader = torch.utils.data.DataLoader(dataset, batch_size=args.batchsize,
- shuffle=True, num_workers=int(args.workers))
- test_dataset = S3DISDataLoader(test_data,test_label)
- testdataloader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batchsize,
- shuffle=True, num_workers=int(args.workers))
-
- num_classes = 13
- blue = lambda x: '\033[94m' + x + '\033[0m'
- model = PointNet2SemSeg(num_classes) if args.model_name == 'pointnet2' else PointNetSeg(num_classes,feature_transform=True,semseg = True)
-
- if args.pretrain is not None:
- model.load_state_dict(torch.load(args.pretrain))
- print('load model %s'%args.pretrain)
- logger.info('load model %s'%args.pretrain)
- else:
- print('Training from scratch')
- logger.info('Training from scratch')
- pretrain = args.pretrain
- init_epoch = int(pretrain[-14:-11]) if args.pretrain is not None else 0
-
- if args.optimizer == 'SGD':
- optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
- elif args.optimizer == 'Adam':
+ log_string('PARAMETER ...')
+ log_string(args)
+
+ root = 'data/stanford_indoor3d/'
+ NUM_CLASSES = 13
+ NUM_POINT = args.npoint
+ BATCH_SIZE = args.batch_size
+ FEATURE_CHANNEL = 3 if args.with_rgb else 0
+
+ print("start loading training data ...")
+ TRAIN_DATASET = S3DISDataset(root, split='train', with_rgb=args.with_rgb, test_area=args.test_area, block_points=NUM_POINT)
+ print("start loading whole scene validation data ...")
+ TEST_DATASET_WHOLE_SCENE = S3DISDatasetWholeScene(root, split='test', with_rgb=args.with_rgb, test_area=args.test_area, block_points=NUM_POINT)
+
+ trainDataLoader = torch.utils.data.DataLoader(TRAIN_DATASET, batch_size=BATCH_SIZE,shuffle=True, num_workers=4)
+
+ log_string("The number of training data is: %d" % len(TRAIN_DATASET))
+ log_string("The number of test data is: %d" % len(TEST_DATASET_WHOLE_SCENE))
+
+ '''MODEL LOADING'''
+ MODEL = importlib.import_module(args.model)
+ shutil.copy('models/%s.py' % args.model, str(experiment_dir))
+ shutil.copy('models/pointnet_util.py', str(experiment_dir))
+
+ classifier = MODEL.get_model(NUM_CLASSES, with_rgb=args.with_rgb).cuda()
+ criterion = MODEL.get_loss().cuda()
+
+
+ def weights_init(m):
+ classname = m.__class__.__name__
+ if classname.find('Conv2d') != -1:
+ torch.nn.init.xavier_normal_(m.weight.data)
+ torch.nn.init.constant_(m.bias.data, 0.0)
+ elif classname.find('Linear') != -1:
+ torch.nn.init.xavier_normal_(m.weight.data)
+ torch.nn.init.constant_(m.bias.data, 0.0)
+
+ try:
+ checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
+ start_epoch = checkpoint['epoch']
+ classifier.load_state_dict(checkpoint['model_state_dict'])
+ log_string('Use pretrain model')
+ except:
+ log_string('No existing model, starting training from scratch...')
+ start_epoch = 0
+ classifier = classifier.apply(weights_init)
+
+ if args.optimizer == 'Adam':
optimizer = torch.optim.Adam(
- model.parameters(),
+ classifier.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=args.decay_rate
)
- scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
- LEARNING_RATE_CLIP = 1e-5
-
- '''GPU selection and multi-GPU'''
- if args.multi_gpu is not None:
- device_ids = [int(x) for x in args.multi_gpu.split(',')]
- torch.backends.cudnn.benchmark = True
- model.cuda(device_ids[0])
- model = torch.nn.DataParallel(model, device_ids=device_ids)
else:
- model.cuda()
+ optimizer = torch.optim.SGD(classifier.parameters(), lr=args.learning_rate, momentum=0.9)
- history = defaultdict(lambda: list())
- best_acc = 0
- best_meaniou = 0
+ def bn_momentum_adjust(m, momentum):
+ if isinstance(m, torch.nn.BatchNorm2d) or isinstance(m, torch.nn.BatchNorm1d):
+ m.momentum = momentum
- for epoch in range(init_epoch,args.epoch):
- scheduler.step()
- lr = max(optimizer.param_groups[0]['lr'],LEARNING_RATE_CLIP)
- print('Learning rate:%f' % lr)
+ LEARNING_RATE_CLIP = 1e-5
+ MOMENTUM_ORIGINAL = 0.1
+ MOMENTUM_DECCAY = 0.5
+ MOMENTUM_DECCAY_STEP = args.step_size
+
+ global_epoch = 0
+ best_iou = 0
+
+ for epoch in range(start_epoch,args.epoch):
+ log_string('Epoch %d (%d/%s):' % (global_epoch + 1, epoch + 1, args.epoch))
+ '''Adjust learning rate and BN momentum'''
+ lr = max(args.learning_rate * (args.lr_decay ** (epoch // args.step_size)), LEARNING_RATE_CLIP)
+ log_string('Learning rate:%f' % lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
- for i, data in tqdm(enumerate(dataloader, 0),total=len(dataloader),smoothing=0.9):
- points, target = data
- points, target = Variable(points.float()), Variable(target.long())
+ mean_correct = []
+ momentum = MOMENTUM_ORIGINAL * (MOMENTUM_DECCAY ** (epoch // MOMENTUM_DECCAY_STEP))
+ if momentum < 0.01:
+ momentum = 0.01
+ print('BN momentum updated to: %f' % momentum)
+ classifier = classifier.apply(lambda x: bn_momentum_adjust(x,momentum))
+
+ '''learning one epoch'''
+ for i, data in tqdm(enumerate(trainDataLoader), total=len(trainDataLoader), smoothing=0.9):
+ points, target, weight = data
+ points = points.data.numpy()
+ points[:, :, :3] = provider.normalize_data(points[:, :, :3])
+ points[:,:, 0:3] = provider.random_scale_point_cloud(points[:,:, 0:3])
+ points[:,:, 0:3] = provider.rotate_point_cloud_z(points[:,:, 0:3])
+ points = torch.Tensor(points)
+ points, target = points.float().cuda(),target.long().cuda()
points = points.transpose(2, 1)
- points, target = points.cuda(), target.cuda()
optimizer.zero_grad()
- model = model.train()
- if args.model_name == 'pointnet':
- pred, trans_feat = model(points)
- else:
- pred = model(points[:,:3,:],points[:,3:,:])
- pred = pred.contiguous().view(-1, num_classes)
+ classifier = classifier.train()
+ seg_pred, trans_feat = classifier(points)
+ seg_pred = seg_pred.contiguous().view(-1, NUM_CLASSES)
target = target.view(-1, 1)[:, 0]
- loss = F.nll_loss(pred, target)
- if args.model_name == 'pointnet':
- loss += feature_transform_reguliarzer(trans_feat) * 0.001
- history['loss'].append(loss.cpu().data.numpy())
+ pred_choice = seg_pred.data.max(1)[1]
+ correct = pred_choice.eq(target.data).cpu().sum()
+ mean_correct.append(correct.item() / (args.batch_size * args.npoint))
+ loss = criterion(seg_pred, target, trans_feat)
loss.backward()
optimizer.step()
- pointnet2 = args.model_name == 'pointnet2'
- test_metrics, test_hist_acc, cat_mean_iou = test_semseg(model.eval(), testdataloader, seg_label_to_cat,num_classes = num_classes,pointnet2=pointnet2)
- mean_iou = np.mean(cat_mean_iou)
- print('Epoch %d %s accuracy: %f meanIOU: %f' % (
- epoch, blue('test'), test_metrics['accuracy'],mean_iou))
- logger.info('Epoch %d %s accuracy: %f meanIOU: %f' % (
- epoch, 'test', test_metrics['accuracy'],mean_iou))
- if test_metrics['accuracy'] > best_acc:
- best_acc = test_metrics['accuracy']
- torch.save(model.state_dict(), '%s/%s_%.3d_%.4f.pth' % (checkpoints_dir,args.model_name, epoch, best_acc))
- logger.info(cat_mean_iou)
- logger.info('Save model..')
- print('Save model..')
- print(cat_mean_iou)
- if mean_iou > best_meaniou:
- best_meaniou = mean_iou
- print('Best accuracy is: %.5f'%best_acc)
- logger.info('Best accuracy is: %.5f'%best_acc)
- print('Best meanIOU is: %.5f'%best_meaniou)
- logger.info('Best meanIOU is: %.5f'%best_meaniou)
+ train_instance_acc = np.mean(mean_correct)
+ log_string('Train accuracy is: %.5f' % train_instance_acc)
+
+ # evaluate on whole scenes, for each block, only sample NUM_POINT points
+ if epoch % 10 ==0:
+ with torch.no_grad():
+ num_batches = len(TEST_DATASET_WHOLE_SCENE)
+
+ total_correct = 0
+ total_seen = 0
+ loss_sum = 0
+ total_seen_class = [0 for _ in range(NUM_CLASSES)]
+ total_correct_class = [0 for _ in range(NUM_CLASSES)]
+ total_iou_deno_class = [0 for _ in range(NUM_CLASSES)]
+
+ labelweights = np.zeros(NUM_CLASSES)
+ is_continue_batch = False
+
+ extra_batch_data = np.zeros((0, NUM_POINT, 3 + FEATURE_CHANNEL))
+ extra_batch_label = np.zeros((0, NUM_POINT))
+ extra_batch_smpw = np.zeros((0, NUM_POINT))
+ for batch_idx in tqdm(range(num_batches),total=num_batches):
+ if not is_continue_batch:
+ batch_data, batch_label, batch_smpw = TEST_DATASET_WHOLE_SCENE[batch_idx]
+ batch_data = np.concatenate((batch_data, extra_batch_data), axis=0)
+ batch_label = np.concatenate((batch_label, extra_batch_label), axis=0)
+ batch_smpw = np.concatenate((batch_smpw, extra_batch_smpw), axis=0)
+ else:
+ batch_data_tmp, batch_label_tmp, batch_smpw_tmp = TEST_DATASET_WHOLE_SCENE[batch_idx]
+ batch_data = np.concatenate((batch_data, batch_data_tmp), axis=0)
+ batch_label = np.concatenate((batch_label, batch_label_tmp), axis=0)
+ batch_smpw = np.concatenate((batch_smpw, batch_smpw_tmp), axis=0)
+ if batch_data.shape[0] < BATCH_SIZE:
+ is_continue_batch = True
+ continue
+ elif batch_data.shape[0] == BATCH_SIZE:
+ is_continue_batch = False
+ extra_batch_data = np.zeros((0, NUM_POINT, 3 + FEATURE_CHANNEL))
+ extra_batch_label = np.zeros((0, NUM_POINT))
+ extra_batch_smpw = np.zeros((0, NUM_POINT))
+ else:
+ is_continue_batch = False
+ extra_batch_data = batch_data[BATCH_SIZE:, :, :]
+ extra_batch_label = batch_label[BATCH_SIZE:, :]
+ extra_batch_smpw = batch_smpw[BATCH_SIZE:, :]
+ batch_data = batch_data[:BATCH_SIZE, :, :]
+ batch_label = batch_label[:BATCH_SIZE, :]
+ batch_smpw = batch_smpw[:BATCH_SIZE, :]
+
+ batch_data[:, :, :3] = provider.normalize_data(batch_data[:, :, :3])
+ batch_label = torch.Tensor(batch_label)
+ batch_data = torch.Tensor(batch_data)
+ batch_data, batch_label = batch_data.float().cuda(), batch_label.long().cuda()
+ batch_data = batch_data.transpose(2, 1)
+ seg_pred, _ = classifier(batch_data)
+ seg_pred = seg_pred.contiguous()
+ batch_label = batch_label.cpu().data.numpy()
+ pred_val = seg_pred.cpu().data.max(2)[1].numpy()
+ correct = np.sum((pred_val == batch_label) & (batch_smpw > 0))
+ total_correct += correct
+ total_seen += np.sum(batch_smpw > 0)
+ tmp, _ = np.histogram(batch_label, range(NUM_CLASSES + 1))
+ labelweights += tmp
+ for l in range(NUM_CLASSES):
+ total_seen_class[l] += np.sum((batch_label == l) & (batch_smpw > 0))
+ total_correct_class[l] += np.sum((pred_val == l) & (batch_label == l) & (batch_smpw > 0))
+ total_iou_deno_class[l] += np.sum(((pred_val == l) | (batch_label == l)) & (batch_smpw > 0))
+
+ mIoU = np.mean(np.array(total_correct_class) / (np.array(total_iou_deno_class, dtype=np.float) + 1e-6))
+ log_string('eval whole scene mean loss: %f' % (loss_sum / float(num_batches)))
+ log_string('eval point avg class IoU: %f' % mIoU)
+ log_string('eval whole scene point accuracy: %f' % (total_correct / float(total_seen)))
+ log_string('eval whole scene point avg class acc: %f' % (
+ np.mean(np.array(total_correct_class) / (np.array(total_seen_class, dtype=np.float) + 1e-6))))
+ labelweights = labelweights.astype(np.float32) / np.sum(labelweights.astype(np.float32))
+
+ iou_per_class_str = '------- IoU --------\n'
+ for l in range(NUM_CLASSES):
+ iou_per_class_str += 'class %s weight: %.3f, IoU: %.3f \n' % (
+ seg_label_to_cat[l] + ' ' * (14 - len(seg_label_to_cat[l])), labelweights[l],
+ total_correct_class[l] / float(total_iou_deno_class[l]))
+ log_string(iou_per_class_str)
+
+ if (mIoU >= best_iou):
+ logger.info('Save model...')
+ savepath = str(checkpoints_dir) + '/best_model.pth'
+ log_string('Saving at %s' % savepath)
+ state = {
+ 'epoch': epoch,
+ 'class_avg_iou': mIoU,
+ 'model_state_dict': classifier.state_dict(),
+ 'optimizer_state_dict': optimizer.state_dict(),
+ }
+ torch.save(state, savepath)
+ log_string('Saving model....')
if __name__ == '__main__':
args = parse_args()
main(args)
+
diff --git a/utils.py b/utils.py
deleted file mode 100644
index 87afafc9f..000000000
--- a/utils.py
+++ /dev/null
@@ -1,229 +0,0 @@
-# *_*coding:utf-8 *_*
-import os
-import numpy as np
-import torch
-import matplotlib.pyplot as plt
-from torch.autograd import Variable
-from tqdm import tqdm
-from collections import defaultdict
-import datetime
-import pandas as pd
-import torch.nn.functional as F
-def to_categorical(y, num_classes):
- """ 1-hot encodes a tensor """
- new_y = torch.eye(num_classes)[y.cpu().data.numpy(),]
- if (y.is_cuda):
- return new_y.cuda()
- return new_y
-
-def show_example(x, y, x_reconstruction, y_pred,save_dir, figname):
- x = x.squeeze().cpu().data.numpy()
- x = x.permute(0,2,1)
- y = y.cpu().data.numpy()
- x_reconstruction = x_reconstruction.squeeze().cpu().data.numpy()
- _, y_pred = torch.max(y_pred, -1)
- y_pred = y_pred.cpu().data.numpy()
-
- fig, ax = plt.subplots(1, 2)
- ax[0].imshow(x, cmap='Greys')
- ax[0].set_title('Input: %d' % y)
- ax[1].imshow(x_reconstruction, cmap='Greys')
- ax[1].set_title('Output: %d' % y_pred)
- plt.savefig(save_dir + figname + '.png')
-
-def save_checkpoint(epoch, train_accuracy, test_accuracy, model, optimizer, path,modelnet='checkpoint'):
- savepath = path + '/%s-%f-%04d.pth' % (modelnet,test_accuracy, epoch)
- state = {
- 'epoch': epoch,
- 'train_accuracy': train_accuracy,
- 'test_accuracy': test_accuracy,
- 'model_state_dict': model.state_dict(),
- 'optimizer_state_dict': optimizer.state_dict(),
- }
- torch.save(state, savepath)
-
-def test(model, loader):
- mean_correct = []
- for j, data in enumerate(loader, 0):
- points, target = data
- target = target[:, 0]
- points = points.transpose(2, 1)
- points, target = points.cuda(), target.cuda()
- classifier = model.eval()
- pred, _ = classifier(points)
- pred_choice = pred.data.max(1)[1]
- correct = pred_choice.eq(target.long().data).cpu().sum()
- mean_correct.append(correct.item()/float(points.size()[0]))
- return np.mean(mean_correct)
-
-def compute_cat_iou(pred,target,iou_tabel):
- iou_list = []
- target = target.cpu().data.numpy()
- for j in range(pred.size(0)):
- batch_pred = pred[j]
- batch_target = target[j]
- batch_choice = batch_pred.data.max(1)[1].cpu().data.numpy()
- for cat in np.unique(batch_target):
- # intersection = np.sum((batch_target == cat) & (batch_choice == cat))
- # union = float(np.sum((batch_target == cat) | (batch_choice == cat)))
- # iou = intersection/union if not union ==0 else 1
- I = np.sum(np.logical_and(batch_choice == cat, batch_target == cat))
- U = np.sum(np.logical_or(batch_choice == cat, batch_target == cat))
- if U == 0:
- iou = 1 # If the union of groundtruth and prediction points is empty, then count part IoU as 1
- else:
- iou = I / float(U)
- iou_tabel[cat,0] += iou
- iou_tabel[cat,1] += 1
- iou_list.append(iou)
- return iou_tabel,iou_list
-
-def compute_overall_iou(pred, target, num_classes):
- shape_ious = []
- pred_np = pred.cpu().data.numpy()
- target_np = target.cpu().data.numpy()
- for shape_idx in range(pred.size(0)):
- part_ious = []
- for part in range(num_classes):
- I = np.sum(np.logical_and(pred_np[shape_idx].max(1) == part, target_np[shape_idx] == part))
- U = np.sum(np.logical_or(pred_np[shape_idx].max(1) == part, target_np[shape_idx] == part))
- if U == 0:
- iou = 1 #If the union of groundtruth and prediction points is empty, then count part IoU as 1
- else:
- iou = I / float(U)
- part_ious.append(iou)
- shape_ious.append(np.mean(part_ious))
- return shape_ious
-
-def test_partseg(model, loader, catdict, num_classes = 50,forpointnet2=False):
- ''' catdict = {0:Airplane, 1:Airplane, ...49:Table} '''
- iou_tabel = np.zeros((len(catdict),3))
- iou_list = []
- metrics = defaultdict(lambda:list())
- hist_acc = []
- # mean_correct = []
- for batch_id, (points, label, target, norm_plt) in tqdm(enumerate(loader), total=len(loader), smoothing=0.9):
- batchsize, num_point,_= points.size()
- points, label, target, norm_plt = Variable(points.float()),Variable(label.long()), Variable(target.long()),Variable(norm_plt.float())
- points = points.transpose(2, 1)
- norm_plt = norm_plt.transpose(2, 1)
- points, label, target, norm_plt = points.cuda(), label.squeeze().cuda(), target.cuda(), norm_plt.cuda()
- if forpointnet2:
- seg_pred = model(points, norm_plt, to_categorical(label, 16))
- else:
- labels_pred, seg_pred, _ = model(points,to_categorical(label,16))
- # labels_pred_choice = labels_pred.data.max(1)[1]
- # labels_correct = labels_pred_choice.eq(label.long().data).cpu().sum()
- # mean_correct.append(labels_correct.item() / float(points.size()[0]))
- # print(pred.size())
- iou_tabel, iou = compute_cat_iou(seg_pred,target,iou_tabel)
- iou_list+=iou
- # shape_ious += compute_overall_iou(pred, target, num_classes)
- seg_pred = seg_pred.contiguous().view(-1, num_classes)
- target = target.view(-1, 1)[:, 0]
- pred_choice = seg_pred.data.max(1)[1]
- correct = pred_choice.eq(target.data).cpu().sum()
- metrics['accuracy'].append(correct.item()/ (batchsize * num_point))
- iou_tabel[:,2] = iou_tabel[:,0] /iou_tabel[:,1]
- hist_acc += metrics['accuracy']
- metrics['accuracy'] = np.mean(hist_acc)
- metrics['inctance_avg_iou'] = np.mean(iou_list)
- # metrics['label_accuracy'] = np.mean(mean_correct)
- iou_tabel = pd.DataFrame(iou_tabel,columns=['iou','count','mean_iou'])
- iou_tabel['Category_IOU'] = [catdict[i] for i in range(len(catdict)) ]
- cat_iou = iou_tabel.groupby('Category_IOU')['mean_iou'].mean()
- metrics['class_avg_iou'] = np.mean(cat_iou)
-
- return metrics, hist_acc, cat_iou
-
-def test_semseg(model, loader, catdict, num_classes = 13, pointnet2=False):
- iou_tabel = np.zeros((len(catdict),3))
- metrics = defaultdict(lambda:list())
- hist_acc = []
- for batch_id, (points, target) in tqdm(enumerate(loader), total=len(loader), smoothing=0.9):
- batchsize, num_point, _ = points.size()
- points, target = Variable(points.float()), Variable(target.long())
- points = points.transpose(2, 1)
- points, target = points.cuda(), target.cuda()
- if pointnet2:
- pred = model(points[:, :3, :], points[:, 3:, :])
- else:
- pred, _ = model(points)
- # print(pred.size())
- iou_tabel, iou_list = compute_cat_iou(pred,target,iou_tabel)
- # shape_ious += compute_overall_iou(pred, target, num_classes)
- pred = pred.contiguous().view(-1, num_classes)
- target = target.view(-1, 1)[:, 0]
- pred_choice = pred.data.max(1)[1]
- correct = pred_choice.eq(target.data).cpu().sum()
- metrics['accuracy'].append(correct.item()/ (batchsize * num_point))
- iou_tabel[:,2] = iou_tabel[:,0] /iou_tabel[:,1]
- hist_acc += metrics['accuracy']
- metrics['accuracy'] = np.mean(metrics['accuracy'])
- metrics['iou'] = np.mean(iou_tabel[:, 2])
- iou_tabel = pd.DataFrame(iou_tabel,columns=['iou','count','mean_iou'])
- iou_tabel['Category_IOU'] = [catdict[i] for i in range(len(catdict)) ]
- # print(iou_tabel)
- cat_iou = iou_tabel.groupby('Category_IOU')['mean_iou'].mean()
-
- return metrics, hist_acc, cat_iou
-
-
-def compute_avg_curve(y, n_points_avg):
- avg_kernel = np.ones((n_points_avg,)) / n_points_avg
- rolling_mean = np.convolve(y, avg_kernel, mode='valid')
- return rolling_mean
-
-def plot_loss_curve(history,n_points_avg,n_points_plot,save_dir):
- curve = np.asarray(history['loss'])[-n_points_plot:]
- avg_curve = compute_avg_curve(curve, n_points_avg)
- plt.plot(avg_curve, '-g')
-
- curve = np.asarray(history['margin_loss'])[-n_points_plot:]
- avg_curve = compute_avg_curve(curve, n_points_avg)
- plt.plot(avg_curve, '-b')
-
- curve = np.asarray(history['reconstruction_loss'])[-n_points_plot:]
- avg_curve = compute_avg_curve(curve, n_points_avg)
- plt.plot(avg_curve, '-r')
-
- plt.legend(['Total Loss', 'Margin Loss', 'Reconstruction Loss'])
- plt.savefig(save_dir + '/'+ str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M')) + '_total_result.png')
- plt.close()
-
-def plot_acc_curve(total_train_acc,total_test_acc,save_dir):
- plt.plot(total_train_acc, '-b',label = 'train_acc')
- plt.plot(total_test_acc, '-r',label = 'test_acc')
- plt.legend()
- plt.ylabel('acc')
- plt.xlabel('epoch')
- plt.title('Accuracy of training and test')
- plt.savefig(save_dir +'/'+ str(datetime.datetime.now().strftime('%Y-%m-%d %H-%M'))+'_total_acc.png')
- plt.close()
-
-def show_point_cloud(tuple,seg_label=[],title=None):
- import matplotlib.pyplot as plt
- if seg_label == []:
- x = [x[0] for x in tuple]
- y = [y[1] for y in tuple]
- z = [z[2] for z in tuple]
- ax = plt.subplot(111, projection='3d')
- ax.scatter(x, y, z, c='b', cmap='spectral')
- ax.set_zlabel('Z')
- ax.set_ylabel('Y')
- ax.set_xlabel('X')
- else:
- category = list(np.unique(seg_label))
- color = ['b','r','g','y','w','b','p']
- ax = plt.subplot(111, projection='3d')
- for categ_index in range(len(category)):
- tuple_seg = tuple[seg_label == category[categ_index]]
- x = [x[0] for x in tuple_seg]
- y = [y[1] for y in tuple_seg]
- z = [z[2] for z in tuple_seg]
- ax.scatter(x, y, z, c=color[categ_index], cmap='spectral')
- ax.set_zlabel('Z')
- ax.set_ylabel('Y')
- ax.set_xlabel('X')
- plt.title(title)
- plt.show()
\ No newline at end of file
diff --git a/visualizer/example.jpg b/visualizer/example.jpg
deleted file mode 100644
index 10ccb46acad94380a6805c3dc405140f661031dc..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 19459
zcmeIZcUV)~wmuwc04ah50jVYm1f+v>0i`OPgpM>piu7Kjt3-Mg#DLTU2uKMdI=$
z00II4fZ+TKIGY0829S}G0!c~8fIuKQIT-~dJr(7J3zSTBmuTo&nc3M`nORskApBe$
zSNXs!EZjGF_ymPSMMc@UBxNOpW%xxzh0mG*7sv?qNu~)1XaQ$!0QU2nlMwv%1NhsA
zfRKoogcL|dPI2M<1nflsApsE)Au$mN2{G~c*+J*+0AgAaI!+-4(n~s)z^iWb!l4PT
z$-s)`%^=;6+Yk{e_b_q_hRcjh%-61Saq~b$#l&w)NJ`zlqok~&s-~`|4}%*R8W~&L
zJg~KUXz$?R>E-R?>*pUH5&7ilvnWJjQgTXa+KcpzoZP(pf{AU4pRkT*UrIxkWpR7x#-5ep_A%DtaQ*BjW?+-T{t!*w&$AFXe=YUTSd=p>4Nh3|_~SaPA;4KherxCa~M
zMj}p(d^>ft&9kS)4>k}@@9rm$)B1+;p?$oZ~)ed&E(F1~?*n7Azivqx-lcCxj_A8rnI>Z}=~;{>jS;A9=;wo;ooz9F2(zP!HeO15Tr
z)M=w6O`XzYF8-~~WVDF+PvVGEDYx1)0M;x5Qf-0)IZXdJ1Awo@xHLOq*o+jo8toi}
z$WF8g*ivq9a4x_s!*W?X#h#+ecVnd~7WtL=Spr7%|0Lcx4syr*ILh~`j&5}Y))+T{
zY3U!x_#9%$khVJ2$^g%UqP?p8uzChoVTa6}P+DH2n?=NOb)c3RO4HgS)WiST{A=@wH9fmlNDDvVMIQr(NUAHb^Mk(^j=(O7dE?vo{!m`K%
zy_m6YLGJ$gs|&13Nge>Xkc=nysqUk$UG-QuV1c>4cCf*%$7N}9Cn=8Pnsp2*0=hfJ
z^o`6|7AMU~A1|QrPYg97G4W@BDxrJ7V_uCYaMCnj{J!m+5Z6%Eqh@CsML$p(AX)%+
z`b9e8cIJMkao0PG`UHRPPKWkJNQX*x>Gh)@a&v50^Z(iLv@zd)5IxHWb*(#jwqNGH
z9UQhXL`+V`;_lX6y4t|WY=T#o@`r<0qllNe6sj`WZW-r|k-G#fRS-tIwzA5$tDx+8
zT)10eO8n0N#)z?xw@frobftbMp@jeQ_O$tdWt+z
zoRI7r-YhK+e?*7(xP-r&u+J@<~9}cUWb|DOpeweXZqOj@HnLw*^lROl-9D6ioZj!9>O^=t7wf1@4!eEwTC5~BAGSau?rwSfWOD*TR#fmJQB)1j5fUhHts4AQIn?hTfv3Fmh
zJ*pBlMJC{hpBmiK0t?x&QC=PufzC6V8Vc95S8Bi7<^-8`6!HOaLlyiDYL28nJp^0G
zgFy!*w_0)upW!}>D{pF|a;oB{yT4Ue8YEU0J^-OP^0d7_VIbi=Q`Hk)d7goC0MO~&+g9D-d
zYybYe^=%oMQ8#4PJ(1KuZttK>(Gp_H03kR&%;Vv*u#%IpMwJZSpWK!6{MQdhgFS+`yYGTPj<892c^?e#K0
zOH_j-YU$el{gJ;VR5hnk@-$gn1||gjv&W~!iXH4y+9uhuysu3R153lo1#q-#yW3O-
zR5koSw>b6wkr%4u-+4nuZsES($H0F_SAX(M9(l+-WyY%;M(>V55;YelU*Y4_Ws2-$
zbA3DW4SWy3^CbU~DA%vv#W`wer)e4O(q{RGYV+T|zpt
z{iMtP2ZA#n6gc0{mbP>`Io-^EXr!Lw=7knj)mzV(vNjE*39FaewYRTDcmxWIWlhZ(vHVM%`jj(KZ|UF@6e@q
z!Ndw1>(^JSI*R#d=XpT~rPA$EI5HCs{5C-uxIVq`ZK5(qVrQZHlix)wnU^hOTq%b;
zO7*BTN0JD|7FKqfx}8sx#>Kz3TvJ8+G4?{+^Y~A5Yd`2sL4uEP?y++NZ?>ARGzN?_
z1v&S-uLF&fs-tGypR1?%1&1kFKd=VL7;7&z*3G7aSVeT;RgHOLSB@
zmv0#QyrLtQPqN>k4xw1o6h%+1qNn#vEqV#UakOFTf)L%3;?jvX^eVfKydLHg_4akU
z%J2CPRv(~y1`v5nSk~!~9Si-QcSoIY?EBk!YB_9mMPOcON?pc}aN!+uwAA
zUtr8(gaN*0lpneRAwl-EFcYew{0F*nFcif_4^?GqCB`l`EW@Ilmj5-5QIFZ2$4`o}
z28pl#0Jgj;D#JD-0+#_Ns_q-#d517ou@}5z3_?hS;GaH?>O3Qo>~QE~(BMN%bjyUC
zM@8Rtt|b8Vv~8*JRJgZDG*S&Tl3(G+D6wj45OvkCm#w40#0=n&Xpz;qX7oGKq8oYR
z!zH~|r}cScI>z@Z1&OJW!dQu-kYLlA8C529hemHQbHFOTl*-ZAlv^Q{{gu?`6O^LG
z%24?OjYQVM%`vatXu3u!n|OMgSo>H|+Gc`LvZ@oy&uZ6~sHj@E~PV`zfkJ?ELvLU);6%QXwOgKMn6dIw{dW<#KN=}#<
zHUu@Jx_T1*M+7qWbg*|i!(r2l#nLvz|A&tK~TS
z4eP+&LISbaU!BEZ58Ed1v+I}!KieWwJ_Cqr(uge{?Y(^diDjh}AX^SY{gM_uJIXdr
z1Q{bW*m*YtrFTGee-2>An@e`GMGxIpDB<6$@z33p2Hhqx>w93@!bJyPKC=4_P2cTJ
z{cgl``jla82k`Ia1!+Jp-M?rf2JTR5+E+$uN{?LS@VlCmF~j7$T;Y6!u$%+NrTBF6
z52@&1BGtIMJYyk*=%H*EX3gxngg#w%mfl#ERkdb_$7d2v4u#IW0vW!{efFNravWuD
zre}_lGM+MkIfRFc_b5`_y4Lc&R0R4$Y)&@sV70yf4-mOTumb|l)NkO!Sk!l54u|NA
z{@ySMDlB+d`Gr_~j20O|jh$G1OoOFnO0K8z|I9g@I}MH`ASxg+`*t(6=Xq?9^yoGi>URmNepg+EH^gi-Yv*ZMTNpAGDz5695CW%#pAvGH&
z85#wcGwB%dN^A%c8jbHG1wJORGwUF>V^w;PyFnUqleM!^-R*;}-_@diz>QE?UL|Ei
z^eEH1bQx*oCkhr?jem6JXFGwe-2&5!4k}fzNblLPP5&)CK@M|TwY6X;8yfiH+Y2`+
z7)L~3*d90xAwF6!U5?8)qIG7!?lE^YRYL8vs-+rFQZ7OL2t|y?)yx8|#}6NH3pcPW
zrYCynL>Hy82i$rk3^yPe%dy@~_zb0!Z3K$nJ`IIh4kxn3@4g5a**
zn-v5}TPkK3j2osu8rif}EQ*NsFnSrCSV0FwmjpQTeRD)hhSCdSafN(ifO3r11H+KakY-A+}&&F}j>+!E#jMPB87E8C8F&
z{ce}rs;&s$DxRx1Q@cD8AB&~j;iV?AoSlkSyD&afS^Z}HR|#;&1+`66h>%Fq&|8h0
z%{LABm=~%GbB<0O^W$2^Qg@(_z${(`cNd;bu$Cz7mwM=LcR!KuDpin4DfM3-FoCzC
zr`Doj{^u}Ytu4j&Us^NjjFvZ*<_xyRD?NdlH5x8gC5~Q|m?SZzEmL5W`-Hyivt6|E
z0bw1bFblYE!BY(GSm+v%LU?XMNa88|^{-W{QVRwkOseH?6TyrPPaE9ZzEV8zBXLy#
z%B4_#8f5))RcoF^0epzM_Hq{0>s=gAt%cR`n;G!eh8*Mz>)YUmep5A!jo~pKMcvOv
zA_lcZvhqE17dCqDToFDm>h^!hp!$(?ohuUIouI-Vmv4K154>_hbvEQH98(%8nk^au
z2ry79Ty-)W;F7_kO}d96
zw}OJDsCG0jP<}lFAV-ze=K5x6^^i>?8a|SbUc1k9%R(Iummn4MbxeE?Mk}e5CvA2K
zUwGwkI@+Q1lt*93Wl?Jh6N(%4T<@eZ<_}omK}vg^lj&XB9{N%v-H&*qBb$!Ua&4^`
zdp7vjq@h(}LVVGSMW)=!>F16O*j&>^fNLVV3z}pS;W0tsgXk(A(he?tP
zhf0k<8{C9k6i075dXKk?mxnGdO}+c_sd{)Z1W>+lzkpZ4So9t-=Qn1n$u#L$5v0<+
zM`7F$cW7oVHAVQw?G-?HmKBss$s%9EMcw9Zl!6Svj}@KPuGTZp_M#t7VY=&~%SWLJ
zbUsE$zHv#>PufT24M#C{11BptOI9M?FIL1ZzFBz}qef5>L}J(PzME+ixsUpV`fy`z
zT)F#5cUz@P{;`$xH-ma#mDX*DTSo#)TMz*HoZA}z+-aRvmN#;T|2f49H^mWS8Q;#$
zFrl}L&4gWkLi<$v`(iDF_HIt?wo&o1bs#ExBYR;}%eTN+Fxom+=^Z$xU^icT8ogNcI?KX>8gC(#wFGC%M#?ZWcK|UumiR8$`OF=p
zh=!j{T1-f_+qfhej7|7qmneEjbnBHlgX)`I$#qT1qWbIM5W!B%jMrYiaNz+3iTNZ`
z+v3u@4ed_(QMjvbuGQ4mTMVlmQ(U^I?FSi6E>~jVI4x?Cm$FJ&AVTEIScDt7Q;PjY
zodk+!xPI^Qplq?#l%`=*%bqdqRAeXPQL&skX^<2mB2%y(d~>rF#irOp!ojL0Ss+f%
zo-YESBN39bg=Do9Ry%2#(O=@$=gRahF8yIulsX@K@OWUBkMew)UGC<>_$&pyU7`{v
z{ZfLcaI9jJS{r!~Db$;g;m*y|X{_&U{v@o^X9&}Iv$y{aHKhBqrlT?sX%9wvGwC7z
zL;NNqi2ksR#<;Vwip0md%)f&J-JJKc!Mm|wwZh>N2|LtBYHZ#}aDF~Y5}>=V$}Whr
zTn7-3;_C$XXKkaNBrR)}6Qs(N?-vW?7OB3u`e0CR)%cCC&T009SVGY%jg^l@%D&iB
znTMZGSiG;7^Lthyq_`T2RhA;M1`3arXJmE<{qOYu_7p${J6xWHUkquSl`WR{Y4gd(>~TkS;I4VK~V
zPVXga53rSfwl$#QHksfp>6
zREbHRI9&%6@Nk{5n$_`;2oR6!VKY2fIUru61Qg!iX1
zOg4qbnIh7!rwsPQ(s*RNdNe@%#KPp3=Wtt0Vm`G;2dSV#o2|In&;vv7LE~7H7GWZW
zPK-`nV_=qGw3k@l7oS8k$vYC}$6H2G&g4hvI~>%R2uBXREOQ=tM&NdVGr?+-s)$hy
z9RbpjXUfC&=b64HS!YhtF|1zNYKYY9mLl)#>MCL)#JhOn`Pf@2s;=*OPO0udnk->w
zfah+Xc`8%|UqDmjM5OqeynRRU$yi!B{UacI;1~T2A+_X@NP1q
z33ukU(ezZcw_Zsag*HjjXS0iN)nX(-iujSsTwE87hVGbz524ixif!iijC|lUT;8~tF`%G^O?m3wknViHYGQ!PP=)x8q{
z*D|FYkk&txGjX1`W4$%l$lUP!g^jLT4Y0OLv};KX{47jRsOi+<;H9$Iy>epO)!5|{
zd2Po;m#J~o*KrS`sqaZO>Npwr_6tTr>G4b#HV<%ILS;Wpc)fNmrddS$E<}%Z)FU%Y
zN?-l;W4ER)xOMGIw_SQJbccPxn63Lp-HRhJ(Ex+*w(}|gtEX0E7U)i`V2p}1kkd$h
za0w<8+8FX<(;>4|CuD`#Lq|BVS(+^@@2)mG>(l&uAAX#2umUjjl^2*Zy$%a8D2h&2
zsqYT@HDn^O{uCc7hbh+^f^dj0e5c}Z3>zt+ghH00V|Rm3q4JHHH$5zk@-fJ*I{6ZDwn63Bw*NF+nV=8OfStNWi_EgNlw!I=FBwCJweNFv>WFffFFGs
z@NAs1r3SrquddN_JKU$nzrXg~MEcfb`KB`@IkTFesFraMcdv_1dk7Frf#GXqbpX2p
zSVqdpi`*=#zu2pE|4Mx?8(2J=!>LGv_HlaLC
z)03BISp&ovAa6ILY6MN$2=K#K_(%!bl_vnLQls|#ka&ws
zisulY(&w#p52v_tt?Dl*QU16O@G0ItO?(69V4?AUqq(&iz5y$^7pv(p%u1!=^EASa
zBXqXBH1>l6jUlA?{ij!_*o1Ce}JJWfP7Q9HOgX_zAThvg1vDbjO5f
z&p+iBHKMWx>lM%%%P?6b|LaMmck~~{)0*@zl4VdO3a&QyTbLgg49woUD;ysCBe|==
z-U86x6HOV7R5HizQIxl
z?0#iwHKWjtd)lubmSMRzSu8p)U|c%Rf*&<1`v|VNy?;Pmv(y2j^YN8?Y
zBRd1y;hD5%*P|A_tJA?5x2rMFMie=R
zogiarU)Y1WuoZ$J;K^e4d)JTLTwDdg6i=?W@jcM=W#u8(y%!iF3EZ+yf(KUyqRs%%
zE;U~*HL7w;pzoTHrG}37NAO1`odHUr5ek^8Ks#b;73+fnEmzjDdlw^idD1QcIel9w
zlZ>PyyAlq=yKkk~s;Ui0Frn^74$6!W;+5@t(Y#ByNjqPo!Raq!CfE+_1t3p1tGDuD
zZ+8eM`PGwRLtuAyMu42T@fNPCYsMMDZ5PO<9P3rGws<;ksejB`q>6T1mX4VXVa9s}
zvzwaJ#Vf-!A6re2gJs*&gk2IU9H2iX&wm_5O54RpZ}Y0pj$U-V#azR38;lqq7C?@+
z`Ssb=MM*-^x{01QLc5UU
z1wPK<(R8QsMcIj?Dg2nENFWuf
z{!7+hEy^$@?X_J0T!m1!F$lUS4W`oqGGU4(lPp&6y-QGMtrdF;w$c`s?-WZ53{R8_
zmRj(lIsTEh_Scd!n$Aa&OcuItur9wI1YP%ODid?(8GXxKnRpBsMO8enidAbjkGy}^2
zmNb*!f9Fr5plxCt#X?Rn?G}ots5Un}ZtB+c-!D3RK?jX{pV05=X2^$Ps@v$dn=>Y}
z(apAVE!yzHstpkCjMwBVa1xkxHwA|F+jUo|gf-3Y3K^x{G-~0OC(4A$E@&Ze=Y5pHjd2lsw149iv0M(fg>9vb@S9aI&nDC;aOOC2$MCRlb;t
zGY!>?BEW@EJ7&FmA5g$y!?jnW|C^{Y}16z@P1vhts0f&e*gj7af{iq}-
zwT5l48>DdAr6uRDmjt?1mC}-`Vd@{zxQ8_*eQ{JAR5gK56Oh(m2o!;V@PzQbXI*$6
zB81ab&6q^lxBC5TN;>H#eJ7PktNL6q=?I&6N4l-saPK6$8{PP^Ys;A*9>lXrVYL5x
zckOmfg<@KBHFl;hjL8jEX@LJ&n-AwRF@Nb%@-ftXv!_5HV##HRPAl{SO#SaR{KJ^P
z?v!S=97&{KtUC!{K72$iZ4ZW9CLOGqTsFVs8jk=Tl)A$&1ilP@S_&P^B(rs5U{%{#
z{Sj2m%+pZ+3Ri&Z%=aA2XmDgB2eFQ{Xxn&%ND-ON^x?Z^$gXE+9N=9E7J_O)$^p$a
z4_Bu%`szkFYm^gJU~J2E0cQZZUU0<};PSV=Tk$25g{cciJxfy~Bc8t`$~jj+d~OVq
z=PT51U-duNT4q$_V4yVonSz;z8@kFq;T0Fkk4JbqguiUa{MR2KSugMB`I`
zj1=D^Yyh0J++A0sYH!VSPj%zyMOMep=W~CL=JUnViU^zeGl0}PG5O-|jWzvEF-Zu7
zi)aj86?+Eoi#oNh`m2?o%7t%kd;{wQ9e~myMtZJ$$QBHhY`PCE%vw2C8Q_s=RfW5f
z)9geovSsjkx2c)wYN~E}&qcD5?&iGb{gnX4W9VPw5P89JXlXWrh4^SbZd^dLh
z#6l?gu;AlgJCq&oP7WM_O^ESWbKC@vVa@;#>&u9(iU+;j#4F?s
zUEhS;$V$cOn7>@ux|B_%Bt4Mw_;JaAMV};I
z7q(lIF54wMrPpeMcI&LSCUgH5asQdE|LI^BGk&yd45Lg6m2V``lxRW|(E&~!?l+LA
zUw%mJiHW;*<2?Ep64lF@d
zFEP>&oQ*NL&<1v=%@_(;)MFp3;NJof)eEkU)$vYMdT@5m^uzvB2Y%7Fnp25y##*S-
zp6*sbP{aE#)HDtq?hHqA1sJ~Y0cdXc)KHnqrK0V;GAk%{U^_4JR*HW5U_$uOQ{ZHh
zCQsv)FH~BAYq3(}vccDnU+$JtKbVq^Q~s4Q{jD#79%>hyV?NOoWx}yFQ3BZ=_kCm?
zj(0G#9|}0+Vsx??XHo}~-z(0IC2C!s?z%Nfu<^Di?vevwg=hG~ut}SLcrf!1m^x1T
z?c1(YUHvlv+58`Siy2AVZlXz|#ng<}UlHTs_yH3JbPpjJ?Gh02wcI2ZLJs4Stz_|`)7rC$PS<_MmWDH~+F5|x2^r;m1qM)OVr(?&H9*AnM
zZgrg9ntrLkwYr7a%R|Ng!Rp^riF2WF_6=OfDl=Zcp&vp6{xB9_>vhk@v!pkQJdL;e
z2h3_#v6h(S5Yy|Op7aC|pnp?Xm_73eaKA_w@GWkeT{A_Ol$>9oDgRZkyGh(VxYE|w
zf_B$EmO;+*`1u2C+kccN2yT{)%8O0K>cS#=Tr$l5(cc~9{g(5Uxqb1vl@BVTZIj(j
zP4(JdO2{Crd$Mhq^TqSZ6D6-#uDnasmX}ug!+iV?dSI)rDh+#0nf2$rwmx}c00MWF
zEA8>q40uUs2dUEQ~Gdi5CFGxjSyS$>Bth0bO^u2CnIkAOT(N0fO
z^;|yuRVE(En>KbI%<5O-@UI&3>+=cPRj*kpq{OIXe*D)h)G_%AJJtrpcveq_MY0UH
zH>Ez-G!Ps4ER@>OJaSn2V^GaDf-ZY!)rfgC0U^%FF`nsgr=*nXHcL65IUaVQH-ch%
zoE@ckEje^e|MP#V_kL~kZ?+clRmGj+G
zcIR!Z8Dmjokkb?i0Wf{d=o)h|_q2rAAw3X#0vBmn+UsdeP1e*|L!3e!V#W#~tqz$q
z#)E$b+s^Ays+Nh1IxLQ?d8`wNi5iFt6a+eyim?#3zqD
z^(wt$zFh(B<_;%zH&PL6o;JZ&b%?Lj%H{mWl=A;Q|<5XaN
z$2BJ)#VSb3su8HMJyWF+)|AVVv3A3=3l}0))v&eTdYceN5T&KweN
zQM5BI_E{=$4C8#EBVyZKMJ#+#^0kmuM^SEBkC16&GD))^*_F87q~X{DcP3_TR`HmK
zOtxbd*m=Dzz-iYS-Dy6$@|a^=^roK8Wo1*G^Q}RLMS)Y{Lnr+-y@`Z&i@TSaInRsE
z2L*pHANZd;4Lh4>R%TOiu
z!fWYAPgh)Se}I`K_7Qfdq%ohiQH!h^Y+z11uj&#i3wk55=!YRE4hT*6S|UmIg`rBX
zAURPB*Zk)m$dq2kw^Db>I6Z1Dc4iym+9#zy5eVMnc4Bur104vWM+^pdXa{P#oVBdczKTWnWTl`EtSNRiFr3|7RbP~oVEs5@XjdnO6!M7
z`}V4I7HHVq=Hb+jM~7*i7bvpU4CrvZ;dmx~VJ%l#4Xwg2y>z>006xF-W%dJ9VZCE9
za-JvwIcJ>jy(ZhkvE(WezG|8kRn@h5!_15TzOLk3kZXU)yUNM#$x`-V)cAilaJlpE
zH%Jm8^W4lA7OuQ#6WZW#73$!{4zKhNi)7ZM4>-DLhh2|x)+TL|eyaNpH(AH~38$PO
z9#4j|FB&^qP>&N$VXYjV#Zuv&U)U=Tx#t}gIr>~Miyw^ZEyK7>uCd!iQ=(a)Z5d^h
zzJUy^0uKA_jOR=ep0=_Vs!!dlRwHO)at@c_F8U}|-`43z(nSFb-ph#n@(?qUXQ^N%
zL8$?|natyaj}ohivf|Erb%(e1EYbrMLLX@
zEpoF-%Qphef}Qb6wU?&%LB%e(Pt)tEvf(mxp9WWr-F!D*n?AXXyfqeV+}pVGy)SW@
zac{G~b4hpVc)jAs0Y%->VkK4!)HwaXdJtTA3**7R0Zy|{mDLdc-v%<9j*MeI_DTM?
zd&ixb$LIChkB8|yagsYfc2H4}L`}6X4|5O+T&Y442_gGHgrl@%Z@Kjs&A%1S{&#ih
T-%tNH0{=$f|Az>G&OZG=|Jw_t
diff --git a/visualizer/pic2.png b/visualizer/pic2.png
new file mode 100644
index 0000000000000000000000000000000000000000..53fddb4b00f37ae8f57a22b0da74a05c7ddab6f8
GIT binary patch
literal 220561
zcmXWD3tZB9x<5X@v(4-@&9rB9nts)>{jI62nK^-Fh{%6-?4ly
z|4P{L)fcP3_~esMzKY)&x9gKnmP~*0$$v0E{|r77l)CmC_|N~L?b`8|Pj0IJ%M9-<
zp?sh8{U@K?zO-!qz^Cy3(xW?5X`g(u{Gspn|9MLNSJo$=EMAL``+oO{bjyPL<`Lc9
zhGj>V{f-y)@~ewy&i(Cge_Qey@n}gY_SAoV_WH;F{Os>xOLqR>&ws~Ww{Pi^je}1*
zJ3m{8E>9^hUKN(GkSm|v^z`Sv;$1oYeqUd^TaGo=mRu4y$``Gd{xB_CoL%e{6|K-a
zCI#A#5~EoMA9#f$>GFh|$2Pe+_-^5(vZU>Yx1wh)_%w3VYb>t#+fv)Sin+xy+SPz8
zkEP6IcGV*Kx_^!&4pEWeiROrzZ&I0+Ii?UTGCtDa*MkaAyyVx@8s`Y^E%tqYLrwE8
zW%IW=mSE|BzFmu88yDBJjYPj5QBX)mWW<4|a;I=-WJX|kO`HS4txiIQbK)E+em#;n
z2k+w>t#KH|5B`l|Y~$XOl!pXP+swA?>ve@2w8+HB45SQwj3EDo<3XA%D0vXdMLqt7
zr;R7QZMe^A?Ot7f_;CwGgw_PHGABbCqpW>YLA7?Ie2h1?eWZ1;IpwFZk@}_0cbQR$MNa!P-liW#6%y@7?RBzK5t54P#yLcBk*H0x+uMg%2oUd=Cq^pREkAdu
zo?qSR_>wWI)IBd-uRn-f=}hjj{Xz7rX!B#$cV&`f29N)hOr5e;YW$FIY?LB6*(Qj
zC%)zSXGaAjwm5=%)y}xN${cT)pUg2Htg4N3P`9NC{h~U8hur~4^Raa0IR7X@*5i-S
zP>N2KVzi0P(!NKPv_Z5c(6n@Rbb_xB9TmBdduPm)OO)HZHMXoMtlAYf7ryuQ&x##+t
zd`AA9POb|c&0|D1y{8skcb+QVR2Aq%&hqEJAPBjF`D
z6fQXF@WTexDk^efgKHO;vW;bFnVX4z*7?v?YLy|hSKX8lSq6ueMOZ)thG)d%JGj$E
zQvB3kuehs!>f!8Q8>2q{DzBF8VDrB!$*$K?ieA6KmFEh7)2v+x(CWuUw?=Z&*z9_A
zVWRDr`utYj{qRICKgl7vc}$q_mM9>#FJANvZu8rVU3!X*dNrF4Er_R-zBs9Jv_{^t3I7#1n)0
zG6@sY&N(#SqghE1yIpD3Ez21we}pd7LNo=~(&iuQgaPo5CAOpa&FH|>Pl%kHYhvFi-#
zq$}WAOoA<`gH8Nh`xp)hXd^5f6090ZK>TJu;32FM63h#?`HcQLv
zI$781b`7h(E%Z0VRMsVZ=n;gRbCEm0LjP)`CTsdvY30k*?=+UmyRG-Tb(Nd}9^Fg~
zplM60cpi~=gWj=`!n%+$J(k}$rwypj?+(eok7)4_M8}2`z4L}=y!5SX=TJAk6H#gXp%>cRd>9!v4<}zxwi~)`Lv>#DC@MCC*b8zo|$!YnfMq!DM)T1
z>t9}!nNx|Jlk7tR{6r=0e#+BULR1pQ~evq2*ZbPyzRR6{{xzQ!LsLVfkPwtiONhilLswZP6sWkJP;vEbGjZ90l-{C~-UKmO^w+*<
zn1aE*^J~~fw$l*)kjv!cax0%xn&Mj=Brg9`mWrqF&t;yC6zMCM8l8{Fc}|I~D-mvA
z3Jg%;E!jFpB~-NLB$`~1f9nlO9#C(KHx^HBF{DqO8(UuWToj!>c&D{HPJKaJ+`o6C
zUez-H0;Ntvpu)KGYzs||k8C>Dckroc?BafgUnUMTt|6}Y9PdlQ#nwTjvZE1?p&Kp-{CyB|`uT;g;G*`xl
z`EG-W!-Z?Kvzy@;6P3oemNofq!{sQpfbzC6E-inl@G;!43C<(-tZZk4F{E$0=^wee
zpmOOycVs+2h#;rY!{AzKn=5lDMPbcla82Xu+}h%GP|IEGxUK55cO#Yxzrs|elIv#b
z_70~PH6BdMZX#z^X%C+17-BlN73tm_zSI(f^@y1#OHAk}f@6<8H+$%PoB7nhH4}~g
zX#8CDwWrLkkW1dBqpt#7!sjlcEyiql7tnr=(pik)NDoBOiY!C2V1`7NUn1@1jCe`&
zd^6K*&u;EYejzA+9~Xjq6<~A3bP6n{`7f*+S=?(x^Gb^P6jvQcSwL*NWKyD>Z#kB~
z%0_FS@i9@g^!RR=31uJt%jyshZBj9i;h6GVZdTu$7?IZ0Sc^EGh~tNql#>&(S``wiJj}xgp8u
zZON$*2rgRb%8r^@#|9*hJp8JJOYHb3t}wYJfEB(jTrty!o5)%;meNb|g-Wz;22Ba6
z-NA2)Zhc2Mlv!h_n3a10ls{2C8mbj9t1u)XtDJG%k;s{DFzPdCV`
z;v9H~S!ZHCesR_A=#t_()h$1iRpgecgBAv)QSPs74RLSDHf{nPeLy0>A|%XTWF*sP
zda10HJgVe7j+cIF(Ax6!p;|$d#g&3aiAD}baGM2&X%cmti|CRFzR6t$LxXX`ZjM!A
z@@>awvqawG>Br{F)H)(#z$}7!rp{7NQ=X
zr@&*R^x%5KWY>AJ(hpx+X@rP{m}Q#XeZ{U&ZOp?BR}L8
zaE;=+!l2t2(X*pMn+TWD@TLJN(P`
zrBICC)@R~FNmmS%Kif#fW2~buo}Dl8^!UR=%9Z>01ilnHJn76@42QCKDwn$1gO5P>
z_}lIj2Qs#jF6=wweercY9^bZ*+d9EV0a%#(S+(J>2nt%O3E31n{4;X
zq`Yxke)_BYN$1MEqF4!Xqz6-WtzW%94;e1ScQ@?YC^$}-c}P)exS|C(iQQ9?`AChE
z`K=1Ij?^STke@z(v+|Q|Z*Tv=Rjv$Aq)xybER*akvS3&I_LKk(J!lwOngs9A%F1&9
z&o17;chlkORpkhGF2?zavbGG`R+Va8^Z={p_{E&d6}I~)HAJg+g{5WqX>~(;$*1L2
zOElSCB`*BsvE#e^5o4+}06EeDK{O{qykTKQ+reNb!?Z<0Dfrg4>xv-sA+w6jzpzlv
zRi!Lk$INhe81y>KOWn;zzfj2PGkURAP!lh6><^FuHQ#ChKTNS
z6zgUQ_G3*mDY~#0&j-Hh*+xVes~nNUcO_fs%*DDj1UfPLtJ#Halu7j~>q#3%clE6U
z56F4S&qySvZ3*?9PS?KR&igOCV-B0V8_PpYZN2wHo4fW70D`jbp0D`l=U6&{44@jj
zBA1!KKaq5j@?dRnelbw4cuR3-16(Ku6eo#oguF{+Zm`6CL;Fm?!_fDc5CVU(tta3S>%~{~0#))DiC}A~DMc1u%oyit*S5)d
zB|_?}ecAv-fcCz30*&gvM@idp%Y^DLZgwn{g&Je*FYbpdq-3QHWTkCic(-bpGIm^h
zEhKGlxR?H~fj@Cm>s6U&Oj}0sEFJ&FC&8?|443z8%V=P@96GbOxJ;p_Np;#W8&-a4Kapmsh{
zu!WRqT9%!ZIq@}}|9*owrsVc?g;I1>alN;wBA0u7+3fUk4acw_*T2>L_i7@>#qGTs
zQHuyE`;}1;RWcQNL?qNWmzYAYMs2cg;s1C_zg}V}vY*yqOOmIAHd#%HnmhJ#4B=^G
zurx=?9%Jl&9aM%X_fqi{#ne_}xv{AzP*e7xCc*lt_E9`zl5g>wB{I@#7+brUC$Wi&
zxK>*Chh>cJKQjoA@`e7Cusi7v0y%9v)a7H@3BCD@=6URtn3umkSMUw%f+}X@4IkM<
zjCg!8H`P&LjwpMDhL%R-na>#|gFZg_5`AEH_Z4@T4_4@iK0MDRX|jy%?#gGW&oOXZ
z{@iMqF&Ti&7)pGqBW?~;Uah!NS4hYTG1O`DIa}ZVE303bdJ8Bf8&I0e5^$(R4_qwJ
zbP5ytkc%cm^Ft*XbjL$DiU$~(M||<8nY8Vj#l^!16cb?uXJ&^q%*@F;4H~7LN2D9k
z{uI(ipR_?MU(0fhaHC)O?{gK4_227f)G{iX=`w3f`WIB)ixrI#Z<9;`K&CpToNBG`
zfWn&X{lkY(wM%s~?T$y|{zj65nb}XSmU5#x!x4Sq6lxbgs_Yu*C>60L0WLQX)2LBj
zFNiQfVXhKuwqn(>#I(Y6<<6;{r%|*Un`TZ3UlKU)Gn`F3-yLo~_!cw5Jmo_Mjr;A;
zsSwB>6Sqa}&n1BZ1mx%SBM+cD&~FTIQZQB-Ff9>&)^_0JzDa6K{HSR_m8z_*Qp)!c
z!6))>+1F}Ly1QCK(o%r3D;to#F`d9--IXu7w{~!=26MP_RUc*h<;EiaBM7Xpd6FH6P{61Om+wb0f<|ZB%qylZ
zn$^m(Ul9zwt))Y*s*UQr>Cw8way2Ehn%5rs>_(Bf2LC91PCVhHry||sbqB@y4zqi-
z-PL`3>RL0U`1cy(ajC|pZ0qUKNZ*YXWOJgFEr$`dg?FjRqBfI?+W(A?s%p6ymlm5l
z$uX|7{qyHgSx
zjL_bqxYZ?p@g0;m=;I-R;?r0WO2m1Psb6qA2DfBauw(Ll%xJo)|Q^6>{>HdnbGOFr*YJe
z>Nn3YW-ZgFN<>uG<3p4#P&oGH+m6jzB6y5BfZTC?3;e^!t;vu
z4wWxLW6xR&dZVuoBqCeWwqK&GFHxU{XCLQDEUP7w#ciSSI2j+oRua=L`LzmjObp=R
zdwi;4%(?2u_Ocah)_Fi1V{I{8i;|++4>AHB*Qth7Oca-|e1R?J&NC`z
zX?H2Y6XC?s-}2kWy1j?Wx{@)kb2~=-5rWgRLfg5!ELsK~E3zT9rA3X7dbFGC?Q-v8
zM@A?wnv~rQn+geZwnzW!@Fo6-Se7y3Mg4}tzJpr?h}#@W=3<-<;Y3kO)&3T1%apxf
z>ddS*_KkBFBH&z0I~y7Rxpv28DaT95u5FxOhEptw0m+K(xgGqMco4q$!7z4zfC@6O)xYQy^9G(v3eyfZ&*E2X1W
zlz{KJ*R$)2d)L9LrABS)39&)#1Zr+elB)`~AWE}?F#J$0@(|!vMc1>&9p%QxduACg
zp^u7Xh5I{Gu)X*Fxh!FE=cxYTbzTRB<&y@)h_`oQj+yH6(ip9ZZp5%&xepOgj@8tU
z@s@MxZ|%{f(gd5`#;TRzW`swJ1~Ec+Ye~mX3CXGJJVLEvf-cgGZv@g>!fC^}&pFq(
z;(C&pPu0X=r!KKvNgO&@#&B&A7f%tuNEcvWK)0;ivd8j0YM0m=w&Vf=%w33uxi5|n4US5t`?AG
zLAZt!0~)~lTt4vI(*^-{SrJh2)!`pv_d8il~DT*HpSy9ueib-
zA(!q)zjyFbmTkEvL_YAf3q=&I0(nhhLbG7XGWJH$!cxhn(H^tl=>5UwsvS#(jFy{n
z6-=P6qiR*8kDBkvK5<4C6GeBQn!T5J$?HEHfSA4iL%A{O)`-h;P4Pxan7)V+BC#Xh
zO5VQMPWhV>#qop?+_6hzcSqV&-V54Un@1AUo|;_CHfv6@rbqIBfs3T5B^h5(veR~?
zk-yD#H@*0S{ZBYz>65U{X;jH5D%^x7_vM*hZ1l%m>e`@Gymhp5Ua9zgG0TimNDi@C
z3xu-W|F@@VM#WD(AQDTE7(2|!>Kt#%&K3uwDyI~tD>U~}BM>Q&gZ`|Yr;C?>On6Y_
zKK`u`NIgh7KrCYXU6&=IKywoIVLLp56=MbWyVda-EdfP!WIk6yN#@=fM8frqA)N-z
zQHpo0w*2_|1uIIs5O!(!Sl^qHZ_w|>%>F+$66t8PRsLYol!_IxNKg64SGO-Jy`e-q
zl|{7Q|C{E-1XDIPB9oNP@%Ju%(ZNr9wN`edl_@z9Wxs56ZgZ?2t@*I5u`Dsbaio?=?Ms?nQC^*hYd_=CiIcH?1)<6flpjQ?bEIlVm%o_^(#n^n`EL2b
zr-tY-4N#Xj2Z+tB32$2+LFK3e)$sTKTc7L5A+!!{)@oE)PyKp`Ts~*?)nVdz`!5W>B}u-w>~)<;z!5ycScT#h^=f4Ym>45N
z)|)nkt_Nfdho;LWDy*UynE-e(qY53Ge4jDXUN8D3;)?Je9zF_lZS4D|+g&5ESWk?j
zCQr$wJPZ(p+m@-Fk1HoO^t~y*s@OwKvX)rS)4pNn|2k;Lb-T>VAf&3jhG180L)bU^p&oH(F((*zt{U(9U>
z*J|88)!^d)=yZ3tl>L@Ye$e*p{6c)TDXB}yosJiiK{6?_zvx*2yJ)^04K5m?LpHhNr^bBZFk0#pd*U1*zAlQs
z2dW)hTLzPJ4Y(y>j0ip!aysx&QnP~uH##YP3XcpQQ8r|Y0L8aqwe6<@M8rxw(?N{s
z#&;;squ|+WCPZiVqva28@nmR(iWEYj4WrbgKuWm-?w$7qtLwNyYdT7e=8>pp7-G2=
zGlJD_Trkg{8Z}vH3FG)$O=%ZZ$^=ZJP=9H2Q}t=4$P}
z*^uCR{CA@0%o;>HM;6M1e$lTJCIC_9l`>7OwD*F+e
z*tfT^Pn5IJU)pWyHH{WLRJm>K(-z(y{k+q1Zj8S*qj2Zc^hFin)kgajyjs#;bjhP7
zbQ3%z>hw}ftf~FrAg{o~ZHn1s75sz6+@a2%+)Ujzf~sGr*5zSTfMQUsv-K}E|04>O
z6K~xg;X3xzjJgP(3nOF0Gl`Ovt!=*qh|F97O<0n@EarfLVsq3SN@(fWe+JW9>q00%
z0HwgC5Dy&5GK~d-tv0CCFc3h8s)&6iJhyr?IqE=lT5q(oSbSdbWvxmr6{VA|76NIZ
z0&v(3!;WhNj0Fy(0q70973HJVs_}EGd|zkxr%p71I+6cT;RiR;E32y0*n)ibaYMlt
zXGwnggLKa2VwiZl!kTL^Dl2G(3)-__>
zNtS6mvCdE_|I7iiNDI@kQTPyKLKu+qhu!YG_`q=7*SW%<3PFH9ht}-AIuoQtCXCno
zw@Cn^7L2$gKSVl^83B=7eGP!su{V3KFoPW64f(h<1J2}vuf1Um-%;V{0wFLk`XLnl
zY;n|ooNS-6l8nb=twTgj$4ga={4<%QuC03bguZ6lYDD2SRxMmbXwYKaSBec<|(f%HiH&kru6-8ed7-NR7>}h@a(FzMoTO41`Al&yqQp1RTRWC2YY8
z$XB#Fx~^7eE!U?bN;M!MBz~1`xfzu2>qyXvfFS!Yklp7rvf~_Ozh@g2iSXS!eu;Ac
zV+V=Q5bsM&NR|L&W>+|McR+joXgzBEN%S|Xid;;f)9FP}hjn#i=bt&!XKN`eW&B+q
z1PQ80pK6Sb(uE^DvgHm-5y4<7a2yhjI>)eMN^c8_Y;G0GFg^0`_BN$A}ugn;1AszZe85+mNch
zGI2$K75$IkZeo*wA&jREDn}jcWbmA?eja39p}sUOa0T7=ZGmK$0T`?Cd9aN&&HJ=ri*E#n&q(NALQ
z-|)FP>95r{cma9aruLpRede*fKRguwbrEs;G)ori*fW?qm6dJ%X-0hb4((wN*0#$(
z++b~ZXS0@$C}joT%2F*cBHou2W0*bc#6(*w2EJzB>^BC^q3=9V9a&u~6Yc-c&KLI1
zU-zu9XGv5YDD`e{X84`89nKrM*#hr(xR}kU=ARA6uE$KxfO*i_p;+Kt$=hX$g6Oa
zS>iSzjt3*iwHlPp9oh+c#8NPZGa$et4!KN({<4%2+Ba`pT4ecah)MEJsKrQx3!$PR
z>vsohIxZKNbMGANe}fI}be#1WNJ~At*Gc!4U9rBqsah@5Yh}Y10F?u==be;h198DRFCXccG)1=M)PL&WZWFP{r4i(^u{$^`Z
z$8jDYZDO=F9}er
zEBPog~b=+h6jLXkyF0hv$6>BRo#-1lXZu
zo|GklKf4y=dZs2~58N4PcBVZYJ?_tFd+11cAUfE0(U+lUrEzkFIr^pbvR$XryL;CA
zbBQIM%9qTi@q>baz2xk{T-^=B&?OOa&fR6eV>l7GB{#|rIhUqf(>{f1XR
z=_upo*U~?A&@W-MqgiRyX)~($?aeTs@ZG-ARPGe(|DbtkLWik6KrM_yGn{R7IBr
z6@>IiOpY4|=|WAIeYdi*u^lQpsFQYe!c|MZ^%Toeu1B@6^Pmx=e335#)PR&uK0hdN
zOIpS_3;QBynj>dYyG8$C%<0YimhYWXZk-|=WkjFjUnp5bj#bLgQjY9JoN@HFV){h7@~9d=lqXPu%3=|8
zjpM}|KMsMZ{BKDeBv((QWHsrD+=6#pD7f??VuA6@(k^&3j%aL!1PTXqQ|R@kDoEl`
zvqCbm27;1>`(68jYoQ|j&7hzr`B|aEFOC6HDOc)l@sNlPP<7lslrg^HtsInUOgZ{}
z612@we+%TpDqg8T&>DTES7&r$8Cq$?)~4pujy+FX)?!6ekg2l_&lyjoTa}>XV(4Rg
z)Q9^~wwAReu77=oqFa6b_#Vo%@&+gMP-Yd%!JIs+LW}ro)Vep!9aFz2MNUe`CDE1rEg^^Vq~xyxM2pxzm#WP|Bv
zp@}PuQqWy)Q!*CKinovh09+8})4%9k==o=%wPu8ahT0XW5P)L;Z_}|D)WMuoPW&ZQ
ziFSGoyMKN+=wyh5v@Tx^{)gRbR2x
zG$0Z}o6RRild`*pnY#``LaPuly6C*fbU-`gvm1|
zODwUdKIZXxyg)D%Y>5OVog13h@tAKiM--MW{Q#f>EHkkA!M%)c$OiU!85~+K#uwm$
zYFtbJXZzztla%&NJM)9+z}eNORymxI$MAf?CgGH$0%PLR64%>v%9V)J2i^<5+e>N8
zSqSMw9cY>e#OU(cDkDV;Rq!NSOH6x03bv3gC4`7tO4M4oHaA_|F+LVNJbH&xhxeZH
zXw|k333}oB55DXLNpefG;9P=Y1yVZK_FryBu)-o1*dwv8^gRK#y8%Vv2-*A|ec__a
z{%T*njN4cg=AAF2AN!Atjbkt6AYhx4T94gVOj>P6H@a&k`R3FRT2WHn#Pxj<4n9d1
zQ?)S0AD;M@bxKgp8%SJu#~(ArUkYG>oELVfAtoCk>~A*yC@V%}UdmYF%^Zpc{3vel
z1?3FieMG1AV1EVm#&C4@Kf@LCMP1?g`HXKJ?ZJFph`b+k#&gwo=2x&mF>8g%`S
zg>G_RA-8@6qdpY^0}Y??4>%g&hN7LLAF3hAlXlk^5`cRKGv&|Fu_70@UvVcR-KRXH
zC=8~&YKI8rfb7MeA?8GZ5%`rOq_-GY?NcG({BfVW$lX(&r55-O5+W`BJrI#t-7W&E
zWFIeHW?zZsL3v7Rbd3=weh`Ozo+S#^r5XsA8J+OGR7y)t?O+MsAr{Bx7QyOjt_{!)e=dK{I9C2Kd_gpE$_oIp+
zWAV}0hZtCLU-ZZNb3xSC7t8_HXz*!XA9Lg&PC$Yc5LZ)Zt^iQ10!&!_pR~v>&}HVu
zD0j#Le~o2yXCyIF!3i^N!lq%*hc+@-39Lj?src(E6r$
z^3M6S+w$erK#oD85Amh9FlbK7T%)u*Wz-hK{ERunZ1pJXJ>oIPMHzzu={eQ|n=rH$
zH!)tv8MRpMBN${;6E^~of^uV&1bcf5aun2lP5TTFVQH$%Xkgy`Tuvu6Ht(I-cUsSe
z(!ajX8=BMtvya7%eM1-ToYNi_k4ob&VIg;4vC89;&|Wtv%WiujSZZkC0Bym$smKAH
z=RBy%Y(d0j!P%DLiW6tb0zvhJXYTc3c#yNpfO!*xAe!V$6Vk;yz;DyBcYu%WLkVYF
z=6xvRCuSh3w;;1A9-5nCJ%{!nAYmLtGhTMr_$b7@L3vM;yjlw$|39uj#iS92(B^a3
zDWclT(ayaD@$!;vGm4iZ=akTQou*}9ur$yK&8iek)}-JA>FHL=C6j-?2Vt?PiU;8!
z-o!HU?f=>S6kLv>j6%tIR{i`b-c6=yfz&bV*=wvxQZ_Ee&#QPQp;I&(c8?M0Z9W29
z0?(x)JsWa)OQn*uF=z|3gGk
zR|~{Nh-*Ybpi^Qmh?j;ZqPk=SrC0g`>Kl|TmN
zw3eMqSUfeVWFbY*u-`|Z3M!Hk`X~$tO6H_L#4^xPI0y;dn{p`k9ce1p-N++E`?s+Q
zYA|rfns8$rYk>;^?UM;1uNWy66xqfCPX(NUM
zW$`xQWENwxPJ}AykPi
z+{x6BHwYI@@7TZH&@#W>XB2w<%bg?4V7MP0P=bnI@ZGUKIO!{$z}t>bQQ$QuTo|}o
zF%a(y>V_iSC%KM(JP;`f<#JP8zAuBS36z^Rq*WhkIgjDevV}-(ao?YMP^9e!6X^JR
z!6f?e!k%i&eF=ISU4J~^XO)Lt@&f~Hh@1V8uT!%=gET+Ct>)>OhU8TYEZFJn5jKz#RNlm=J{VkC(B&kp?R=qRbnO}C7
z@<3Rr;IA|g-qrU&-M&?=VFfzMHcg++?Lh{=`DxDXcfWkZPeIC$50U6O%C`wFQy
zrlO6U%)LAfD;4G+LCXMm>Drl%URDGmV#D^S_zNnb+DTMrO|P-p;vgp|RQ?RFtj9X+-Z4derZ`puMM4D?@FX{P
z?E#x~@qtF+G~bqTa!itsD^7s4A%?scSe%&5iBq3kOHK=cGy?zt2Am69Rk_(W<2k-K
zG31&7PD98oRNL%R8sNh~mO+BY>&DOJ5)98NRLIj*HB75&D+X?>)}uJYdxra@?M)L`
z+@M(_wxTlPBkh%=-RRu0fX15islO{%s)`B0&b<<5m>9OdVt7uv^tPg#9{&^bs8?>X>$O@%E7<(Z0D0drVn|d#qu@v7$k(-mDL|rn6MFjNi?sx1EV;dlRmi
zNep_3M)fuC!L^ysj(C^Yx$3OJ+_v$I1_<=Rb`l}FI#$g*S9~TZG$}^|jVI~$lM9iSd4cJ)OY
zKx!BHh;eUm_h@#jA6yz@Ft|*~J|9q+F!~|sLbW~nfIWy6h$NedhrY{$WUpG7(t>^E
z%NpJJ1#8p!^SMBTKs01_|BI~^D)ZHuwOWK~=!nnxQDaW*7&+pL+Gd&{ok`o$D19Ih
z`zK?ZX|b>JU5K*BlWVon@l!*2y0Zg{wl_*|uD+|D;E`=QV6i6uk5&|#?z1cg%H%+IbRy?z9r$O=&n<-*DV^CSdj
zxT?!U-664Ry?>8?V_9FJP&(AZc2shkb4#n*2O&lu4EsB|!u>u=`hAuLj%0W_;Kmg;
zA93SYcw=y63&eYEl0vF$O+9Q?=}FTB4N>%PfrrWW5Yf!#nApWTZ&vPrV6NGFYL-dF
zkT7Xf2y9754sItZexg;MDl;3oLK(LtEt8QiJm8H7oFWnYu>TQd
znJD%7)%@nN58Vs6|L;=A7@M`Y^L$_rDk^nqc(2l8e1s+^OEqX7KjYU~_h^t9)iy(YkF|E}sUAI^
zGVZkzvbCv}>;spIs`{}A&C#cFx_EGCqIJAP25c_u1Hr(f9qiXx3GWD#-`d^_jble&
zn3nyE%~3CvvGS(}?nJrkn93Kon=%?o2%EIr4>i*%k7ZxdJZ2q$_cVPwgUxWvpDq1R
zh+m9_=kpuxx;)Oav9>hUGjH0uV5&;aSfYkG1p+7`e(EO(e}X8N>|1*B3Ez-o2XO?5
zY6>jIVHU4HIwS_F;afKW?_hiPJ^rfc-CsQBByKeu=*32Lr554bOPT1UVw)!^16$3X&yF6UfWni^Jw_SK
zYoK#CQDZgA^_Wgi`!8!DML)XrY8<5Fz>x1rY<{RSfhhO*2BYZrN+7@n4X}>@r>RND*Ik^e0X!X
zr;@JOI}whVfx!0cKNn=>U{K{vcW-*-u-|ZL2FDfSBN;YTKAEns+n#Mv{2(TWu}Oxy
z#VRnSwq0=>;5RXd&<>QtXU9M
z5yxSo%C)1nYLGn{JxM1`Pc|A}EsO;5
zGd^H0+|WzcCT6_bAXZT{{*A4?PvzBXC|oUYyO;b_=Tu
zwvj?mndMJ55{G7)!<5*=U_^JF%u!6HM2zX$XjW@>`Fm-EO?(Z8Aze^T^%fVUA*hyq|an{
zP}==DM&@0QQ~qm3{aR@?Z{Hc+&AjH;5$^pMQ3XcE{8SdjvWJMQ=!LABD4~V_k_AHf
zQrVbbaJp`WbYV|~iLIcIbgw%XNzEMWDXuj;@F5DY{%YV6sEV^|gyR`^d3NbbOMr)&
zkJ^@23QMWo;6A3@H|zh6j(!jvuN_`?3j!L7tHZc;i|)m_W!_zx6G<)kJlwwx6^OJf
zh6O>0*tqE1f1t4G9pyQtfDf-ji*5%T0N2Sk)<3rQ4bo{pI+aZCIB*6DGKL{b%oYCA
zP#}dS7HhAp!}S1Ce~|B=8;iZMsj#mb8sO^iH-v2edI9e|B#q94iKQ#)I{te*wQc`E
zQ4gk*btG!x`bcpR*Jq#q-e)aD2YZ@qYvxE#VP_%~H*vgda&y^a>3`tPrFB9Q>)wvd
zX|A&f3yODNiL*>^^V5i#qSqwNYq=KHH`_i5oxz#7MN^I4Eyn-(HS=>y5vc&eUBj4)IwQ!H^ub|>Ylpetd9p2SLRoG
zFNCB;ZN5Eu*Rose#%P{{E8Ym#VueqiUR1-1Bxs3EzIge;C(;8FUnI&3yK#G=nRd6}
zk=ps)Fzvt2ZQE8$yo5B9JAUyXe7qHeS`Zma=v%=jnhrtrNW4o}+K=g;a&F;YT;Q0P
z9f#9|f~79RzXM{LhUS1a*bn5Ehe~l1${SBvX{G^$Y%EvQhUsDbzTWepeUF`SU`BJa
z0!g+wa>M1)m65Cg7JvO#HjUL(ITl^%4EQ^G{zFSz;h8-8S?1rsi-h
zBhWdND|`v#^&qZ){}Y>9vRZ;I7?G_&EAdl*_XRIYAUDrV!h-}f3f?Ft-zC{lqL!iL
z{$hD9p8!5-UV=iqU_gX^DFMdSibKF4QEuS>g$ZF0#>
zvW#?hn7Oj_$s5WF35H^|m{i9=w0!#LqxFQ4MNf8Ecfrz=~4tV3h`UoD~-9Gx(rjAhsG
z-9DCS2fPd0R0Dv3{c>R!R0k4R++3)%7ShdTnpkz+@No(9D~!627X$_)FX)86a}1Q$
z{{43+H;-oI?ijiKrOB7e3~M&EDe<(qC_H1l{dJwE?xBt{xgy{Gl18JNl3+xEM)T0C
z)HSqoZXTcydj#(^R*?W~fc6x0nvJw6i>4D)2sm;;dxW
z>O#(r#&Bq_`B;(r%J<_Nnla2+K%?0{X@Ff2MLASub>1BNV~5ZzZN-^ewCO*O0p#B)
zf+JFz-Yt6#MuHu)Sb9~TCalo6n)no7mBp*J0
zLD`v|-QYs(7jl0hpNwUN_QsS0rcKN?@dh@jfMcyHvedz);~6OyPKH9z|Y$5TSwdX
zim`LDF2>V#NBJ6LmW(`@(E
z7fWR}^FEuvMMbwU1`pU!!Qz@PV_UJVS`!L(6i@t)qHBZgB!SKQ?oLDOtjkU|
zT^n9@185}?WO0OD)|C;+fq@HlsbD8ws{HvmV5X)Rh=Hg0R`-2fMGCTy?B5{>f=?ra
z%wucJ0h2G)76;bJNiTbe<||hU5@Tq&cwl?5WK|V@Rmu9)LfErSJE0|eJGUCP
zKe34Lt2IouKLX202BAF!r^IX>L2);OjL7)GMr35swVQizGa<=ip;vK*s~aIU!He27
zA;aDtA?w?-y54&5)Ysy$bTjPOrRdIbFreraFbZo>UE}l1Pe?R9yvqd;(m|9mv+_+V
zBx3`Zv62UqOf8Cgd*ns>mYp*FI4P(^O^kahFuoYK$E698;XLQV!3t4{m-Q@lC{Mnd
zwP|Me6@lp4XjaHV)2`A3M*(v|=I;ATeo^ou0PvC|i^-sKs{O&j8uK|e5U)Y>8Q_2b
zwWWN7zoNkK)<+lM4(&Zn>;6g#OdH&s)J&5<9X7Rv`>TD}?VDV5RYNq$tngwH>)Xtq
zy8t28$4HHm9h=2`uqU`#MEt%@h1w{yl;_7G(%Y2j{_$neYgkE2h=Zlg)=3{?MkjQU
z)Cc>zAXtzXf}L&#$E
z|Hl=y?qaPf4AUa>YN968XLAu?YB9RTSMKy7H%M(HzJ3CdD7-F2*?NdCy8!F@)O9N~#1qi_yB1I($;N~FR7Y6POA|Dh_4pX)i#*g?r
ziBhLH_1)+Y7-RK3to#u|7`=IT0kn0*_(hX>tr6oU|BhnxJ7!3DE8UR+uT%gcj`a~%
z2=V@?`??-Dv1K?mFiLysz`n
z13Br&K+I_{-xaW-!X@r5N_xM^XLeH4bxBp>EQ})G0KgI(aorXe5`$JGzSacPaN}+
ze(RTN^|n|Oggy2Q-s8yoZ7tGYFtUX`w(|kPHT9#b9AoXTTc*~^_%P{icU4$ybC_fX
zUh~hgiX^GeKMZewN-4rLv!t?v_w$&zjP-X`;vrhTS%de6iiQEp{8z&C=60$kI_soJ7(
zJT-tF!|7)5u+Slqlagnx!aLnPDPyQ!SH3-Sk)DFbge_9vk^7FP_ZDewHDJC#~q>tLr=NL{g@(ik9b>xw|qib8<`Re^|F
zP>_TWAn$9fmTILCwn&z#0vSZ45)dIk#@(Qh#DRN-|{515v!yCHVP`Vh!KmPl;ApowMiR(KZOs){
zs6HR>k1Wfk1K8%vd;{He}ZlBrSeEcs{x<+gfmc%bW4`P=SqR+1FGAL`l=x
z6|ymROv$_hBZ{K)G}mNL3pnLEh4?6CeV?C>Vwzi*nAO5skP7rZQ%D4*^XI1iuvWER
z8;M}>8)m43*BxMpI#!!8F6C76=6IJ1Pzd}fYqVnb
zL@6WN?sY9;+=FjYcC<-v7On08mLbb}cCKLd!5z;WVg0gSp@;<7-1v4wk?
zr$wH&2!F>;t?-+0u+^wf<=-D?RKflIa9J
zAkUAPBKpjaKrd=8e-xMzlvn7`kV}O658oD$
z8N2a8#uk+vR9+-9>4drJKuv4Ops#e?>lu4BMKmU1Y%RaYsaToJX#5^geC0p?Bz*!$
z=cT9|R#ju=^_#M!2a0HBxllpZUiB!?s4rTiM=63G`AzDSX#k2QT^ylNOuS)jieMjh
z9P5E1N}@VqInDUNvKO90?z0P5murr5>=D&*vAxGd#k0e;y}u_Sv@kX{@(02Sh3Ur$C;
zx{u(`pbe)(oxq_Ee@t8s{+?j25XU$k1@)tQ4MG?^2KN%m&JKKy0dUVFfB-Xs6`6CQ
z1)GNVJ-1y<1X$uC3YYO|s7UrX4
zo-6mJS0Z*MR@%V-V
zoF#XP-mN0_y>x~m=3(2_mUy4?7fO2xOgr(Exffe&Rr7E@wOmYJ2Ma(iJKciYJhlJR
zjp8-4ai>?q^K^chM5MULw!SupZ-`LtwjwydFKiCPi*j^6htRH2C}rV69f)mBQle_t
z#B@$W-+NNBC0EEy6n(opruO>UUsd{_F_8pm>o0RpuI|B?M;M00|H>MW>=}$zDu+wj
zNn0Z%?A6Uo5fOao{d&slNJ}g2cjamSGE-k*_UIC&Nqc(Vz<2$f)9Ucy6R6c!8F&}_
zQIxhtD+AifTflsHl0Y%W*=ZuJN=TZazmzgn$AuEk@bJ>k^qiRCs=g_K#k_s&FM3=7
zdydr_%T*diw&$u>f4F-x=eIT;9aZykot{6pjaR|Ch8E6l`8A<;xoKB-_U+?d;%}>=
zgf$R~Gj2&AXWO&o{1C#2je;7Ag&a|CG5$_gP^*+*xu0D>nG^@V-fp5gBwno5j2$Sf
z;CEStQUk5s!QEcs#ws-tq=48AYpn@Mge0)EYA~hB!FcH3ZCZqG1_^fzTD_Z-f8n5<
z2#SYl4fMyyqG3w^{Far|e7^DP!`lZpn@qC-&=S$w%INk8N37#?PII}Nr1{}E8Sx6n
zfz
zSclUnY&j}ARM>nlo;GV7J^{(ZV52KCj~_0Q|3SQ9TOpCrIU+h!(V@J-N$>5x
zN)_&EE5?wUQFtA|vAj8*_Or2|;Y8W-UWN0+=)Vhqvb*N?gavecBkqkZU02vwyYh~Ol7HBbpOU75r1!v
zRJA@h=5PLpTw-Wt((a6)Wi#gsFh`W^f-34_fiSUf5Qc$z!jcEz6L@bBu?W+DOYG%A
zC85H4RlhnT^~>{@g?JSE1lQgGU+GZNI8A6<3%&A?Cy*i)r~17s=ePK5jqvZ_0g&er
zK}*^`_zQ=qojWtqZ6q3}d`hgp1D+2O6{<}tFf|wzq+#>;je{P=Gfm?co{%*z<_3yz
zr7}zynKpRZFtqAYh@aIZQ9>@Cs$71&}C})Orv=CY*zf?q6@~a>uYsA
zQHZZO*gdY2=Cd5VA9*Eb%w<$|qj)Qg!`_00_=Sw
zu;s2F_EQVyFAcP4Ok~MLMoZdWm3O8{yqB=5j@LR4RO=VUv&PS>Ix#N?ILuug9TZ+3
zx7JSljOr$9Nc`3r7`1mhFWik;Lm|8wG8B^febB<_D6L~^QRfgzE==^3kQiBuuL`{#
zxKNU1J_KhH$_PqH7k!D{&JgFIK)}L7oaJFY8H%^d&=HW$jPVa~3d!s8tUF@gxCQ+ZMi=l#95AILF
zY1#|F_Yzhq;F_X4_720S530pUz-nz=c#rGX=9&Gt_38&C
z;Yz0CMYznoq+0*!+~3mgd@J^J1JJJ4v3y^|qr*z?tN(+L;CR%rAOK5)i9%Yk+E#OqJxP#P0pc;rU|Oiy+qSR*
zUnCr}^ZVjq{s#qM-~`ApgpE;H#vtD4)gia6K&$;89whZs;)O4V;a}Wi?z}Ff(+-Z#t~?QC;fqC+wcqd7mjwHV>CXLS&>wR7hxAa_+4p2xm
z^EGnJn(&`7KM9^+B8xJ}M2akR$mCLGVT^B@Qk0S#rFxG;;oF#Ga6mIVYsv>UCAhT;
zropGXXA7q@j#O7PpG_|UE@-^doB!3C<_ZP@X>f7AIdYJw1a%1*iP#}}aaVWM5OE|r
z5Utg?-?H%zE(-Nr@Llmf;eY0lTUgecamPsnBnpzcU`+GOE+7?uwLU_;dcCIDO!ZpMvu7+2B-O*tD7-)N-h0Vi`+1IQyjJTL@%OV^PNjboO;KkvU_90BiVlE(>qyavMXBVoAr}R
z=6FqPUi%j6L9bM%VdWyH$Pbh32d-b90ajA{<6LR#b8st=1OK>P1OGyDZ_GL(?tp2C<4huelPlE#0G@?>=SJ{5Qgya$Dm9-;i3
zDqlud`a^HRG~(rAi_o9+l#S61E@0wzfwg1B)PKh2{c(F(6oW76-HyDV{ZjUrD9j?)
z4NiH>`I(8U6v|74w6>9?L?VRrB1^B3K6Vx+a8s+LYe^&__2rvT&!5MMQtoOQ>1#5X
zV=JeP8j3rsoV}mp{QIe1Z-qYvP7|wIy0;C)M$SPP#*kZ_dqM6u=uY>KT#&5912Pn@
zU>28gSJ$)#O(3mM_Lg5(L>8EN0OQ3D?>mavNHx~)S?IRgHG>`;HC&Tyht(f6`#4bS@&vw!do*gKmU
zV*iBs3P1qu{hDK*e_zA&dy})8r9F;6usspu{-cc_D=u1xgD*Eyo5Vc^ZB59MmW;-)
z%y|{K5efR`7!Mb-jHjn9k$fK1pzvuQlnu4eqzL~ES`&D%(D`&`
zV0w=4SNw^_iH4;N4sEg9sGR8aijhRRDIc%7w}EjNK3g9ASt_t*M0037KBH%OISAv4R;yfqe
z&|g|3)(Qm{iLLm`YE<`I?}c2I3RE^-a~kIHxKjs
z7$SY;zFGG#vRz_M!fCHOY!JrE(R3^n{6}z;F9TArelq0gL&A5p2IVy-;Je*z@E&jI
z0h}+sqlg$4q1%Jbw{|h4wb%MurkEuBcVi_bH1j4!U2-%^^dQkgiDu`gpsVA
zy@XOFpX^$Aui7`Me`er9Q%3lGvh+Tg<7-K)G3XP-K{7Hu$Zy)edd;N!&bx`2uOlwQ
zxyI5-mx=F48S3r6@+%+e=`(_|!mY6Nc$Tl>tujRBj^=8C0u%7L5liI%^XiAL#muo+
zy=DzSbU2QP;~8}m6p%O5V{qxmzyb50@B-Ag#UbW#!ggj0o;LO)oP1ORgE0}DSq)^!
zuVSI#bw4pC2Z6+(jbJSBKQ(~Gst9-}Se2RQ=J1Fa&UnLejCI|2nlPIAeJm}#OBQU4Y22157F?U#Bk!m;ji7QryQrL`%8s7Fp<05S?F**uQ
z&*#Tat3j%ZFQ|HTUr6ykRc-l3HJ=9(a2HYI}ZFd@FLM;Gi(YpNJnh&pYm@(#lG1)NwxTNmiUpWFRVY=wTwT
zFtZPjM@)Il{*V{5VN9~fJk5K;hBpSi$-cvv-^KwAfIDZkx`r^!5SA;HL#Wo56Tw^>
zp$2;(I!feXi1P*3SO0)5DgcTQZ}~T^DK)Jhm^l)Y4;@e~;1D-K0Y0l0
zo2B%+qIaJJFi7}>eo_@H>(k@Hwri+kd#O!Dv)?$weT1jiMV%z`pAsuFXSZ0s6JO;FT@y^4{0NylstBFx2hyrvV
zL=k;oSVuU;>Buv%J_#SO_5JEe17B6}8wWI_mAN)a_~xsa@!{2RvG0evwbVKq7!Y4o
z--e`t`R1VSj_Q&1T0^w*oWFZo%g||6vr%HX$jF7UIvMGP#z*t{Z^D~*smD(=wqjbt
z9|asDI5WBHQ;p7aBRSsR#}f$Ps6l6k@aG;kABS`zW#0>E3VT8+?V-*0bx)G~XOwBQ
zW1~g8<1bfaM~Q}oYW7_>`}0m8n?!jZo>B4ImjC%{#@eAvhl>2B5kqY2;z;~6xh@G9
zl7VCx-EU}BQR>>tGRSoyurfswKJ^m>puUJbofo5rl9Dg~ZyMt8(65n!!PI~Pkrxc8
z66j6YD={a4xsjH6>M4OoBj!Z1pB`DM(j7POTx&XA|;lmkcSjjo_G!Mo-bASj29G;#>dzvxcndHW4
zPkWyfK3$rcw#~TtnLu;@m?RFh{GN$&fSuwCvI!uH2Y2<%cXIZ6{i#Q{N3P5s<-u>RWX~k7Sr8Qx<1lyQ^i_+>KV(h&X@P3Gw{!MivB9F25A%e
zM}q4q8dbwssg?GUy?AmxvT1Jkib9nAZP$=(@vQPnC>s39v0@ad7?Zn>Qa8P?b#YZ@O&ia6gBiv@nN_3Ol$k62))R1&`c@!w{14qCs55;5
zImp_)+rfE#&|&7tKX$5)EdeHAKD{%xlXG{yO!8`lIE$928aY8P6?=a83F`yRciT;6
zGzYUeYCdsM3D3?-`_N=9C|)tTKHFY1u6vr|(I?&+p~nQFg_q!|^WT^`1dHSRm3{(s
zyqs_Acdkb~2(J@Co(O@IvS?p7L}t6>5Yq&LPgtDcJ%T_QvQq?(0#4c8hqqSOuI30#
z*cDJexi;4ute0Ixf|f`(GNae>CkaTwBk8xw>uemr-^xwf=zVn4Q9rX~a^>wJ*;sJO
zG{h7jkz$`5%>)sL{?ae_=Yt+zaud=0-)!}`ol;{_b|1M#@yYRh8R36Q?e~DyGNZ3P
zOI5k;78>KGM%EZ}w
zMHdC{qR<>`Iq&$FYRj$&|A67Z^_USIytP`Um4s?&)y91$_kPy<^7em%+e^F<_+d{y
z!$E}9h6LXZfa9m0EhH{J6LiEuEx!w#`DLWytuaC20KLyIC6r)F?(gBXCoXq%2W&<4
z4QDY%o8f1{q=Uf9Pywwnxj|}W8Pk=TUr|sXgo4|a-uMHjS}Q%{D2IU|*n@GZ3%;xu
zH^ye)8$5L-?6c-)3Te|0zY(b1&#NYZs11#JsxOT!Z&uIHID;$Zy^Zs1FYz?jnH3Pb
zH3A|Z>H0L?(kc`t^#y*){exxFZ7e)v#i1?LjTGl4;odg6tYy+LKa^Rk3WHH7fSlx6
zjq8Wz*Remg=z>k-_bw}l6KT?4Kye*W4{Wx&LZuTuqeOK}uueN2%>e=mc-0-`W+{SP!*HxhAfwG^Nw?fmIP
zv|Uot^Ei*i5#T3Y&U0iA*>ZFaR`-oQ?HX}@P)k~k`lQ&4%q!HOA}f2iDJfRWirsc+
zmE)fe0}M-=s&(I0$9YKASwIDO3~m-HDcZP&*o9JcTB;@1ajUm+q0sY3kBF=q$2y5a
zKADonlDO2g%>|f#ID053#T>MZlH;x7l`Dto{Zqkhp~om5a|NMuAL@6^oT%W@e)6n|
z#*bZEL65*Ob&~>-&9jMypP906*!3g%-!RJH9)!9^IFFR)vPRLeEWR&<67U0&PQ)|`
z8%_w{rwcw6j4uhJ1y)fgx|?%`dDKyVRNklLUUifeq
ze#)bH1z*d&==T%)xZ;ebOF^nBmff?iWuHG9?PdBerGIt4ah4}M7`9nOv$lUXXxy&U
zX{L5Deg8kt3lNR~rbLG|0kZz^5t3$!Pt;RCvpN1$S3<2`af4FzXhYqR@j2&e%aW{L
zhv&v%A^XB?8hzO9ybtr(tI7Xeb9ILBu#cZve-p#~$UCsF`tuG6C9Ral-TbDo
zt-64SEVxZTyGamFP`abzg+JELhVC}lC!g;GGEIN9=}caS4ayRo^lcg%c|8ad_=Fue1cNw>Sgk
z!^nx})x@+3(q%;mZMvyQH!1-OZgw|O8DZJX+{$Ee%09t#DceOe1%;iv-^*G33yVf@
zVRu^Zw;sRV=pUk}qHDuVbm6F4I@lCj^yX*?7yf}$N#j9Uv(C;VJb~4pPL0*mtW1H~
zm@aYrSYqDuGO!}M-A^+3*_hT#$8Rwx4CI)d#eyGLw@De%?E3A^A9K#>Lri82ufF6w
ze5aZx9;2+8?K%!7na!iB@k;Ywry02||9yF(`ja8g9?U<`GZ4;mF(`_!GdLd7YCL&!`%=PFCXEPld^by9ZRhF#0nhozKGpKW}&+{c{&(67SAI>6g}~mnYDxu
z=O@AFIjO4Mg&FFXr_-5l$V)m<`cz?+1d0H1$!K*`VXhIA&i%;im6K|t^kdhBA>u1K
zs-GI~7S?LWhYC2iIgMfp;F=V8%P(WNWHj@WrETIDHkx_FSzYvn{HKh0o3&4}*SgHy
z*k;{W8O3<-rljZ{Umv2V$Al}Uj6VA}OH)c?+ZM}t!wc)Mq1F3P(z2KF5$E>jGQ
zfb8Zci(_(&m3O`U?p?&>qn{t$wS7H(vlNM#AS_W(RkaZveBP*3)#U67>30)IQtq*_
z8ALa-dM@Esk#h;SPGS0hCu_>d3Bmx0ahlJ$^{8PjAFA^nNZik!VEG8pqCBy{zSo>~
z%co;KrlGCRgId(AvrchVwkCN_1WTiE+wLbCm>*s_YQ(XwYp3;@VwG$lK01a;bUWQ^9l7Im
zj(umkV2pc)Nfl3QFl0v=hL_$;G>!HpMEK-kJBW;T2L^%g&yBD2R^7-O^Iqx*t{-TdW~8<4-D#%eI?Rf^!RfM%rre
zFbG>@DCbuzhCK_azh{nZ<#v6yaNustV`w|vFNl-a4K)o7|2Yc?KVs2T@Oi$q0I?N>
zC0?|hn6FY3{Gb4oyM<6uoP31g9&wIm3o)l{a-IoKf19aGSR^6#kj!O72aykAc+s&S
zSjBcS%Ym&WQ)JVT;(lZ%S-v5BTGC|M+Z`6
zjAx9kW>eC7P4mH2i@}#|rw=JjZk16;LrxpZ{R23nd}H#M({z#(N;D+0~U
z%gtX(f-;-aP8NEudCQL#V$5)Luab6kD7&|AEL1mFN-24ns*$tF&`}s84nT(crzXm
zZnAsI3P7x#c3k!=!7KwQzotC9Vqg{tF+uROfWFg1{F`^GcS-t{sptGrcg;%yCnQIZ
zCgX2L^Yy#0kfh}^lRcFCeXiS|Gf0@pKD$UvQG$QQ5~)KHwhPJa=6Kx(KNW@{#9}yu
zAfOGgHJY-1_=dr@Yr1NOm&2FM7ye6#QR3Jp&KfDR+>yXkwKp+se|WBY^
zvXteih|5%eJ;iqh1-CpXdN2P}PVGEGbWkqZunUW&zypZ{$xt*sw6&j+{_!1G#^h&%
z#N=o|F_)2Tl?BACTAx%@@7zsi?XsqRSTMgO@~*)+;~VfwzX(@W*Pv)#L^h}iKN|{b
zgbY^_vT9?Q?xuR@#5Mxf4F2#DXhL-9S)CA}I(XGv4$-Li6#4}ZyaH)IDh9FX+_|m0qRpBe>4{t#wDX6qt)c5848LSZaPD3vuIgw
z2@+pE>O6Kn+@3tY^m@4U-wkj6%KF_k_`NySFD#x-B09M()r0#wD?>?n
zE_s!Amyxb%`v(4)qs`sd>RbJP9b*G;K56X@}x=E6GH
z?bYKUjx31y^fJ$y(ZJa&$JwoOE2(}nAy0;hP$Fin=r83k#ZWgAvob>YhZ>2Kc^!!J
zNf9FWw3dSi+V;=onxW`pSEjm-dr4qUQU;ehO&NKnZ^eWMDhb@A(IwOV>VeK_n^D4u
zscrASIq|8ZVXfk#)Gt?hd#ZQ@qX^a?EUlc#-IjAb=5b~FXY%CtMUro}8yxlmu+n^f!p$*i=zeGAYE=!9`P?Ytr+
z+>d~&RbxoDk5GX2wh=uzLG**k2Bg}1bNJmC^ybGtWKVXH0I;Z)8vcavp|2GKyDcH{
z1-dSC0tj3>!Da_PGkrZ3sx2Kl{{^IS+MSBj^0)nuwD311&Vl{bJEnRE?t!
zoja<4z{9|b61ptJ9-|-nl=XO9L~AbhK90Wj>4+`NtK;MLnl^ym;j)#*^^Y_ghXt{O
zA9b*l+wZ-&l)@_z#GQONU|UWW!YNUnHC@go8SR4VExAqwt)@SKJHCXW42!zwW-qC~
ziUg~ows$sU@fb?t+eKPOEaRz=B}|?PawIw0D`%dn{N#UyOHs*cCS^QT{)syuCVoUI
zr<9Op|8Rzr=c}H~w}fXC>o}-AOTM%0R=e3GgnjJEJK!bvV!KU6v#%_-c(f-xO(}@x
zu&qloC(V(;)C(;~FE8{&%>507?gR!atZgK7Kg=;iEd(>f3{o+0!AR~Oqb;T%9{
z$!HjCLogPUx|F*qE8AhJFLbZE=Uj7@8Ec8JzbW%fks^Wa3y1bEPbgFOhp9vM8^(Gx
zt_%&aE-^aY#R!}Lbtu+RFav8P+cMVj%DS*ywCg#;!(+kix;g7|)r)9|ke3o54%uwD
zi0@r;?{l0OnI%LEj0+r)+5_6t{5#)KC5R>YwMz>;m7&W_0#J6CY$Z8Hza_X?J%>3S
zwV>rJh*f&FME*&nN8O2f^{{Y2r^T0q;|HrbnsIj_>OB~KMToC9f6;bDgj@3=Eg~6`
z1;f_2iPUBSWa#b@Q+py0gn~%IdqwP-NIq!mZx?e*+_3FEH0DJ63VjzU;N$
zaq!lt65C2wVns)3-dcli9Pj?=_cva>zu4ICoG>;OUI673fg)`!ZoR^wdE}+MBYUIU
zc8(gc8!u^(`Af_2u#)cUaC>2xWKWY9+9w};S<~7cKmh72w3%w{#{T(isEyjBR@e6)
zHvFRE<6y-EBFhd`>xb(mfu+?Bq)(o;ybF=!GTo^
zxbcCI!2}T!;(F+5FkzMJf0nD)e#esFb6yX-C<^ItyDwI4`eH+exAU9~+j)|oNemi&
zKFN*};na4N6Og9C6&P%^p?`aZNdFeq^+t+2&l7JVc`7f;a9Vz+9hbW`@rA!-i2XDQ
zi*h(3t;OvpY@M-?p^*Hv#X`9wmdyzOZQ__N71D);euvX5x=*s|)swttmUM=^pVCj`
z#q8fQEMt7eVU>ZtHim*Xq+MdA)XG%E^4cfYtoOVpy@v&;lKbEMkp$0J&`
zrEtL5Ue8TQm+znd!*NW+W~FI9J=<-fA76y?&dz%;7@MzTJj|K*CNv;oDh9!e@H!q?
ztbvFvRydnk*fXafjomE5uvPd=rV>Hc#R@S53k0#bM)HV_{xnas17M>iUKFp=!X7Rb=_vM4>dur*4a}
zEavNy5mNMDhV5zFh@$Hcn^Oz6dZu
z4Wro-z(LDmabq+Aa0TOtT^&elhJjoWT|h9%HF^`jkC^!!-d+MvnTIj%5%$vLCx{L)
zueyZln68FJMzWD6l)UiQ(}RuML;RgsM)er(A*F!fV|6K~&`x1iCSR!s0A{&Q6v~Bv
zW*1i53Gb`3X}r!@*mOQ6o(L}OFL$b)IE6Tq8!WT#^8a%gk)z6}dwP?yJFd`A)$t7O
z2CYozt~F$)uef8qWaO63&IVL!0l06S0bX?s6vxM^UsBZ9)Y1QKT!UWwxwL)~gF`DV
z^MUsU|G=0}+G#!ZP*$AX)=>SUYD;^SUHZnHL7_6OD&XE~*>5s9JDU-J#@7`nNP{s6
zEIq=-3D4c!niOUI>W$Ap7{&geS^-!rRw#ldt*>`SexDG=0PN6oA(tE^21*6(0r7o@
zm`}rTH=Y6J+yHgNJt4yT+#9-E8V&4mzRQluj)Q5UDZ7G`
z^62fBqOJ=|j^qpnzGYVk=%hq{OiGl>3GY1L8e%{D}(-*~7$$xygtwEL$gzZ$G-V14&5NZ=E6m{ppQyIJa=
zp44M-8auw23B=nKe>tq>_Y*moU+V*MP8znYmVtj`P4@eG!}$`h*(@goI2OzTM392_dmw;NClP?
zQes{NA^A~?xB>A*3W{)7UOp-;VDxDMVD(2ztz)?4V0_IpeilXi&spChch|YkgW>Ib8z?+d
zGOhp8-kfGMf)^=(Ry|$AsaaoKjfKd-HvY_0l3w}vou1nNd~GF`&-M@*8L(@nDNQA;
zVePk%3w8efnEXqo{klDG
z6ruTrC1ad2r(8C>>$|vz?EMous1~Z8uvY3Lc^*dlsH7;lE=2f~zc!M6R_Kx~0hzc3
zHV@(v97hPRi-G6j3qSH=x_kDI5SZyT%_V8A&(cT#2cZWjm@NuH}(_Ls-)=X`F3lWAd{^Fl=&-CYkM`_*g-cQ;Gp
zYTRpT(OR|hg3SF{W($gJ!a*D`v1QS>K!`bK5DhPYFDEYVvzM2Xy5LWQ-TY7%u|J{~
zX%uG#HgbSS@YU5R44ObB-j6n9G&^A%yj9-2$_3b3O4LVfzH`RwHRmXlY{AM9rOtSL
zlo?%dvo-OkqwgxUW9&_WF!>EO%=+!faf8^Kq~N55wQWegZ2C5b19S>0jeEGLIxVig
z(=07o4VP8*|4`e;8#Yo=NnV(i=RT4qh)!)Aq1>h>94;z(B4yQxuiI(n)90_KT;D@=
z9ceiLei?9n@GL|v8^RRR_3s>gD~o)U|xr%VqeES#CScUX41nQNL_C~9>j}`&*iDc
zx*$NA+<7TpBlbvw@Erm$&fqTpNB?F#Y#4c3dL&eH=~keM0b9%;Ekjwzqr68}{6*GI
z82Z0$naFY%&xU8PLh>}-F5oiu-vpNPD7m?icW}Pmm3_wZ?Bmws;RN&9_;U7*re4R(
zjKG2B29-`Y`NQj~D$i4f5d0))jOK4d%n8cbRKXpU`Cy&vQV_T>FsbU#t{qhRbm=kkBCw`|*t)e7G`z)4ftb7E#%{)o
zMM7yv{C4eoM~%SkBznWc5rlOFz~730Dy<*i+=lSD3$OzfR3}>SqaclEO)O!e0uNAG
zhY@IBWaVxpY_#>!?v^24{!j)d(Ks2ICRXMD?);Pf*<8Z5jYqQn=m=Y;CbPR{miw~j
zYmdmsmiN0gm0a$d^59;XO0xW&+w%BXP0XK!|0>#}MPKK)E@8e~H`IJ<#zsp{KhNRW
z_Mb7_;zdLGBI$`1J3qX`pZf#+*Vs-xJ#@*w0>68nE&21|H_yXF-L(h}v#o78%f=Nn
z=+2F)4O)CgWr$Z}>n1gFx_;rUE6xfvXFy!%q3b0c6e}Op9E{VRQ+dQ3esE2HLkE<0
zJU<`45QQQ4;Nk7yuw#`SIR(HWC6!fkknh;ov)?%j#$qlul1$*FO
ze4SrP=wa|w!!=z}h&d?5@fyyS4VN8nv{(u_5*^bLP(MNnnxSB=ctE@$Gk!i_;xO7H
zL?Xke-~T#<`kUjEo9rG7XM2>UsKd_H)gaUmgIV1C!4re}+@KjBhlr^=LiNk`Tw5OW
zTYbocPm(*<4d9W)lx6rc?#=F|YCHY!;lY^V?v+9(_U>4p+uk2o98fXCeV30-*llPMw^3sb@T^2M0-0*>KjU=R_-bb^O5
z1I#D^ubs@@568ug6cPO>#AycC@b0v-pG`=Ds`h`2({0!x|E7w9Im5sj4}>&BS19xt
zHbMT%p2M!@_#!NrSNqwMYAWb72OeE!{33||ZA*I15iF{vW3^`YTGCvFw7=sbG!Q^M
z!r8&-daGD)*r}r1ME~OSlY5Sar;F6op%0^n-<5B9iWT+FLiOB}!0J@}|5~PWPJuuH
zUvmmC`oOCVvy*ojc4KDP+)rY>^H%Zm^Nd5+(KG__I&@EiBHs13v#M9juQ;`;(2JC;Ioy?Gf+W^$Xc-^L?m^O5uaS+7qY0
z_F$(n^y<_n2%JCYM*5kSF2klx9(K%7d){Hysy6qeFNxVa+%y>wABtQ?MWOZecFQ_0
ztM>X>`Ak5a_?ewiV*ZC0#9j5XpgQV~v6rfWeOb&l=A;?#4YD!LYHWv+UqD=hSc6f?
zMz;Y>!2n{hWy_|GWqD5$(Ai<1FcHneU=5Je4V_9G&UIq=0P@s{zPR&@T<858k21!8
zd~zKBjiA;#fkn7*tRF
zRn@;ZJVI^_cq4L16DRa#Hum0bXv0?P#qkn=8}8@e(Nqxt`rp=w)la(mQ)CbhM!Wmci6{Y`DPjbsiHq_{>
zGMk%v)!+&MOcW%r_tDSTKQCPW4b9DXdv(w3)w+Rdq*uWqeV}KF3!b;5aAYc9KU}=E=6X
z``(OWlbCtjsO@Kzr%dR57e-a|`3Bys_uh^2<(3&j@puCYJd6sMdl079LpYU88L&Q`
z(YQd|hfsD-*dEFJf3*=)RP42n*k>+i!GPr=9lJFd@jAdnEZ&FNA(ANA5~q+3X@R4jN-w{c{5seBTfC%nrT2UCCqx3Cxq^DmnO1Yx7oNb2$ub
zIi72iZ;?K$o^-jXvQpt)W**~=VIb7Qc)Wok7zjK^L=B)zO&h~aWo?9NBWbSAZ#j~f
z(KblO9IW3;%Q|P5ip+H>YO_j}Z&is)4W*8qjSNh;EyS8pf=LAf%;Mh2MXLTHzDoQP
z>L9{AhLG7x{1(GwbG+SQheJP1LMH4pB*2aTCWY+g7KH=(P=CKF>2lG@UL8PT#p&!n
z8$5EQ0ZHRa;x#bXq!DYP6p7ZqNDn6MG6#G6d8g7(Eu78)l#%igIB+eR5z@&bRvByG
z#K2V*zoD6x%ZmJ&N4shg=udT}FnatH5sU)iZv&ik%l*@X)TYPJ+SbUwZ2T2PymPJI
zPnV=qQ~FYh2)MB}J>$?vWrOM84gLltLqpjWK@*h3MY)eX_ND&@eo%$MwkOU4-c6^tJ6`&zkgpu)~&cx_bU2&4B5}%vnG|e%?
zdxLj4(G9HB80-CY^o6xDPBfb~P*=2?C;ly^i{7*Y^Hf}@^$wlAMJ4^akBD?bCG$4=WCVX5rNTFRR*{m`5sz6#tBKDZ`K
zz(~hr&b$!}<`<+1(}r$Vn00VtR@jZE$mT}w)!CF+n_i}*6~R7;W=jWk^$kSz)-5_)
zDg4pu(XfJlGb{%Yq@$rLDLln4d)HDDzD~({x{Egx#D*+}2&b!6dy3lTdZxj`617!P%jWZ`U6+p><2H(srW^97VJ??X9k
z+}^}q3>krJ@8bBmNnrB+73A=Y}5x$@qUwTjUrt9!_2l(hbTD}f2O+v|%m)L~r*_F-D
zF1ZA&ay^UX6XHP3)0R0I)(V>ymgUYG%;=pI{s2*%V+%5_+1Cs)A9YP)QS>5b5+fXx
z(s*W)#K&%WbQm$f+ZneJ>oRW^)he@#()Y=bhR*zHi#P=`{Oz6_2QSM_-m&ZHh~n94Wv>6_
z&&yDJo?|EgJzFlsj(d9mk$Q_>N1-+?rK*HAgV7tVuen?l`zRugbj6HZ-D4!@q_zeQ{aG$#{5N
zv0?~TU(k$#j)4&m;rwU|b{5`7}AT$^?6W*Z5&
zv;Zh|{Mix?O$LCIvA&7D
z9x(6il(8R)@od%`mI}|ww_^SaVT!0t<(K9T-r7Ehof3mJwhi&v=FBQw74v1^z*Juu
zDU<-PKUo(O#}gS#CztN9xGeTx5h1kabwzIy%YZ6aUD4N@0woS!A@_@EQ9_oByr}Dl
zWv8~G`D4}qdqj7%PA-e&?k0wbV5
ztWsv}Do4o&l^0*ETKDT