diff --git a/README.md b/README.md index 63364e62..176173fe 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Hardware Setup: ***Important Notes***: - The time data that shown on the following tables presents the time elapsed from preprocess (resize is excluded), to a forward pass of a network, and postprocess to get final results. -- The time data that shown on the following tables is averaged from a 100-time run. +- The time data that shown on the following tables is the median of benchmark runs. - View [benchmark/config](./benchmark/config) for more details on benchmarking different models. | Model | Input Size | CPU x86_64 (ms) | CPU ARM (ms) | |-------|------------|-----------------|--------------| -| [YuNet](./models/face_detection_yunet) | 160x120 | 2.17 | 8.87 | -| [DB](./models/text_detection_db) | 640x480 | 148.65 | 2759.88 | -| [CRNN](./models/text_recognition_crnn) | 100x32 | 23.23 | 235.87 | +| [YuNet](./models/face_detection_yunet) | 160x120 | 2.35 | 8.72 | +| [DB](./models/text_detection_db) | 640x480 | 137.38 | 2780.78 | +| [CRNN](./models/text_recognition_crnn) | 100x32 | 50.21 | 234.32 | +| [SFace](./models/face_recognition_sface) | 112x112 | 8.69 | 96.79 | ## License diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index ada01fa0..a2338663 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -78,7 +78,7 @@ def __init__(self, **kwargs): def _load_label(self): labels = dict.fromkeys(self._files, None) for filename in self._files: - labels[filename] = np.loadtxt(os.path.join(self._path, '{}.txt'.format(filename[:-4]))) + labels[filename] = np.loadtxt(os.path.join(self._path, '{}.txt'.format(filename[:-4])), ndmin=2) return labels def __getitem__(self, idx): diff --git a/benchmark/config/face_detection_yunet.yaml b/benchmark/config/face_detection_yunet.yaml index 6f7e78a9..a61a99ab 100644 --- a/benchmark/config/face_detection_yunet.yaml +++ b/benchmark/config/face_detection_yunet.yaml @@ -1,7 +1,7 @@ Benchmark: name: "Face Detection Benchmark" data: - path: "benchmark/data/face" + path: "benchmark/data/face/detection" files: ["group.jpg", "concerts.jpg", "dance.jpg"] metric: sizes: # [[w1, h1], ...], Omit to run at original scale diff --git a/benchmark/config/face_recognition_sface.yaml b/benchmark/config/face_recognition_sface.yaml new file mode 100644 index 00000000..88ac9066 --- /dev/null +++ b/benchmark/config/face_recognition_sface.yaml @@ -0,0 +1,17 @@ +Benchmark: + name: "Face Recognition Benchmark" + data: + path: "benchmark/data/face/recognition" + files: ["Aaron_Tippin_0001.jpg", "Alvaro_Uribe_0028.jpg", "Alvaro_Uribe_0029.jpg", "Jose_Luis_Rodriguez_Zapatero_0001.jpg"] + useLabel: True + metric: # 'sizes' is omitted since this model requires input of fixed size + warmup: 3 + repeat: 10 + batchSize: 1 + reduction: 'median' + backend: "default" + target: "cpu" + +Model: + name: "SFace" + modelPath: "models/face_recognition_sface/face_recognition_sface.onnx" \ No newline at end of file diff --git a/models/__init__.py b/models/__init__.py index 7716cbfe..200b952f 100644 --- a/models/__init__.py +++ b/models/__init__.py @@ -1,6 +1,7 @@ from .face_detection_yunet.yunet import YuNet from .text_detection_db.db import DB from .text_recognition_crnn.crnn import CRNN +from .face_recognition_sface.sface import SFace class Registery: def __init__(self, name): @@ -16,4 +17,5 @@ def register(self, item): MODELS = Registery('Models') MODELS.register(YuNet) MODELS.register(DB) -MODELS.register(CRNN) \ No newline at end of file +MODELS.register(CRNN) +MODELS.register(SFace) \ No newline at end of file diff --git a/models/face_detection_yunet/demo.py b/models/face_detection_yunet/demo.py index efb07970..dc100f3d 100644 --- a/models/face_detection_yunet/demo.py +++ b/models/face_detection_yunet/demo.py @@ -77,8 +77,8 @@ def visualize(image, results, box_color=(0, 255, 0), text_color=(0, 0, 255), fps # Print results print('{} faces detected.'.format(results.shape[0])) for idx, det in enumerate(results): - print('{}: [{:.0f}, {:.0f}] [{:.0f}, {:.0f}], {:.2f}'.format( - idx, det[0], det[1], det[2], det[3], det[-1]) + print('{}: {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f}'.format( + idx, *det[:-1]) ) # Draw results on the input image diff --git a/models/face_recognition_sface/LICENSE b/models/face_recognition_sface/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/models/face_recognition_sface/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/models/face_recognition_sface/README.md b/models/face_recognition_sface/README.md new file mode 100644 index 00000000..71a3c94f --- /dev/null +++ b/models/face_recognition_sface/README.md @@ -0,0 +1,27 @@ +# SFace + +SFace: Sigmoid-Constrained Hypersphere Loss for Robust Face Recognition + +SFace is contributed by [Yaoyao Zhong](https://github.com/zhongyy/SFace). [face_recognition_sface.onnx](./face_recognition_sface.onnx) is converted from the model from https://github.com/zhongyy/SFace thanks to [Chengrui Wang](https://github.com/crywang). + +Note: +- There is [a PR for OpenCV adding this model](https://github.com/opencv/opencv/pull/20422) to work with OpenCV DNN in C++ implementation. +- Support 5-landmark warp for now. +- `demo.py` requires [../face_detection_yunet](../face_detection_yunet) to run. + +## Demo + +Run the following command to try the demo: +```shell +# recognize on images +python demo.py --input1 /path/to/image1 --input2 /path/to/image2 +``` + +## License + +All files in this directory are licensed under [Apache 2.0 License](./LICENSE). + +## Reference + +- https://ieeexplore.ieee.org/document/9318547 +- https://github.com/zhongyy/SFace \ No newline at end of file diff --git a/models/face_recognition_sface/demo.py b/models/face_recognition_sface/demo.py new file mode 100644 index 00000000..5c64b99c --- /dev/null +++ b/models/face_recognition_sface/demo.py @@ -0,0 +1,71 @@ +# This file is part of OpenCV Zoo project. +# It is subject to the license terms in the LICENSE file found in the same directory. +# +# Copyright (C) 2021, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved. +# Third party copyrights are property of their respective owners. + +import sys +import argparse + +import numpy as np +import cv2 as cv + +from sface import SFace + +sys.path.append('../face_detection_yunet') +from yunet import YuNet + +def str2bool(v): + if v.lower() in ['on', 'yes', 'true', 'y', 't']: + return True + elif v.lower() in ['off', 'no', 'false', 'n', 'f']: + return False + else: + raise NotImplementedError + +parser = argparse.ArgumentParser( + description="SFace: Sigmoid-Constrained Hypersphere Loss for Robust Face Recognition (https://ieeexplore.ieee.org/document/9318547)") +parser.add_argument('--input1', '-i1', type=str, help='Path to the input image 1.') +parser.add_argument('--input2', '-i2', type=str, help='Path to the input image 2.') +parser.add_argument('--model', '-m', type=str, default='face_recognition_sface.onnx', help='Path to the model.') +parser.add_argument('--dis_type', type=int, choices=[0, 1], default=0, help='Distance type. \'0\': cosine, \'1\': norm_l1.') +parser.add_argument('--save', '-s', type=str, default=False, help='Set true to save results. This flag is invalid when using camera.') +parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.') +args = parser.parse_args() + +if __name__ == '__main__': + # Instantiate SFace for face recognition + recognizer = SFace(modelPath=args.model) + # Instantiate YuNet for face detection + detector = YuNet(modelPath='../face_detection_yunet/face_detection_yunet.onnx', + inputSize=[320, 320], + confThreshold=0.9, + nmsThreshold=0.3, + topK=5000, + keepTopK=750) + + img1 = cv.imread(args.input1) + img2 = cv.imread(args.input2) + + # Detect faces + detector.setInputSize([img1.shape[1], img1.shape[0]]) + face1 = detector.infer(img1) + assert face1.shape[0] > 0, 'Cannot find a face in {}'.format(args.input1) + detector.setInputSize([img2.shape[1], img2.shape[0]]) + face2 = detector.infer(img2) + assert face2.shape[0] > 0, 'Cannot find a face in {}'.format(args.input2) + + # Match + distance = recognizer.match(img1, face1[0][:-1], img2, face2[0][:-1], args.dis_type) + print(distance) + if args.dis_type == 0: + dis_type = 'Cosine' + threshold = 0.363 + result = 'same identity' if distance >= threshold else 'different identity' + elif args.dis_type == 1: + dis_type = 'Norm-L2' + threshold = 1.128 + result = 'same identity' if distance <= threshold else 'different identity' + else: + raise NotImplementedError() + print('Using {} distance, threshold {}: {}.'.format(dis_type, threshold, result)) \ No newline at end of file diff --git a/models/face_recognition_sface/face_recognition_sface.onnx b/models/face_recognition_sface/face_recognition_sface.onnx new file mode 100644 index 00000000..49f8a699 --- /dev/null +++ b/models/face_recognition_sface/face_recognition_sface.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9279be00242f4240ba4de7b6b93837fdd5a17b98d2f278f63711d55b65b4312 +size 38696080 diff --git a/models/face_recognition_sface/sface.py b/models/face_recognition_sface/sface.py new file mode 100644 index 00000000..81fff0d0 --- /dev/null +++ b/models/face_recognition_sface/sface.py @@ -0,0 +1,165 @@ +# This file is part of OpenCV Zoo project. +# It is subject to the license terms in the LICENSE file found in the same directory. +# +# Copyright (C) 2021, Shenzhen Institute of Artificial Intelligence and Robotics for Society, all rights reserved. +# Third party copyrights are property of their respective owners. + +import numpy as np +import cv2 as cv + +from _testcapi import FLT_MIN + +class SFace: + def __init__(self, modelPath): + self._model = cv.dnn.readNet(modelPath) + self._input_size = [112, 112] + self._dst = np.array([ + [38.2946, 51.6963], + [73.5318, 51.5014], + [56.0252, 71.7366], + [41.5493, 92.3655], + [70.7299, 92.2041] + ], dtype=np.float32) + self._dst_mean = np.array([56.0262, 71.9008], dtype=np.float32) + + @property + def name(self): + return self.__class__.__name__ + + def setBackend(self, backend_id): + self._model.setPreferableBackend(backend_id) + + def setTarget(self, target_id): + self._model.setPreferableTarget(target_id) + + def _preprocess(self, image, bbox): + aligned_image = self._alignCrop(image, bbox) + return cv.dnn.blobFromImage(aligned_image) + + def infer(self, image, bbox): + # Preprocess + inputBlob = self._preprocess(image, bbox) + + # Forward + self._model.setInput(inputBlob) + outputBlob = self._model.forward() + + # Postprocess + results = self._postprocess(outputBlob) + + return results + + def _postprocess(self, outputBlob): + return outputBlob / cv.norm(outputBlob) + + def match(self, image1, face1, image2, face2, dis_type=0): + feature1 = self.infer(image1, face1) + feature2 = self.infer(image2, face2) + + if dis_type == 0: # COSINE + return np.sum(feature1 * feature2) + elif dis_type == 1: # NORM_L2 + return cv.norm(feature1, feature2) + else: + raise NotImplementedError() + + def _alignCrop(self, image, face): + # Retrieve landmarks + if face.shape[-1] == (4 + 5 * 2): + landmarks = face[4:].reshape(5, 2) + else: + raise NotImplementedError() + warp_mat = self._getSimilarityTransformMatrix(landmarks) + aligned_image = cv.warpAffine(image, warp_mat, self._input_size, flags=cv.INTER_LINEAR) + return aligned_image + + def _getSimilarityTransformMatrix(self, src): + # compute the mean of src and dst + src_mean = np.array([np.mean(src[:, 0]), np.mean(src[:, 1])], dtype=np.float32) + dst_mean = np.array([56.0262, 71.9008], dtype=np.float32) + # subtract the means from src and dst + src_demean = src.copy() + src_demean[:, 0] = src_demean[:, 0] - src_mean[0] + src_demean[:, 1] = src_demean[:, 1] - src_mean[1] + dst_demean = self._dst.copy() + dst_demean[:, 0] = dst_demean[:, 0] - dst_mean[0] + dst_demean[:, 1] = dst_demean[:, 1] - dst_mean[1] + + A = np.array([[0., 0.], [0., 0.]], dtype=np.float64) + for i in range(5): + A[0][0] += dst_demean[i][0] * src_demean[i][0] + A[0][1] += dst_demean[i][0] * src_demean[i][1] + A[1][0] += dst_demean[i][1] * src_demean[i][0] + A[1][1] += dst_demean[i][1] * src_demean[i][1] + A = A / 5 + + d = np.array([1.0, 1.0], dtype=np.float64) + if A[0][0] * A[1][1] - A[0][1] * A[1][0] < 0: + d[1] = -1 + + T = np.array([ + [1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0] + ], dtype=np.float64) + + s, u, vt = cv.SVDecomp(A) + smax = s[0][0] if s[0][0] > s[1][0] else s[1][0] + tol = smax * 2 * FLT_MIN + rank = int(0) + if s[0][0] > tol: + rank += 1 + if s[1][0] > tol: + rank += 1 + det_u = u[0][0] * u[1][1] - u[0][1] * u[1][0] + det_vt = vt[0][0] * vt[1][1] - vt[0][1] * vt[1][0] + if rank == 1: + if det_u * det_vt > 0: + uvt = np.matmul(u, vt) + T[0][0] = uvt[0][0] + T[0][1] = uvt[0][1] + T[1][0] = uvt[1][0] + T[1][1] = uvt[1][1] + else: + temp = d[1] + d[1] = -1 + D = np.array([[d[0], 0.0], [0.0, d[1]]], dtype=np.float64) + Dvt = np.matmul(D, vt) + uDvt = np.matmul(u, Dvt) + T[0][0] = uDvt[0][0] + T[0][1] = uDvt[0][1] + T[1][0] = uDvt[1][0] + T[1][1] = uDvt[1][1] + d[1] = temp + else: + D = np.array([[d[0], 0.0], [0.0, d[1]]], dtype=np.float64) + Dvt = np.matmul(D, vt) + uDvt = np.matmul(u, Dvt) + T[0][0] = uDvt[0][0] + T[0][1] = uDvt[0][1] + T[1][0] = uDvt[1][0] + T[1][1] = uDvt[1][1] + + var1 = 0.0 + var2 = 0.0 + for i in range(5): + var1 += src_demean[i][0] * src_demean[i][0] + var2 += src_demean[i][1] * src_demean[i][1] + var1 /= 5 + var2 /= 5 + + scale = 1.0 / (var1 + var2) * (s[0][0] * d[0] + s[1][0] * d[1]) + TS = [ + T[0][0] * src_mean[0] + T[0][1] * src_mean[1], + T[1][0] * src_mean[0] + T[1][1] * src_mean[1] + ] + T[0][2] = dst_mean[0] - scale * TS[0] + T[1][2] = dst_mean[1] - scale * TS[1] + T[0][0] *= scale + T[0][1] *= scale + T[1][0] *= scale + T[1][1] *= scale + return np.array([ + [T[0][0], T[0][1], T[0][2]], + [T[1][0], T[1][1], T[1][2]] + ], dtype=np.float64) \ No newline at end of file