update face recognition

yuta0306 · yuta0306 · commit 545ece355389 · 2023-07-06T15:52:42.000+09:00
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -1,9 +1,9 @@
 name: gh-pages
 on:
   workflow_dispatch:
-  push:
-    branches:
-      - main
+  # push:
+  #   branches:
+  #     - main
 
 jobs:
   release:
diff --git a/dslclib/src/body_controller.py b/dslclib/src/body_controller.py
@@ -1,6 +1,6 @@
 import json
 from dataclasses import dataclass
-from typing import Optional
+from typing import Literal, Optional
 
 from dslclib.src.base import BaseClient
 
@@ -26,13 +26,13 @@ class MotionType:
     righthandbaseposition
     """
 
-    Default: str = "DefaultMotion"
-    Greeting: str = "greeting"
-    Nono: str = "nono"
-    NodDeep: str = "nod_deep"
-    Nod: str = "nod"
-    RightHandBasePosition: str = "righthandbaseposition"
-    LeftHandBasePositin: str = "lefthandbaseposition"
+    Default: Literal["DefaultMotion"] = "DefaultMotion"
+    Greeting: Literal["greeting"] = "greeting"
+    Nono: Literal["nono"] = "nono"
+    NodDeep: Literal["nod_deep"] = "nod_deep"
+    Nod: Literal["nod"] = "nod"
+    RightHandBasePosition: Literal["righthandbaseposition"] = "righthandbaseposition"
+    LeftHandBasePositin: Literal["lefthandbaseposition"] = "lefthandbaseposition"
 
 
 """
@@ -96,9 +96,9 @@ class GazeObject:
     Sofa
     """
 
-    Monitor: str = "monitor"
-    Desk: str = "desk"
-    Sofa: str = "Sofa"
+    Monitor: Literal["monitor"] = "monitor"
+    Desk: Literal["desk"] = "desk"
+    Sofa: Literal["Sofa"] = "Sofa"
 
 
 """
@@ -129,8 +129,8 @@ class ControllerType:
     HeadController
     """
 
-    Eye: str = "EyeController"
-    Head: str = "HeadController"
+    Eye: Literal["EyeController"] = "EyeController"
+    Head: Literal["HeadController"] = "HeadController"
 
 
 """
diff --git a/dslclib/src/expression_controller.py b/dslclib/src/expression_controller.py
@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Optional
+from typing import Literal, Optional
 
 from dslclib.src.base import BaseClient
 
@@ -12,20 +12,20 @@ class ExpressionType:
     ExpressionType()とインスタンス化する必要はない．
     """
 
-    MouthA: str = "mouth-a"
-    MouthI: str = "mouth-i"
-    MouthU: str = "mouth-u"
-    MouthE: str = "mouth-e"
-    MouthO: str = "mouth-o"
-    Normal: str = "normal"
-    FullSmile: str = "fullsmile"
-    Smile: str = "smile"
-    Bad: str = "bad"
-    Angry: str = "angry"
-    EyeClose: str = "eye-close"
-    EyeOpen: str = "eye-open"
-    EyeUp: str = "eye-up"
-    EyeDown: str = "eye-down"
+    MouthA: Literal["mouth-a"] = "mouth-a"
+    MouthI: Literal["mouth-i"] = "mouth-i"
+    MouthU: Literal["mouth-u"] = "mouth-u"
+    MouthE: Literal["mouth-e"] = "mouth-e"
+    MouthO: Literal["mouth-o"] = "mouth-o"
+    Normal: Literal["normal"] = "normal"
+    FullSmile: Literal["fullsmile"] = "fullsmile"
+    Smile: Literal["smile"] = "smile"
+    Bad: Literal["bad"] = "bad"
+    Angry: Literal["angry"] = "angry"
+    EyeClose: Literal["eye-close"] = "eye-close"
+    EyeOpen: Literal["eye-open"] = "eye-open"
+    EyeUp: Literal["eye-up"] = "eye-up"
+    EyeDown: Literal["eye-down"] = "eye-down"
 
 
 """
diff --git a/dslclib/src/face_recognition.py b/dslclib/src/face_recognition.py
@@ -7,19 +7,47 @@
 from dslclib.src.base import BaseClient
 
 
+@dataclass
+class Rotation:
+    """Rotation
+    ユーザの顔の向きを示すデータクラス"""
+
+    pitch: float
+    roll: float
+    yaw: float
+
+
+@dataclass
+class OutputForFaceRecognition:
+    """OutputForFaceRecognition
+    感情認識サーバのクライアントが出力するデータクラス"""
+
+    summarized: bool
+    timestamp: float
+    emotion: str
+    emotion_score: float
+    rotations: list[Rotation]
+    age: Optional[int] = None
+    gender: Optional[Literal["Man", "Woman"]] = None
+    gender_score: Optional[float] = None
+
+    def __getitem__(self, key):
+        return getattr(self, key)
+
+
 @dataclass
 class EmotionType:
     """EmotionType
     感情認識サーバが出力する感情値を定義したデータクラス
     """
 
-    Neutral: str = "neutral"
-    Happiness: str = "happiness"
-    Surprise: str = "surprise"
-    Sadness: str = "sadness"
-    Anger: str = "anger"
-    Disgust: str = "disgust"
-    fear: str = "fear"
+    Neutral: Literal["neutral"] = "neutral"
+    Happiness: Literal["happiness"] = "happiness"
+    Surprise: Literal["surprise"] = "surprise"
+    Sadness: Literal["sadness"] = "sadness"
+    Anger: Literal["anger"] = "anger"
+    Disgust: Literal["disgust"] = "disgust"
+    Fear: Literal["fear"] = "fear"
 
 
 class FaceRecognitionClient(BaseClient):
@@ -58,6 +86,14 @@ def __init__(self, ip: Optional[str] = None, port: int = 4500) -> None:
         >>>
         """
         super().__init__(ip, port)
+        self.__age: Optional[int] = None
+        self.__gender: Optional[Literal["Man", "Woman"]] = None
+        self.__gender_score: Optional[float] = None
+
+    def get_user_info(
+        self,
+    ) -> tuple[Optional[int], Optional[Literal["Man", "Woman"]], Optional[float]]:
+        return self.__age, self.__gender, self.__gender_score
 
     def receive_line(self) -> str:
         """
@@ -168,9 +204,7 @@ def summarize_timestamps(
 
         return True, timestamp, emotion, score
 
-    def listen(
-        self, func: Optional[Callable] = None
-    ) -> tuple[Optional[float], str, Optional[float]]:
+    def listen(self, func: Optional[Callable] = None) -> OutputForFaceRecognition:
         """
         感情認識サーバとのソケット通信をするメソッド
 
@@ -195,27 +229,60 @@ def listen(
 
         Returns
         -------
-        tuple[Optional[float], str, Optional[float]]
+        OutputForFaceRecognition
         """
         do_return: bool = False
         pool: list[tuple[float, str, float]] = []
+        rotations: list[Rotation] = []
         while True:
             received = self.receive_line()
             if not re.search(r"^\{[^\n]+\}\n", received):
                 continue
 
             data = json.loads(received)
+            timestamp: float = data["timestamp"]
+            emotion: str = data["emotion_class"]
+            score: float = data["emotion_score"]
+            rot = data["rotation"]
+            rotations.append(
+                Rotation(pitch=rot["pitch"], roll=rot["roll"], yaw=rot["yaw"])
+            )
+            # user info
+            if "age" in list(data.keys()):
+                self.__age = data["age"]
+                self.__gender = data["gender_class"]
+                self.__gender_score = data["gender_score"]
+
+            age, gender, gender_score = self.get_user_info()
             if func is None:
-                return data["timestamp"], data["emotion_class"], data["emotion_score"]
-
-            pool.append(tuple(data.values()))
+                return OutputForFaceRecognition(
+                    summarized=True,
+                    timestamp=timestamp,
+                    emotion=emotion,
+                    emotion_score=score,
+                    rotations=rotations,
+                    age=age,
+                    gender=gender,
+                    gender_score=gender_score,
+                )
+
+            pool.append((timestamp, emotion, score))
             do_return, timestamp, emotion, score = func(data=pool)
             if do_return:
                 if not isinstance(emotion, str):
                     raise TypeError(
                         f"出力する感情クラスは`str`のみですが、{type(str)}があたえられています。`func`を要件を満たすように設計してください。"
                     )
-                return timestamp, emotion, score
+                return OutputForFaceRecognition(
+                    summarized=True,
+                    timestamp=timestamp,
+                    emotion=emotion,
+                    emotion_score=score,
+                    rotations=rotations,
+                    age=age,
+                    gender=gender,
+                    gender_score=gender_score,
+                )
 
 
 if __name__ == "__main__":
diff --git a/dslclib/src/speech_recognition.py b/dslclib/src/speech_recognition.py
@@ -1,14 +1,23 @@
 import re
 from dataclasses import dataclass
-from typing import Optional
+from typing import Literal, Optional
 
 from dslclib.src.base import BaseClient
 
 
 @dataclass
 class STTRecognitionType:
-    InterimResult = "interimresult"
-    Result = "result"
+    InterimResult: Literal["interimresult"] = "interimresult"
+    Result: Literal["result"] = "result"
+
+
+@dataclass
+class OutputForSTTRecognition:
+    type: Literal["interimresult", "result"]
+    result: str
+
+    def __getitem__(self, key):
+        return getattr(self, key)
 
 
 class SpeechRecognitionClient(BaseClient):
@@ -67,7 +76,7 @@ def receive_line(self) -> str:
         received = str(self.sock.recv(4096).decode())
         return received
 
-    def listen(self, interim: bool = True) -> tuple[str, str]:
+    def listen(self, interim: bool = True) -> OutputForSTTRecognition:
         """
         データがたまってからデータを出力するメソッド．
 
@@ -96,11 +105,17 @@ def listen(self, interim: bool = True) -> tuple[str, str]:
             if interim:  # 発話中でも出力するかどうか
                 matching_result = re.search(r"^interimresult:([^\n]+)\n", received)
                 if matching_result is not None:
-                    return STTRecognitionType.InterimResult, matching_result.group(1)
+                    return OutputForSTTRecognition(
+                        type=STTRecognitionType.InterimResult,
+                        result=matching_result.group(1),
+                    )
 
             matching_result = re.search(r"^result:([^\n]+)\n", received)
             if matching_result is not None:
-                return STTRecognitionType.Result, matching_result.group(1)
+                return OutputForSTTRecognition(
+                    type=STTRecognitionType.Result,
+                    result=matching_result.group(1),
+                )
 
 
 if __name__ == "__main__":
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
