fix eyes, break out interpolation data and extend it for models

Former-commit-id: d66253a

Author: you-win

entities/BasicModel.gd

@@ -45,7 +45,7 @@ func _ready() -> void:
 # Public functions                                                            #
 ###############################################################################
 
-func custom_update(_open_see_data: OpenSeeGD.OpenSeeData) -> void:
+func custom_update(_open_see_data: OpenSeeGD.OpenSeeData, _interpolation_data: InterpolationData) -> void:
 	push_error("Model custom update not implemented")
 
 func get_mapped_bones() -> Dictionary:

entities/basic-models/Duck.gd

@@ -43,7 +43,7 @@ func _unhandled_input(_event: InputEvent) -> void:
 # Public functions                                                            #
 ###############################################################################
 
-func custom_update(open_see_data: OpenSeeGD.OpenSeeData) -> void:
+func custom_update(open_see_data: OpenSeeGD.OpenSeeData, _interpolation_data: InterpolationData) -> void:
 	if not is_blinking:
 		if(open_see_data.left_eye_open < blink_threshold and open_see_data.right_eye_open < blink_threshold):
 			blink()

entities/vrm/VRMModel.gd

@@ -22,6 +22,9 @@ var mapped_meshes: Dictionary
 var blink_threshold: float = 0.3
 var eco_mode_is_blinking: bool = false
 
+# Gaze
+var gaze_strength: float = 0.5
+
 # Mouth
 var min_mouth_value: float = 0.0
 
@@ -34,6 +37,7 @@ func _ready() -> void:
 	rotation_damp = 0.01
 	additional_bone_damp = 0.6
 
+	# TODO this is gross
 	stored_offsets = get_parent().get_parent().stored_offsets
 
 	# Read vrm mappings
@@ -73,89 +77,46 @@ func _modify_blend_shape(mesh_instance: MeshInstance, blend_shape: String, value
 # Public functions                                                            #
 ###############################################################################
 
-func custom_update(data: OpenSeeGD.OpenSeeData) -> void:
+func set_expression(expression_name: String, expression_weight: float) -> void:
+	for mesh_name in vrm_mappings[expression_name].get_meshes():
+		for blend_name in vrm_mappings[expression_name].expression_data[mesh_name]:
+			_modify_blend_shape(mapped_meshes[mesh_name], blend_name, expression_weight)
+
+func custom_update(data: OpenSeeGD.OpenSeeData, interpolation_data: InterpolationData) -> void:
+	# NOTE: Eye mappings are intentionally reversed so that the model mirrors the data
 	# TODO i think this can be made more efficient
 	if not eco_mode:
 		# Left eye blinking
 		if data.left_eye_open >= blink_threshold:
-			for mesh_name in vrm_mappings.blink_l.get_meshes():
-				for blend_name in vrm_mappings.blink_l.expression_data[mesh_name]:
-					_modify_blend_shape(
-						mapped_meshes[mesh_name],
-						blend_name,
-						1.0 - data.left_eye_open
-					)
+			set_expression("blink_r", 1.0 - data.left_eye_open)
 		else:
-			for mesh_name in vrm_mappings.blink_l.get_meshes():
-				for blend_name in vrm_mappings.blink_l.expression_data[mesh_name]:
-					_modify_blend_shape(
-						mapped_meshes[mesh_name],
-						blend_name,
-						1.0
-					)
+			set_expression("blink_r", 1.0)
 
 		# Right eye blinking
 		if data.right_eye_open >= blink_threshold:
-			for mesh_name in vrm_mappings.blink_r.get_meshes():
-				for blend_name in vrm_mappings.blink_r.expression_data[mesh_name]:
-					_modify_blend_shape(
-						mapped_meshes[mesh_name],
-						blend_name,
-						1.0 - data.right_eye_open
-					)
+			set_expression("blink_l", 1.0 - data.left_eye_open)
 		else:
-			for mesh_name in vrm_mappings.blink_r.get_meshes():
-				for blend_name in vrm_mappings.blink_r.expression_data[mesh_name]:
-					_modify_blend_shape(
-						mapped_meshes[mesh_name],
-						blend_name,
-						1.0
-					)
+			set_expression("blink_l", 1.0)
 
 		# TODO eyes are a bit wonky
-		# TODO left eye is biased towards corners
-		# TODO right eye doesn't really look up or down
-		# NOTE: We don't want the y-rotation when tracking gaze otherwise your eye will rotate in its socket
 		# Left eye gaze
 		var left_eye_transform: Transform = Transform()
-		var left_eye_rotation: Vector3 = (stored_offsets.left_eye_gaze_offset - data.left_gaze.get_euler()) * 4
-		left_eye_transform = left_eye_transform.rotated(Vector3.UP, left_eye_rotation.x)
-		# if left_eye_rotation.z > 0:
-		# 	left_eye_transform = left_eye_transform.rotated(Vector3.LEFT, -left_eye_rotation.z)
-		# else:
-		# 	left_eye_transform = left_eye_transform.rotated(Vector3.LEFT, left_eye_rotation.z)
-		skeleton.set_bone_pose(left_eye_id, left_eye_transform)
+		var left_eye_rotation: Vector3 = interpolation_data.interpolate(InterpolationData.InterpolationDataType.LEFT_EYE_ROTATION, gaze_strength)
+		left_eye_transform = left_eye_transform.rotated(Vector3.RIGHT, -left_eye_rotation.x)
+		left_eye_transform = left_eye_transform.rotated(Vector3.UP, left_eye_rotation.y)
+		if Input.is_key_pressed(KEY_0): AppManager.log_message(str(left_eye_rotation))
+		skeleton.set_bone_pose(right_eye_id, left_eye_transform)
 
 		# Right eye gaze
 		var right_eye_transform: Transform = Transform()
-		var right_eye_rotation: Vector3 = (stored_offsets.right_eye_gaze_offset - data.right_gaze.get_euler()) * 4
-		right_eye_transform = right_eye_transform.rotated(Vector3.UP, right_eye_rotation.x)
-		# if right_eye_rotation.z > 0:
-		# 	right_eye_transform = right_eye_transform.rotated(Vector3.LEFT, right_eye_rotation.z)
-		# else:
-		# 	right_eye_transform = right_eye_transform.rotated(Vector3.LEFT, -right_eye_rotation.z)
-		skeleton.set_bone_pose(right_eye_id, right_eye_transform)
-
-		# TODO im not sure what these are tracking?
-		# Features eye left
-		# var left_eye_transform: Transform = Transform()
-		# print(data.features.eye_left)
-		# left_eye_transform.rotated(Vector3.UP, data.features.eye_left * 1.5)
-		# skeleton.set_bone_pose(left_eye_id, left_eye_transform)
-
-		# # Features eye right
-		# var right_eye_transform: Transform = Transform()
-		# right_eye_transform.rotated(Vector3.UP, data.features.eye_right * 1.5)
-		# skeleton.set_bone_pose(right_eye_id, right_eye_transform)
+		var right_eye_rotation: Vector3 = interpolation_data.interpolate(InterpolationData.InterpolationDataType.RIGHT_EYE_ROTATION, gaze_strength)
+		right_eye_transform = right_eye_transform.rotated(Vector3.RIGHT, -right_eye_rotation.x)
+		right_eye_transform = right_eye_transform.rotated(Vector3.UP, right_eye_rotation.y)
+		if Input.is_key_pressed(KEY_1): AppManager.log_message(str(right_eye_rotation))
+		skeleton.set_bone_pose(left_eye_id, right_eye_transform)
 
 		# Mouth tracking
-		for mesh_name in vrm_mappings.a.get_meshes():
-			for blend_name in vrm_mappings.a.expression_data[mesh_name]:
-				_modify_blend_shape(
-					mapped_meshes[mesh_name],
-					blend_name,
-					max(min_mouth_value, data.features.mouth_open)
-				)
+		set_expression("a", min(max(min_mouth_value, data.features.mouth_open * 2.0), 1.0))
 	else:
 		# TODO implement eco mode, should be more efficient than standard mode
 		# Eco-mode blinking

project.godot

@@ -75,6 +75,11 @@ _global_script_classes=[ {
 "path": "res://screens/gui/elements/InputLabel.gd"
 }, {
 "base": "Reference",
+"class": "InterpolationData",
+"language": "GDScript",
+"path": "res://utils/InterpolationData.gd"
+}, {
+"base": "Reference",
 "class": "JSONUtil",
 "language": "GDScript",
 "path": "res://utils/JSONUtil.gd"
@@ -123,6 +128,7 @@ _global_script_class_icons={
 "IKCube": "",
 "ImportVRM": "",
 "InputLabel": "",
+"InterpolationData": "",
 "JSONUtil": "",
 "MainScreen": "",
 "ModelDisplayScreen": "",

screens/ModelDisplayScreen.gd

@@ -59,32 +59,9 @@ var translation_adjustment: Vector3 = Vector3.ONE
 export var apply_rotation: bool = true
 var rotation_adjustment: Vector3 = Vector3.ONE
 export var interpolate_model: bool = true
-var interpolation_rate: float = 0.1
+var interpolation_rate: float = 0.1 setget _set_interpolation_rate
 var interpolation_data: InterpolationData = InterpolationData.new()
 
-class InterpolationData:
-	var last_updated: float
-	var last_translation: Vector3
-	var last_rotation: Vector3
-	var target_translation: Vector3
-	var target_rotation: Vector3
-
-	func _init() -> void:
-		last_updated = 0.0
-		last_translation = Vector3.ZERO
-		last_rotation = Vector3.ZERO
-		target_translation = Vector3.ZERO
-		target_rotation = Vector3.ZERO
-
-	func update_values(
-		p_last_updated: float,
-		p_target_translation: Vector3,
-		p_target_rotation: Vector3
-	) -> void:
-		last_updated = p_last_updated
-		target_translation = p_target_translation
-		target_rotation = p_target_rotation
-
 export var tracking_start_delay: float = 2.0
 
 ###
@@ -194,82 +171,43 @@ func _ready() -> void:
 	offset_timer.autostart = true
 	self.call_deferred("add_child", offset_timer)
 
-func _process(_delta: float) -> void:
+func _physics_process(_delta: float) -> void:
 	if not stored_offsets:
 		return
-	if not interpolate_model:
-		self.open_see_data = open_see.get_open_see_data(face_id)
 
-		if(not open_see_data or open_see_data.fit_3d_error > open_see.max_fit_3d_error):
-			return
-		
-		if open_see_data.time > updated:
-			updated = open_see_data.time
-		else:
-			return
-		
-		if apply_translation:
-			head_translation = (stored_offsets.translation_offset - open_see_data.translation) * model.translation_damp
-
-		if apply_rotation:
-			var corrected_euler: Vector3 = open_see_data.raw_euler
-			if corrected_euler.x < 0.0:
-				corrected_euler.x = 360 + corrected_euler.x
-			head_rotation = (stored_offsets.euler_offset - corrected_euler) * model.rotation_damp
+	self.open_see_data = open_see.get_open_see_data(face_id)
 
-		if model.has_custom_update:
-			model.custom_update(open_see_data)
-
-		model.move_head(
-			head_translation * translation_adjustment,
-			head_rotation * rotation_adjustment
-		)
-
-func _physics_process(_delta: float) -> void:
-	if not stored_offsets:
+	if(not open_see_data or open_see_data.fit_3d_error > open_see.max_fit_3d_error):
 		return
-	if interpolate_model:
-		self.open_see_data = open_see.get_open_see_data(face_id)
-	
-		if(not open_see_data or open_see_data.fit_3d_error > open_see.max_fit_3d_error):
-			return
-		
-		# Don't return early if we are interpolating
-		if open_see_data.time > updated:
-			updated = open_see_data.time
-			var corrected_euler: Vector3 = open_see_data.raw_euler
-			if corrected_euler.x < 0.0:
-				corrected_euler.x = 360 + corrected_euler.x
-			interpolation_data.update_values(
-				updated,
-				(stored_offsets.translation_offset - open_see_data.translation),
-				(stored_offsets.euler_offset - corrected_euler)
-			)
-
-		if apply_translation:
-			head_translation = lerp(
-				interpolation_data.last_translation,
-				interpolation_data.target_translation * model.translation_damp,
-				interpolation_rate
-			)
-			interpolation_data.last_translation = head_translation
-
-		if apply_rotation:
-			head_rotation = lerp(
-				interpolation_data.last_rotation,
-				interpolation_data.target_rotation * model.rotation_damp,
-				interpolation_rate
-			)
-			interpolation_data.last_rotation = head_rotation
-
-		if model.has_custom_update:
-			model.custom_update(open_see_data)
 
-		model.move_head(
-			head_translation * translation_adjustment,
-			head_rotation * rotation_adjustment
+	# Don't return early if we are interpolating
+	if open_see_data.time > updated:
+		updated = open_see_data.time
+		var corrected_euler: Vector3 = open_see_data.raw_euler
+		if corrected_euler.x < 0.0:
+			corrected_euler.x = 360 + corrected_euler.x
+		interpolation_data.update_values(
+			updated,
+			stored_offsets.translation_offset - open_see_data.translation,
+			stored_offsets.euler_offset - corrected_euler,
+			stored_offsets.left_eye_gaze_offset - open_see_data.left_gaze.get_euler(),
+			stored_offsets.right_eye_gaze_offset - open_see_data.right_gaze.get_euler()
 		)
 
+	if apply_translation:
+		head_translation = interpolation_data.interpolate(InterpolationData.InterpolationDataType.TRANSLATION, model.translation_damp)
+
+	if apply_rotation:
+		head_rotation = interpolation_data.interpolate(InterpolationData.InterpolationDataType.ROTATION, model.rotation_damp)
+
+	if model.has_custom_update:
+		model.custom_update(open_see_data, interpolation_data)
+
+	model.move_head(
+		head_translation * translation_adjustment,
+		head_rotation * rotation_adjustment
+	)
+
 func _unhandled_input(event: InputEvent) -> void:
 	if event.is_action_pressed("ui_accept"):
 		_save_offsets()
@@ -395,6 +333,10 @@ static func _find_bone_chain(skeleton: Skeleton, root_bone: int, tip_bone: int)
 
 	return result
 
+func _set_interpolation_rate(value: float) -> void:
+	interpolation_rate = value
+	interpolation_data.rate = value
+
 ###############################################################################
 # Public functions                                                            #
 ###############################################################################

screens/gui/ModelViewRight.gd

@@ -74,6 +74,13 @@ func _generate_properties(p_initial_properties: Dictionary = {}) -> void:
 			data_source.interpolation_rate, TYPE_REAL)
 
 func _apply_properties() -> void:
+	# Interpolation data must be handled in a specific order since we don't check for interpolation anymore
+	main_screen.model_display_screen.interpolate_model = v_box_container.get_node("interpolate_model").get_value()
+	if not main_screen.model_display_screen.interpolate_model:
+		main_screen.model_display_screen.interpolation_rate = 1.0
+	else:
+		main_screen.model_display_screen.interpolation_rate = v_box_container.get_node("interpolation_rate").get_value()
+
 	for c in v_box_container.get_children():
 		# Null checks and value checks
 		if c.get("line_edit"):
@@ -82,6 +89,7 @@ func _apply_properties() -> void:
 			if c.line_edit_type == TYPE_REAL:
 				if not c.line_edit.text.is_valid_float():
 					continue
+
 		match c.name:
 			"translation_damp":
 				current_model.translation_damp = c.get_value()
@@ -93,10 +101,6 @@ func _apply_properties() -> void:
 				main_screen.model_display_screen.apply_translation = c.get_value()
 			"apply_rotation":
 				main_screen.model_display_screen.apply_rotation = c.get_value()
-			"interpolate_model":
-				main_screen.model_display_screen.interpolate_model = c.get_value()
-			"interpolation_rate":
-				main_screen.model_display_screen.interpolation_rate = c.get_value()
 
 func _setup() -> void:
 	current_model = main_screen.model_display_screen.model

screens/gui/ModelViewRight.tscn

@@ -9,6 +9,7 @@ script = ExtResource( 2 )
 [node name="LoadModelButton" type="Button" parent="Control/MarginContainer/VBoxContainer" index="0"]
 margin_right = 1580.0
 margin_bottom = 79.0
+focus_mode = 0
 size_flags_vertical = 3
 size_flags_stretch_ratio = 0.1
 text = "Load Model"