Skip to content

Instantly share code, notes, and snippets.

@anselm

anselm/blender_python_facial_expressions_cc4_or_rpm_to_vrm.py

Created July 25, 2024 01:01
Show Gist options
  • Save anselm/199ee0290d6f0cb97e147b3dd98b351d to your computer and use it in GitHub Desktop.
Save anselm/199ee0290d6f0cb97e147b3dd98b351d to your computer and use it in GitHub Desktop.
Download ZIP
A blender python script to set vrm 1 facial expressions from a cc4 or rpm rigged avatar such as from character creator 4 or ready player me
Raw
blender_python_facial_expressions_cc4_or_rpm_to_vrm.py
## Stuff extra shape keys or morph targets or whatever we call them into vrms in blender - this file gets run from inside blender scripting
# Tip: It"s very easy to extend blender since any button click action you do is logged as the equivalent python command to the python console.
print("Copying expressions from avatar to VRM 1.0 - make sure you have the blender vrm plugin installed and an avatar loaded from RPM or CC4")
import bpy
import pprint
import time
#
# note that the remapping below has the same keys as this...
#
expressions_arkit_oculus = [
"viseme_sil", "viseme_PP", "viseme_FF", "viseme_TH", "viseme_DD", "viseme_kk", "viseme_CH", "viseme_SS", "viseme_nn",
"viseme_RR", "viseme_aa", "viseme_E", "viseme_I", "viseme_O", "viseme_U",
"eyeBlinkLeft", "eyeLookDownLeft", "eyeLookInLeft", "eyeLookOutLeft", "eyeLookUpLeft", "eyeSquintLeft", "eyeWideLeft",
"eyeBlinkRight", "eyeLookDownRight", "eyeLookInRight", "eyeLookOutRight", "eyeLookUpRight", "eyeSquintRight", "eyeWideRight",
"eyesClosed", "eyesLookUp", "eyesLookDown",
"jawForward", "jawLeft", "jawRight", "jawOpen",
"browDownLeft", "browDownRight", "browInnerUp", "browOuterUpLeft", "browOuterUpRight",
"cheekPuff", "cheekSquintLeft", "cheekSquintRight",
"noseSneerLeft", "noseSneerRight", "tongueOut",
"mouthClose", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthSmileLeft", "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight",
"mouthDimpleLeft", "mouthDimpleRight", "mouthStretchLeft", "mouthStretchRight",
"mouthRollLower", "mouthRollUpper", "mouthShrugLower", "mouthShrugUpper",
"mouthPressLeft", "mouthPressRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthUpperUpLeft", "mouthUpperUpRight",
"mouthOpen", "mouthSmile",
]
#
# for cc4 and some other datasets we will use this to set vrm 1.0 fields
# we want to create the left side things from the right side hints
#
# see also https://github.com/pkhungurn/talking-head-anime-3-demo/blob/main/tha3/mocap/ifacialmocap_constants.py
#
expressions_remap_cc4_to_arkit = {
"viseme_sil" : None, #"basis",
"viseme_PP" : "B_M_P",
"viseme_FF" : "F_V",
"viseme_TH" : "TH",
"viseme_DD" : "T_L_D_N",
"viseme_kk" : "K_G_H_NG",
"viseme_CH" : "Ch_J",
"viseme_SS" : "S_Z",
"viseme_nn" : "T_L_D_N",
"viseme_RR" : "R",
"viseme_aa" : "Ah",
"viseme_E" : "EE",
"viseme_I" : "IH",
"viseme_O" : "Oh",
"viseme_U" : "W_OO",
# None : "Er",
# None : "AE",
# None : "Eyes_Blink",
"eyeBlinkLeft" : "Eye_Blink_L",
"eyeLookDownLeft" : None,
"eyeLookInLeft" : None,
"eyeLookOutLeft" : None,
"eyeLookUpLeft" : None,
"eyeSquintLeft" : "Eye_Squint_L",
"eyeWideLeft" : "Eye_Wide_L",
"eyeBlinkRight" : "Eye_Blink_R",
"eyeLookDownRight" : None,
"eyeLookInRight" : None,
"eyeLookOutRight" : None,
"eyeLookUpRight" : None,
"eyeSquintRight" : "Eye_Squint_R",
"eyeWideRight" : "Eye_Wide_R",
"eyesClosed" : [ "Eye_Blink_L", "Eye_Blink_R" ],
"eyesLookUp" : None,
"eyesLookDown" : None,
"jawForward" : None,
"jawLeft" : None,
"jawRight" : None,
"jawOpen" : None,
"browDownLeft" : "Brow_Drop_Left",
"browDownRight" : "Brow_Drop_Right",
"browInnerUp" : [ "Brow_Raise_Inner_Left", "Brow_Raise_Inner_Right" ],
"browOuterUpLeft" : "Brow_Raise_Outer_Left",
"browOuterUpRight" : "Brow_Raise_Outer_Right",
# None : "Brow_Raise_Left",
# None : "Brow_Raise_Right",
"cheekPuff" : [ "Cheek_Blow_L", "Cheek_Blow_R" ],
"cheekSquintLeft" : "Cheek_Raise_L",
"cheekSquintRight" : "Cheek_Raise_R",
# None : "Cheeks_Suck",
"noseSneerLeft" : "Nose_Flank_Raise_L",
"noseSneerRight" : "Nose_Flank_Raise_R",
# None" : "Nose_Flanks_Raise",
# None" : "Nose_Nostrils_Flare",
# None" : "Nose_Scrunch",
"tongueOut" : None,
"mouthOpen": "Mouth_Open",
"mouthSmile": "Mouth_Smile",
"mouthClose" : None,
"mouthFunnel" : "Mouth_Pucker_Open",
"mouthPucker" : "Mouth_Pucker",
"mouthLeft" : "Mouth_L",
"mouthRight" : "Mouth_R",
"mouthSmileLeft" : "Mouth_Smile_L",
"mouthSmileRight" : "Mouth_Smile_R",
"mouthFrownLeft" : "Mouth_Frown_L",
"mouthFrownRight" : "Mouth_Frown_R",
"mouthDimpleLeft" : "Mouth_Dimple_L",
"mouthDimpleRight" : "Mouth_Dimple_R",
"mouthStretchLeft" : "Mouth_L",
"mouthStretchRight" : "Mouth_R",
"mouthRollLower" : "Mouth_Bottom_Lip_Under",
"mouthRollUpper" : "Mouth_Top_Lip_Under",
"mouthShrugLower" : None,
"mouthShrugUpper" : None,
"mouthPressLeft" : None,
"mouthPressRight" : None,
"mouthLowerDownLeft": "Mouth_Frown_L",
"mouthLowerDownRight": "Mouth_Frown_R",
"mouthUpperUpLeft": "Mouth_Smile_L",
"mouthUpperUpRight": "Mouth_Smile_R",
# None : Mouth_Frown
# None : Mouth_Blow
# None : Mouth_Widen
# None : Mouth_Plosive
# None : Mouth_Lips_Tight
# None : Mouth_Lips_Tuck
# None : Mouth_Lips_Open
# None : Mouth_Lips_Part
# None : Mouth_Bottom_Lip_Down
# None : Mouth_Top_Lip_Up
# None : Mouth_Snarl_Upper_L
# None : Mouth_Snarl_Upper_R
# None : Mouth_Snarl_Lower_L
# None : Mouth_Snarl_Lower_R
# None : Mouth_Bottom_Lip_Bite
# None : Mouth_Down
# None : Mouth_Up
# None : Mouth_Lips_Jaw_Adjust
# None : Mouth_Bottom_Lip_Trans
# None : Mouth_Skewer
}
#
# find parts
#
armature = bpy.data.objects["Armature"]
iscc4 = bpy.data.objects.get("CC_Base_Body")
isrpm = bpy.data.objects.get("Wolf3D_Head")
parts = []
part_names = [
# "CC_Base_Eye", ... appear to be no shape keys here
"CC_Base_Teeth",
"CC_Base_Tongue",
"CC_Base_Body",
"Wolf3D_Head",
]
for name in part_names:
found = bpy.data.objects.get(name)
if found:
print("Building a parts list to write expressions to",name)
parts.append(found)
if armature is None:
raise ValueError("Avatar must have an armature")
if iscc4 is None and isrpm is None:
raise ValueError("Avatar is neither CC4 nor RPM")
#
# Sanitize
#
def sanitize():
for index, expression in enumerate(expressions_arkit_oculus):
bpy.ops.vrm.remove_vrm1_expressions_custom_expression(armature_name="Armature", custom_expression_name=expression)
#
# For RPM we expect to see the expressions_arkit_oculus morph targets
#
def expressions_rpm():
if isrpm is not None:
for index, expression in enumerate(expressions_arkit_oculus):
bpy.ops.vrm.remove_vrm1_expressions_custom_expression(armature_name="Armature", custom_expression_name=expression)
bpy.ops.vrm.add_vrm1_expressions_custom_expression(armature_name="Armature", custom_expression_name=expression)
bpy.ops.vrm.add_vrm1_expression_morph_target_bind(armature_name="Armature", expression_name=expression)
custom = bpy.data.armatures["Armature"].vrm_addon_extension.vrm1.expressions.custom[index]
custom = custom.custom_name=expression
for index2, part in enumerate(parts):
print("binding",expression,index2)
custom.morph_target_binds[index2].node.bpy_object = part
custom.morph_target_binds[index2].index = expression
#
# For CC4 we start with the arkit_oculus name and look up the CC4 name
#
def expressions_cc4():
if iscc4 is not None:
for (expression,shapes) in expressions_remap_cc4_to_arkit.items():
# Add a custom expression
bpy.ops.vrm.add_vrm1_expressions_custom_expression(armature_name="Armature", custom_expression_name=expression)
# No mapping from rpm to cc4 - create but leave it unbound
if shapes is None:
continue
# poke it around a bit to exercise it because there seems to be a race condition - try remove this @todo
# bpy.ops.vrm.move_up_vrm1_expressions_custom_expression(armature_name="Armature", custom_expression_name=expression)
# bpy.ops.vrm.update_vrm1_expression_ui_list_elements()
# Get the index to the fresh custom expression
index = len( armature.data.vrm_addon_extension.vrm1.expressions.custom ) - 1
print("currently thinking about expression",expression,index)
# Get a handle on the fresh custom expression
custom = bpy.data.armatures["Armature"].vrm_addon_extension.vrm1.expressions.custom[index]
# Go ahead and reinforce the name - it is unclear why here are two separate places the name is stored
custom.custom_name = expression
# For targets, for cc4, there may be an array of shape keys to adjust
if isinstance(shapes, str):
shapes = [shapes]
# wire up each cc4 shape key to our rpm named expressions
for part in parts:
# if part has no shape keys then ignore
if not part.data.shape_keys:
print("part has no shape keys at all")
continue
# count how many of the expressions exist on the cc4 model source
count = 0
for shape in shapes:
if part.data.shape_keys.key_blocks.get(shape):
count = count + 1
# if there are no expressions on the source then skip this
if not count:
print("part does not have shapes",shapes[0])
continue
for shape in shapes:
print("... currently writing",expression,index,shape)
# for this custom expression, create a fresh morph target
bpy.ops.vrm.add_vrm1_expression_morph_target_bind(armature_name="Armature", expression_name=expression)
# add morph target and shape
index = len(custom.morph_target_binds) - 1
custom.morph_target_binds[index].node.bpy_object = part
custom.morph_target_binds[index].index = shape
sanitize()
expressions_rpm()
expressions_cc4()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment