using UnityEngine; using System.Collections; using System.Collections.Generic; using System.Xml; using System.Xml.Serialization; using System.IO; using UnityEditor; using System.Diagnostics; using System; using System.Globalization; namespace PuppetFace { [RequireComponent(typeof(AudioSource))] [AddComponentMenu("Puppet Face/LipSync")] public class LipSync : MonoBehaviour { public int LipSyncIndex = 0; public TextAsset[] LipSyncFiles; public AudioClip[] AudioClips; public int[] BlendShapeIndexes = new int[] { 99999, 99999, 99999, 99999, 99999, 99999, 99999, 99999, 99999 }; public Transform[] FaceBones; public float Strength = 1f; public bool PlayOnAwake = false; public bool PlayAll = false; public bool Repeat = false; private bool isPlaying = false; public bool Initialized = false; public bool NewAudioAdded = false; public SkinnedMeshRenderer Skin; public LipSync[] LipSyncs = new LipSync[0]; private AudioSource _audioSource; public List FaceShapes = new List(); public List TransformStates = new List(); [HideInInspector] public ListTransformStates _currentTransforms = new ListTransformStates(); public float TimeVal = 0; public float _internalTimeVal = 0f; public bool IsPlaying { get { return isPlaying; } set { isPlaying = value; } } // Use this for initialization void Awake() { #if UNITY_EDITOR if (!Initialized) InitializeFromFile(); else #endif _audioSource = GetComponent(); if (PlayOnAwake) Play(LipSyncIndex); } private void LateUpdate() { if (IsPlaying) { EvaluateLipSync(LipSyncIndex); if (TimeVal >= _audioSource.clip.length) { if (LipSyncIndex + 1 < AudioClips.Length) { if (PlayAll) { LipSyncIndex++; Play(LipSyncIndex, 0f); } else if (Repeat) { Play(LipSyncIndex); } else { Stop(); } } else if (Repeat) { LipSyncIndex = 0; Play(LipSyncIndex); } else { Stop(); } } } if (_internalTimeVal != TimeVal) { _internalTimeVal = TimeVal; SetPhoneme(LipSyncIndex, TimeVal); } } public void Play(int index, float startTime = 0f) { for (int i = 0; i < LipSyncs.Length; i++) { LipSyncs[i].Play(index, startTime); } if (_audioSource!=null && AudioClips != null && index < AudioClips.Length && index>=0) { _audioSource.clip = AudioClips[index]; _audioSource.Play(); if (_audioSource.time >= startTime && startTime >=0f) { _audioSource.time = startTime; TimeVal = startTime; } IsPlaying = true; } } public void EvaluateLipSync(int index) { SetPhoneme(index, TimeVal); TimeVal += Time.deltaTime; } public void Stop() { for (int i = 0; i < LipSyncs.Length; i++) { LipSyncs[i].Stop(); } IsPlaying = false; TimeVal = 0f; _audioSource.Stop(); } public void InitializeFromData(ListFaceShapes FaceShapesData, int index) { for (int i = 0; i < LipSyncs.Length; i++) { LipSyncs[i].LipSyncFiles = LipSyncFiles; LipSyncs[i].AudioClips = AudioClips; LipSyncs[i].InitializeFromData(FaceShapesData, index); } if (FaceShapes.Count > index) { FaceShapes[index] = FaceShapesData; #if UNITY_EDITOR EditorUtility.SetDirty(this); #endif } } #if UNITY_EDITOR public void InitializeFromFile() { for (int i = 0; i < LipSyncs.Length; i++) { LipSyncs[i].LipSyncFiles = LipSyncFiles; LipSyncs[i].AudioClips = AudioClips; LipSyncs[i].InitializeFromFile(); } FaceShapes.Clear(); for (int t = 0; t < LipSyncFiles.Length; t++) { string path = AssetDatabase.GetAssetPath(LipSyncFiles[t]);// "Assets/LipSync/output.xml"; //UnityEngine.Debug.Log("path before " + path); path = Application.dataPath + path.Substring(6); //UnityEngine.Debug.Log("path after " + path); if (new FileInfo(path).Length == 0) { continue; } var stream = new StreamReader(path); XmlDocument xmlDoc = new XmlDocument(); // xmlDoc is the new xml document. xmlDoc.LoadXml(stream.ReadToEnd()); XmlNodeList mouthCueList = xmlDoc.GetElementsByTagName("mouthCues"); ListFaceShapes newFaceShape = new ListFaceShapes(); foreach (XmlNode mouthCue in mouthCueList) { XmlNodeList mouthCuesList = xmlDoc.GetElementsByTagName("mouthCue"); foreach (XmlNode mouthCues in mouthCuesList) { float startVal = float.Parse(mouthCues.Attributes["start"].Value, CultureInfo.InvariantCulture); float endVal = float.Parse(mouthCues.Attributes["end"].Value, CultureInfo.InvariantCulture); string mouthShape = mouthCues.ChildNodes[0].Value; float bsID = -1; if (mouthCues.Attributes["blendShapeID"] != null) { bsID = float.Parse(mouthCues.Attributes["blendShapeID"].Value, CultureInfo.InvariantCulture); } newFaceShape.Add(new FaceShape(startVal, endVal, mouthShape, 1f, (int)bsID)); } } FaceShapes.Add(newFaceShape); } _audioSource = GetComponent(); _audioSource.playOnAwake = false; Initialized = true; } #endif private int GetIndexFromMouthShape(string mouthShape) { switch (mouthShape) { case "A": return BlendShapeIndexes[0]; case "B": return BlendShapeIndexes[1]; case "C": return BlendShapeIndexes[2]; case "D": return BlendShapeIndexes[3]; case "E": return BlendShapeIndexes[4]; case "F": return BlendShapeIndexes[5]; case "G": return BlendShapeIndexes[6]; case "H": return BlendShapeIndexes[7]; case "X": return BlendShapeIndexes[8]; } return 0; } private int GetOrderFromMouthShape(string mouthShape ) { switch (mouthShape) { case "A": return 0; case "B": return 1; case "C": return 2; case "D": return 3; case "E": return 4; case "F": return 5; case "G": return 6; case "H": return 7; case "X": return 8; } return 0; } #if UNITY_EDITOR public string[] GetPhonemes(int index) { List phonemeList = new List(); string path = AssetDatabase.GetAssetPath(LipSyncFiles[index]);// "Assets/LipSync/output.xml"; StreamReader stream = new StreamReader(path); if (new FileInfo(path).Length == 0) { return phonemeList.ToArray(); } XmlDocument xmlDoc = new XmlDocument(); // xmlDoc is the new xml document. xmlDoc.LoadXml(stream.ReadToEnd()); XmlNodeList mouthCueList = xmlDoc.GetElementsByTagName("mouthCues"); foreach (XmlNode mouthCue in mouthCueList) { XmlNodeList mouthCuesList = xmlDoc.GetElementsByTagName("mouthCue"); foreach (XmlNode mouthCues in mouthCuesList) { float startVal = float.Parse(mouthCues.Attributes["start"].Value, CultureInfo.InvariantCulture); float endVal = float.Parse(mouthCues.Attributes["end"].Value, CultureInfo.InvariantCulture); float strength = 1f; if (mouthCues.Attributes["strength"] !=null) strength = float.Parse(mouthCues.Attributes["strength"].Value, CultureInfo.InvariantCulture); string mouthShape = mouthCues.ChildNodes[0].Value; float bsID = -1; if (mouthCues.Attributes["blendShapeID"] != null) { bsID = float.Parse(mouthCues.Attributes["blendShapeID"].Value, CultureInfo.InvariantCulture); } phonemeList.Add(startVal + ";" + endVal + ";" + mouthShape + ";" + strength + ";" + bsID); } } return phonemeList.ToArray(); } #endif public void ResetPhonemeBones() { if (TransformStates.Count > 0) { for (int f = 0; f < FaceBones.Length; f++) { _currentTransforms.transformStates[f].position = TransformStates[8].transformStates[f].position; _currentTransforms.transformStates[f].rotation = TransformStates[8].transformStates[f].rotation; _currentTransforms.transformStates[f].scale = TransformStates[8].transformStates[f].scale; } } } public void SetPhonemeOnly(int index, float amount) { if (!float.IsNaN(amount)) { int blendShapeIndex = BlendShapeIndexes[index]; Skin.SetBlendShapeWeight(blendShapeIndex, amount); if (TransformStates[index].transformStates.Count > 0) { for (int i = 0; i < FaceBones.Length; i++) { _currentTransforms.transformStates[i].position = Vector3.Lerp(_currentTransforms.transformStates[i].position, TransformStates[index].transformStates[i].position, amount / 100f); _currentTransforms.transformStates[i].rotation = Quaternion.Lerp(_currentTransforms.transformStates[i].rotation, TransformStates[index].transformStates[i].rotation, amount / 100f); _currentTransforms.transformStates[i].scale = Vector3.Lerp(_currentTransforms.transformStates[i].scale, TransformStates[index].transformStates[i].scale, amount / 100f); FaceBones[i].localPosition = _currentTransforms.transformStates[i].position; FaceBones[i].localRotation = _currentTransforms.transformStates[i].rotation; FaceBones[i].localScale = _currentTransforms.transformStates[i].scale; } } } } public void SetPhoneme(int index, float timeVal) { for (int i = 0; i < LipSyncs.Length; i++) { LipSyncs[i].SetPhoneme(index, timeVal); } if (FaceShapes.Count > 0) { foreach (FaceShape f in FaceShapes[index].faceShapes) { int blendShapeIndex = GetIndexFromMouthShape(f.shapeName); if(Skin.sharedMesh.blendShapeCount > blendShapeIndex) Skin.SetBlendShapeWeight(blendShapeIndex, 0); if(f.blendShapeIndex>=0) Skin.SetBlendShapeWeight(f.blendShapeIndex, 0); } if (TransformStates.Count > 0) { for (int f = 0; f < FaceBones.Length; f++) { _currentTransforms.transformStates[f].position = TransformStates[8].transformStates[f].position; _currentTransforms.transformStates[f].rotation = TransformStates[8].transformStates[f].rotation; _currentTransforms.transformStates[f].scale = TransformStates[8].transformStates[f].scale; } } foreach (FaceShape f in FaceShapes[index].faceShapes) { float shapeDuration = (f.end - f.start); float end = (f.end - f.start) * (1.35f - 0.35f * Mathf.Clamp01(shapeDuration)) + f.start; float start = (f.start - f.end) * (1.35f - 0.35f * Mathf.Clamp01(shapeDuration)) + f.end; if (end >= timeVal && timeVal >= start) { float t = (timeVal - start) / (end - start); t = Mathf.Sin(t * Mathf.PI) * Strength * f.strength * 100f; int blendShapeIndex = GetIndexFromMouthShape(f.shapeName); if (f.blendShapeIndex >= 0) { Skin.SetBlendShapeWeight(f.blendShapeIndex, t); } else if (Skin.sharedMesh.blendShapeCount > blendShapeIndex) Skin.SetBlendShapeWeight(blendShapeIndex, t); if (TransformStates.Count > 0) { int order = GetOrderFromMouthShape(f.shapeName); if (TransformStates[order].transformStates.Count > 0) { for (int i = 0; i < FaceBones.Length; i++) { _currentTransforms.transformStates[i].position = Vector3.Lerp(_currentTransforms.transformStates[i].position, TransformStates[order].transformStates[i].position, t/100f); _currentTransforms.transformStates[i].rotation = Quaternion.Lerp(_currentTransforms.transformStates[i].rotation, TransformStates[order].transformStates[i].rotation, t / 100f); _currentTransforms.transformStates[i].scale = Vector3.Lerp(_currentTransforms.transformStates[i].scale, TransformStates[order].transformStates[i].scale, t / 100f); FaceBones[i].localPosition = _currentTransforms.transformStates[i].position; FaceBones[i].localRotation = _currentTransforms.transformStates[i].rotation; FaceBones[i].localScale = _currentTransforms.transformStates[i].scale; } } } } } } else Initialized = false; } public void SetIndex(int index) { LipSyncIndex = Mathf.Clamp(index, 0, LipSyncFiles.Length-1); for (int i = 0; i < LipSyncs.Length; i++) { LipSyncs[i].SetIndex(index); } } public int getBlendShapeName(string val) { Mesh m = Skin.sharedMesh; for (int i = 0; i < m.blendShapeCount; i++) { string s = m.GetBlendShapeName(i); if (s == val) return i; } return -1; } #if UNITY_EDITOR static public string ConvertAudioToPhoneme(AudioClip audioclip) { string audioFilePath = ""; string fullPathAudio; if (audioclip == null) fullPathAudio = "Assets"; else fullPathAudio = AssetDatabase.GetAssetPath(audioclip); string fullPathOutput = fullPathAudio.Replace(".wav", ".xml"); audioFilePath = fullPathOutput; string puppetFacePath = RecursivelyFindFolderPath(); string fullPath = puppetFacePath + "/Tools/LipSync/rhubarb/rhubarb.exe "; System.IO.File.WriteAllText(audioFilePath, ""); ProcessStartInfo startInfo = new ProcessStartInfo(fullPath); startInfo.WindowStyle = ProcessWindowStyle.Normal; startInfo.Arguments = "-o \"" + fullPathOutput + "\" -f xml -r phonetic \"" + fullPathAudio + "\""; Process.Start(startInfo); return audioFilePath; } private static string RecursivelyFindFolderPath() { DirectoryInfo directoryInfo = new DirectoryInfo(Application.dataPath); DirectoryInfo[] dirInfos = directoryInfo.GetDirectories("*", SearchOption.AllDirectories); foreach (DirectoryInfo d in dirInfos) { if (d.Name == "PuppetFace" && d.Parent.Name != "Gizmos") return d.FullName; } return ""; } #endif } [Serializable] public class FaceShape { public float start; public float end; public string shapeName; public float strength; public int blendShapeIndex; public FaceShape(float s, float e, string shape, float strengthVal, int blendShapeId = -1) { start = s; end = e; shapeName = shape; strength = strengthVal; blendShapeIndex = blendShapeId; } } [Serializable] public class ListFaceShapes { public List faceShapes = new List(); public FaceShape this[int key] { get { return faceShapes[key]; } set { faceShapes[key] = value; } } public void Add(FaceShape fs) { faceShapes.Add(fs); } } [Serializable] public class TransformState { public Vector3 position = Vector3.zero; public Quaternion rotation = Quaternion.identity; public Vector3 scale = Vector3.one; public TransformState(Vector3 pos, Quaternion rot, Vector3 scal) { position = pos; rotation = rot; scale = scal; } } [Serializable] public class ListTransformStates { public List transformStates = new List(); public TransformState this[int key] { get { return transformStates[key]; } set { transformStates[key] = value; } } public void Add(TransformState fs) { transformStates.Add(fs); } } }