|
|
@@ -4,13 +4,10 @@ import android.app.Activity
|
|
|
import android.content.Context
|
|
|
import android.text.TextUtils
|
|
|
import cn.i2edu.dubbing_lib.bean.SimpleWord
|
|
|
-import cn.i2edu.dubbing_lib.util.ChivoxAudioEvaluatorUtil
|
|
|
-import cn.i2edu.dubbing_lib.util.ChivoxEvaluatorCallBack
|
|
|
import cn.i2edu.speech_plugin.audioUtils.audioEvaluator.resultParse.ReadSentenceResult
|
|
|
import cn.i2edu.speech_plugin.audioUtils.audioEvaluator.resultParse.XmlResultParser
|
|
|
import cn.i2edu.speech_plugin.model.chivox.ChivoxResult
|
|
|
-import cn.i2edu.speech_plugin.util.IFlyAudioEvaluatorUtil
|
|
|
-import cn.i2edu.speech_plugin.util.IFlyEvaluatorCallBack
|
|
|
+import cn.i2edu.speech_plugin.util.*
|
|
|
import com.alibaba.fastjson.JSON
|
|
|
import com.googlecode.mp4parser.authoring.Movie
|
|
|
import com.googlecode.mp4parser.authoring.Track
|
|
|
@@ -21,16 +18,15 @@ import com.iflytek.cloud.EvaluatorResult
|
|
|
import com.iflytek.cloud.SpeechConstant
|
|
|
import com.iflytek.cloud.SpeechError
|
|
|
import com.iflytek.cloud.SpeechUtility
|
|
|
+import com.tencent.taisdk.TAIOralEvaluationRet
|
|
|
import io.flutter.plugin.common.MethodCall
|
|
|
import io.flutter.plugin.common.MethodChannel
|
|
|
import io.flutter.plugin.common.MethodChannel.MethodCallHandler
|
|
|
import io.flutter.plugin.common.MethodChannel.Result
|
|
|
import io.flutter.plugin.common.PluginRegistry.Registrar
|
|
|
import java.io.File
|
|
|
-import java.io.FileInputStream
|
|
|
import java.io.RandomAccessFile
|
|
|
import java.util.*
|
|
|
-import com.chivox.AIEngine.*
|
|
|
|
|
|
class SpeechPlugin : MethodCallHandler {
|
|
|
|
|
|
@@ -115,28 +111,49 @@ class SpeechPlugin : MethodCallHandler {
|
|
|
private fun evaluatorRecord(pathEvaluatorDecode: String, index: Int, recordPath: String, en: String, isVideo: Boolean, sdk: SpeechSdk) {
|
|
|
val dir = File(pathEvaluatorDecode)
|
|
|
if (!dir.exists()) dir.mkdirs()
|
|
|
- val decodePath = pathEvaluatorDecode + UUID.randomUUID().toString() + ".pcm"
|
|
|
+ var decodePath = pathEvaluatorDecode + UUID.randomUUID().toString() + ".pcm"
|
|
|
|
|
|
if (sdk == SpeechSdk.Chivox) {
|
|
|
+ // 驰声评测
|
|
|
ChivoxAudioEvaluatorUtil.getInstance(context)
|
|
|
- .setEvaluatorCallBack(object: ChivoxEvaluatorCallBack {
|
|
|
+ .setEvaluatorCallBack(object : ChivoxEvaluatorCallBack {
|
|
|
override fun onResult(result: ChivoxResult) {
|
|
|
- (context as Activity).runOnUiThread {
|
|
|
- val formatWords = arrayListOf<SimpleWord>()
|
|
|
- for (detail in result.result.details) {
|
|
|
- formatWords.add(SimpleWord(detail.char, detail.score / 20.0f))
|
|
|
- }
|
|
|
- channel.invokeMethod("evaluatorResult",
|
|
|
- mapOf("index" to index, "score" to result.result.overall / 20.0f, "integrity_score" to result.result.integrity / 20.0f,
|
|
|
- "accuracy_score" to result.result.accuracy / 20.0f, "fluency_score" to result.result.fluency.overall / 20.0f,
|
|
|
- "words" to JSON.toJSONString(formatWords)))
|
|
|
+ val formatWords = arrayListOf<SimpleWord>()
|
|
|
+ for (detail in result.result.details) {
|
|
|
+ formatWords.add(SimpleWord(detail.char, detail.score / 20.0f))
|
|
|
}
|
|
|
+ invokeEvaluatorResult(index, score = result.result.overall / 20.0f, integrityScore = result.result.integrity / 20.0f,
|
|
|
+ accuracyScore = result.result.accuracy / 20.0f, fluencyScore = result.result.fluency.overall / 20.0f,
|
|
|
+ words = JSON.toJSONString(formatWords))
|
|
|
}
|
|
|
|
|
|
override fun onError(message: String) {
|
|
|
- (context as Activity).runOnUiThread {
|
|
|
- channel.invokeMethod("evaluatorResult", mapOf("index" to index, "score" to null, "integrity_score" to null, "accuracy_score" to null, "fluency_score" to null))
|
|
|
+ invokeEvaluatorResult(index)
|
|
|
+ }
|
|
|
+ })
|
|
|
+ ?.startEvaluator(en, recordPath, decodePath)
|
|
|
+ } else if (sdk == SpeechSdk.SmartOral) {
|
|
|
+ // 腾讯智聆
|
|
|
+ decodePath = pathEvaluatorDecode + UUID.randomUUID().toString() + ".mp3"
|
|
|
+ SmartOralAudioEvaluatorUtil.getInstance(context)
|
|
|
+ .setEvaluatorCallBack(object : SmartOralEvaluatorCallBack {
|
|
|
+ override fun onResult(result: TAIOralEvaluationRet?) {
|
|
|
+ val formatWords = arrayListOf<SimpleWord>()
|
|
|
+ for (detail in result!!.words) {
|
|
|
+ formatWords.add(SimpleWord(detail.word, if (detail.pronAccuracy == -1.0 ) 0f else (detail.pronAccuracy * 5).toFloat()))
|
|
|
}
|
|
|
+ val pronAccuracy = if (result.pronAccuracy == -1.0 ) 0f else (result.pronAccuracy * 5).toFloat()
|
|
|
+ val pronFluency = result.pronFluency
|
|
|
+ val pronCompletion = result.pronCompletion
|
|
|
+
|
|
|
+ val totalScore = (pronAccuracy + pronFluency + pronCompletion) / 3
|
|
|
+ invokeEvaluatorResult(index, score = totalScore.toFloat(), integrityScore = pronCompletion.toFloat(),
|
|
|
+ accuracyScore = pronAccuracy.toFloat(), fluencyScore = pronFluency.toFloat(),
|
|
|
+ words = JSON.toJSONString(formatWords))
|
|
|
+ }
|
|
|
+
|
|
|
+ override fun onError(message: String) {
|
|
|
+ invokeEvaluatorResult(index)
|
|
|
}
|
|
|
})
|
|
|
?.startEvaluator(en, recordPath, decodePath)
|
|
|
@@ -144,51 +161,56 @@ class SpeechPlugin : MethodCallHandler {
|
|
|
IFlyAudioEvaluatorUtil.getInstance(context = context)
|
|
|
.setEvaluatorCallBack(object : IFlyEvaluatorCallBack {
|
|
|
override fun onResult(result: EvaluatorResult) {
|
|
|
- (context as Activity).runOnUiThread {
|
|
|
- if (isVideo) {
|
|
|
- try {
|
|
|
- File(recordPath).delete()
|
|
|
- } catch (e: java.lang.Exception) {
|
|
|
- e.printStackTrace()
|
|
|
- }
|
|
|
+ if (isVideo) {
|
|
|
+ try {
|
|
|
+ File(recordPath).delete()
|
|
|
+ } catch (e: java.lang.Exception) {
|
|
|
+ e.printStackTrace()
|
|
|
}
|
|
|
- val builder = StringBuilder()
|
|
|
- builder.append(result.resultString)
|
|
|
- if (!TextUtils.isEmpty(builder)) {
|
|
|
- val resultParser = XmlResultParser()
|
|
|
- val parseResult = resultParser.parse(builder.toString()) as ReadSentenceResult
|
|
|
- val formatWords = arrayListOf<SimpleWord>()
|
|
|
- for (sentence in parseResult.sentences) {
|
|
|
- val words = sentence.words
|
|
|
- for (word in words) {
|
|
|
- // 过滤掉sil、 silv 表示静音， fil 表示噪音
|
|
|
- if (word.content == "sil" || word.content == "silv" || word.content == "fil")
|
|
|
- continue
|
|
|
- formatWords.add(SimpleWord(word.content, word.total_score))
|
|
|
- }
|
|
|
+ }
|
|
|
+ val builder = StringBuilder()
|
|
|
+ builder.append(result.resultString)
|
|
|
+ if (!TextUtils.isEmpty(builder)) {
|
|
|
+ val resultParser = XmlResultParser()
|
|
|
+ val parseResult = resultParser.parse(builder.toString()) as ReadSentenceResult
|
|
|
+ val formatWords = arrayListOf<SimpleWord>()
|
|
|
+ for (sentence in parseResult.sentences) {
|
|
|
+ val words = sentence.words
|
|
|
+ for (word in words) {
|
|
|
+ // 过滤掉sil、 silv 表示静音， fil 表示噪音
|
|
|
+ if (word.content == "sil" || word.content == "silv" || word.content == "fil")
|
|
|
+ continue
|
|
|
+ formatWords.add(SimpleWord(word.content, word.total_score))
|
|
|
}
|
|
|
-
|
|
|
- channel.invokeMethod("evaluatorResult",
|
|
|
- mapOf("index" to index, "score" to (if (parseResult.is_rejected) null else parseResult.total_score), "integrity_score" to (if (parseResult.is_rejected) null else parseResult.integrity_score),
|
|
|
- "accuracy_score" to (if (parseResult.is_rejected) null else parseResult.accuracy_score), "fluency_score" to (if (parseResult.is_rejected) null else parseResult.fluency_score),
|
|
|
- "words" to JSON.toJSONString(formatWords)))
|
|
|
- } else {
|
|
|
- channel.invokeMethod("evaluatorResult", mapOf("index" to index, "score" to null, "integrity_score" to null, "accuracy_score" to null, "fluency_score" to null))
|
|
|
}
|
|
|
+
|
|
|
+ invokeEvaluatorResult(index, score = (if (parseResult.is_rejected) null else parseResult.total_score), integrityScore = if (parseResult.is_rejected) null else parseResult.integrity_score,
|
|
|
+ accuracyScore = if (parseResult.is_rejected) null else parseResult.accuracy_score, fluencyScore = if (parseResult.is_rejected) null else parseResult.fluency_score,
|
|
|
+ words = JSON.toJSONString(formatWords))
|
|
|
+ } else {
|
|
|
+ invokeEvaluatorResult(index)
|
|
|
}
|
|
|
}
|
|
|
|
|
|
override fun onError(error: SpeechError) {
|
|
|
- (context as Activity).runOnUiThread {
|
|
|
- channel.invokeMethod("evaluatorResult", mapOf("index" to index, "score" to null))
|
|
|
- }
|
|
|
+ invokeEvaluatorResult(index)
|
|
|
}
|
|
|
})
|
|
|
?.startEvaluator(en, recordPath, decodePath, evaluatorType)
|
|
|
}
|
|
|
}
|
|
|
+
|
|
|
+ fun invokeEvaluatorResult(index: Int, score: Float? = null, integrityScore: Float? = null, accuracyScore: Float? = null, fluencyScore: Float? = null,
|
|
|
+ words: String? = null) {
|
|
|
+ (context as Activity).runOnUiThread {
|
|
|
+ channel.invokeMethod("evaluatorResult",
|
|
|
+ mapOf("index" to index, "score" to score, "integrity_score" to integrityScore,
|
|
|
+ "accuracy_score" to accuracyScore, "fluency_score" to fluencyScore,
|
|
|
+ "words" to words))
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
enum class SpeechSdk {
|
|
|
- IFly, Chivox,
|
|
|
+ IFly, Chivox, SmartOral
|
|
|
}
|
hwh97