#import "SpeechPlugin.h" #import #import "M4aToPcmHelper.h" #import "Mp4ToPcmHelper.h" #import "Results/ISEResult.h" #import "Results/ISEResultXmlParser.h" #import "Results/ISEResultTools.h" #import "aiengine.h" #import #import @interface SpeechPlugin () @property (nonatomic, strong) IFlySpeechEvaluator *iFlySpeechEvaluator; @property (nonatomic, strong) NSNumber *index; @property struct aiengine * engine; @property (strong, nonatomic) TAIOralEvaluation *oralEvaluation; @property (nonatomic) BOOL xunFeiOneVoice; @end @implementation SpeechPlugin + (void)registerWithRegistrar:(NSObject*)registrar { _channel = [FlutterMethodChannel methodChannelWithName:@"speech_plugin" binaryMessenger:[registrar messenger]]; // [IFlySetting setLogFile:LVL_ALL]; SpeechPlugin* instance = [[SpeechPlugin alloc] init]; [registrar addMethodCallDelegate:instance channel: _channel]; } - (void)handleMethodCall:(FlutterMethodCall*)call result:(FlutterResult)result { if ([@"getPlatformVersion" isEqualToString:call.method]) { result([@"iOS " stringByAppendingString:[[UIDevice currentDevice] systemVersion]]); } else if ([@"initSpeechSdk" isEqualToString:call.method]) { [self iflyInit]; self.oralEvaluation=[[TAIOralEvaluation alloc]init]; self.oralEvaluation.delegate=self; result(nil); } else if ([@"evaluatorByAudio" isEqualToString:call.method]) { NSNumber* index = call.arguments[@"index"]; NSString* recordPath = call.arguments[@"recordPath"]; NSString* text = call.arguments[@"en"]; NSString* pathEvaluatorDecode = call.arguments[@"pathEvaluatorDecode"]; NSString* videoId = call.arguments[@"videoId"]; NSNumber* evaluatorType = call.arguments[@"evaluatorType"]; NSNumber* sdkType = call.arguments[@"sdkType"]; [self evaluateVoice:index andPath:recordPath andText:text andIsVideo:false pathEvaluatorDecode:pathEvaluatorDecode videoId:videoId evaluatorType:evaluatorType sdkType:sdkType]; } else if ([@"evaluatorByMp4" isEqualToString:call.method]) { NSNumber* index = call.arguments[@"index"]; NSString* recordPath = call.arguments[@"recordPath"]; NSString* text = call.arguments[@"en"]; NSString* pathEvaluatorDecode = call.arguments[@"pathEvaluatorDecode"]; NSString* videoId = call.arguments[@"videoId"]; NSNumber* evaluatorType = call.arguments[@"evaluatorType"]; NSNumber* sdkType = call.arguments[@"sdkType"]; [self evaluateVoice:index andPath:recordPath andText:text andIsVideo:true pathEvaluatorDecode:pathEvaluatorDecode videoId:videoId evaluatorType:evaluatorType sdkType:sdkType]; } else { result(FlutterMethodNotImplemented); } } #pragma mark - Bridge Actions - (void)iflyInit { [IFlySpeechUtility createUtility:@"appid=5db7af6b"]; self.iFlySpeechEvaluator = [IFlySpeechEvaluator sharedInstance]; self.iFlySpeechEvaluator.delegate = self; [self configEvaluator]; } - (void)chivosInit:(NSString *)en { char cfg[4096]; char version[512] = {0}; char record_id[64] = {0}; char param[4096]; NSString* user_id = @"ios_user"; /*获取当前SDK版本号*/ aiengine_opt(NULL, AIENGINE_OPT_GET_VERSION, version, sizeof(version)); NSLog(@"version: %s\n",version); /*获取证书路径*/ NSString * provision = [[NSBundle mainBundle] pathForResource:@"aiengine" ofType:@"provision"]; /*引擎初始化传参设置*/ NSMutableDictionary *jsonDic = [[NSMutableDictionary alloc] initWithCapacity:10]; /*授权参数*/ [jsonDic setValue:@"157775873600002d" forKey:@"appKey"]; [jsonDic setValue:@"a6c766845f9a83974aed16f103e60621" forKey:@"secretKey"]; [jsonDic setValue:provision forKey:@"provision"]; /*日志传参*/ NSMutableDictionary *logDic = [[NSMutableDictionary alloc] init]; NSString *logFileName = @"log.txt"; [logDic setValue:[NSNumber numberWithInt:0] forKey:@"enable"]; [logDic setValue:logFileName forKey:@"output"]; /*服务链接参数*/ NSMutableDictionary *cloudDic = [[NSMutableDictionary alloc] init]; NSString *serverPath = @"wss://cloud.chivox.com:443"; [cloudDic setValue:[NSNumber numberWithInt:1] forKey:@"enable"]; [cloudDic setValue:serverPath forKey:@"server"]; [cloudDic setValue:[NSNumber numberWithInt:60] forKey:@"serverTimeout"]; /*初始化传参设置*/ [jsonDic setValue:logDic forKey:@"prof"]; [jsonDic setValue:cloudDic forKey:@"cloud"]; NSString *jsonString; jsonString = [self dicToString:jsonDic]; /*cfg赋值:string转char*/ strcpy(cfg, [jsonString UTF8String]); NSLog(@"cfg: %s\n",cfg); /*评分引擎初始化*/ _engine = aiengine_new(cfg); NSLog(@"engine: %p\n", _engine); /*引擎启用param传参设置*/ NSMutableDictionary *paramDic = [[NSMutableDictionary alloc] initWithCapacity:10]; /*在线离线参数配置(cloud/native)*/ [paramDic setValue:@"cloud" forKey:@"coreProvideType"]; /*音量实时返回参数设置(0关/1开)*/ [paramDic setValue:[NSNumber numberWithInt:0] forKey:@"soundIntensityEnable"]; /*appUser传参*/ NSMutableDictionary *appDic = [[NSMutableDictionary alloc] init]; [appDic setValue:user_id forKey:@"userId"]; /*audio音频数据传参*/ NSMutableDictionary *audioDic = [[NSMutableDictionary alloc] init]; [audioDic setValue:@"m4a" forKey:@"audioType"];//音频编码格式 [audioDic setValue:[NSNumber numberWithInt:16000] forKey:@"sampleRate"];//音频采样率 [audioDic setValue:[NSNumber numberWithInt:1] forKey:@"channel"];//单声道设置 [audioDic setValue:[NSNumber numberWithInt:2] forKey:@"sampleBytes"];//采样字节数(1-单字节-8位,2-双字节-16位) /*内核request传参*/ NSMutableDictionary *requestDic = [[NSMutableDictionary alloc] init]; //内核类型设置cn.word.raw/cn.sent.raw/cn.pred.raw [requestDic setValue:@"en.sent.score" forKey:@"coreType"]; [requestDic setValue:en forKey:@"refText"];//评测文本 [requestDic setValue:[NSNumber numberWithInt:100] forKey:@"rank"];//总分分制 [requestDic setValue:[NSNumber numberWithInt:1] forKey:@"attachAudioUrl"]; [requestDic setValue:[NSNumber numberWithInt:1] forKey:@"precision"];//段落评分精度设置(0.5/1) [paramDic setValue:appDic forKey:@"app"]; [paramDic setValue:audioDic forKey:@"audio"]; [paramDic setValue:requestDic forKey:@"request"]; NSString *paramString; paramString = [self dicToString:paramDic]; /*cfg赋值:string转char*/ strcpy(param, [paramString UTF8String]); int rv = 0; rv = aiengine_start(_engine,param,record_id,(aiengine_callback)_aiengine_callback, (__bridge const void *)(self)); if (rv) { NSLog(@"aiengine_start() failed: %d\n", rv); return; } } /*字典转为String*/ - (NSString *)dicToString: (NSMutableDictionary *) jsonDictionary{ NSError *error; NSData *jsonData = [NSJSONSerialization dataWithJSONObject:jsonDictionary options:0 error:&error]; NSString *jsString; if (!jsonData) { NSLog(@"sorry you get an error:%@",error); } else { jsString = [[NSString alloc]initWithData:jsonData encoding:NSUTF8StringEncoding]; } NSLog(@"引擎初始化传参#####:%@",jsString); return jsString; } - (void)configEvaluator { [self.iFlySpeechEvaluator setParameter:@"" forKey:[IFlySpeechConstant PARAMS]]; [self.iFlySpeechEvaluator setParameter:@"read_sentence" forKey:[IFlySpeechConstant ISE_CATEGORY]]; [self.iFlySpeechEvaluator setParameter:@"en_us" forKey:[IFlySpeechConstant LANGUAGE]]; [self.iFlySpeechEvaluator setParameter:@"5000" forKey:[IFlySpeechConstant VAD_BOS]]; [self.iFlySpeechEvaluator setParameter:@"1800" forKey:[IFlySpeechConstant VAD_EOS]]; [self.iFlySpeechEvaluator setParameter:@"-1" forKey:[IFlySpeechConstant SPEECH_TIMEOUT]]; [self.iFlySpeechEvaluator setParameter:@"complete" forKey:[IFlySpeechConstant ISE_RESULT_LEVEL]]; [self.iFlySpeechEvaluator setParameter:@"16000" forKey:[IFlySpeechConstant SAMPLE_RATE]]; [self.iFlySpeechEvaluator setParameter:@"xml" forKey:[IFlySpeechConstant ISE_RESULT_TYPE]]; [self.iFlySpeechEvaluator setParameter:@"0" forKey:@"plev"]; [self.iFlySpeechEvaluator setParameter:@"-1" forKey:@"audio_source"]; } - (void) evaluateVoice: (NSNumber*)index andPath:(NSString*)path andText:(NSString*)text andIsVideo:(BOOL) isVideo pathEvaluatorDecode:(NSString*)pathEvaluatorDecode videoId:(NSString*)videoId evaluatorType:(NSNumber*)evaluatorType sdkType:(NSNumber*)sdkType { self.index = index; if (sdkType.intValue == 0) { //调用讯飞解析 self.xunFeiOneVoice = true; [self.iFlySpeechEvaluator setParameter:evaluatorType.stringValue forKey:@"plev"]; if(isVideo) { [Mp4ToPcmHelper Mp4ToPcmWithUrl:[[NSURL alloc] initFileURLWithPath:path] completion:^(NSData *data) { if(data == nil) { [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; return; } NSStringEncoding encoding = CFStringConvertEncodingToNSStringEncoding(kCFStringEncodingGB_18030_2000); NSMutableData *buffer= [NSMutableData dataWithData:[text dataUsingEncoding:encoding]]; [self.iFlySpeechEvaluator startListening:buffer params:nil]; [self.iFlySpeechEvaluator writeAudio:data]; [self.iFlySpeechEvaluator stopListening]; }]; } else { NSURL *voiceUrl = [M4aToPcmHelper M4aToPcmWithUrl:[[NSURL alloc] initFileURLWithPath:path]]; if(voiceUrl == nil) { NSLog(@"voice url is nil"); [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; return; } NSStringEncoding encoding = CFStringConvertEncodingToNSStringEncoding(kCFStringEncodingGB_18030_2000); NSMutableData *buffer= [NSMutableData dataWithData:[text dataUsingEncoding:encoding]]; [self.iFlySpeechEvaluator startListening:buffer params:nil]; dispatch_sync(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ NSData *data = [[NSData alloc] initWithContentsOfURL:voiceUrl]; dispatch_async(dispatch_get_main_queue(), ^{ [self.iFlySpeechEvaluator writeAudio:data]; [self.iFlySpeechEvaluator stopListening]; }); }); } }else if(sdkType.intValue == 1){ //调用池声解析 [self chivosInit:text]; int bytes = 0; char buf[1024]={0}; FILE *file = NULL; const char * audiopath = [path UTF8String]; file = fopen(audiopath, "rb");//待评分音频地址 if(!file) { printf("read file error!\n"); return; } // fseek(file, 44, SEEK_SET); //wav音频文件需要跳过头文件信息,其它音频格式不需要 while ((bytes = (int)fread(buf, 1, 1024, file))) { aiengine_feed(_engine, buf, bytes); } aiengine_stop(_engine); }else { //调用腾讯解析 TAIOralEvaluationParam *param =[[TAIOralEvaluationParam alloc]init]; param.sessionId = [[NSUUID UUID]UUIDString]; param.appId =@"1301049120"; param.workMode = TAIOralEvaluationWorkMode_Once; param.evalMode = TAIOralEvaluationEvalMode_Sentence; param.storageMode = TAIOralEvaluationStorageMode_Disable; param.serverType = TAIOralEvaluationServerType_English; param.scoreCoeff = 1.0; param.fileType = TAIOralEvaluationFileType_Wav; param.refText = text; param.secretId = @"AKIDUf0EEzc8NMpPmu4Po1zhXBZicKp5G7xZ"; param.secretKey = @"WnsjKaqtgDVbV9cMtABwVptarwpWyBAt"; TAIOralEvaluationData *data = [[TAIOralEvaluationData alloc] init]; data.seqId = 1; data.bEnd = YES; data.audio = [NSData dataWithContentsOfFile:path]; [self.oralEvaluation oralEvaluation:param data:data callback:^(TAIError *error) { //接口调用结果返回 if(error){ NSLog(@"调用腾讯失败:code:%ld , msg:%@",(long)error.code ,error.desc); }else{ NSLog(@"调用腾讯成功"); } }]; } } #pragma mark - iFly delegate // 评测结果回调 - (void)onResults:(NSData *)results isLast:(BOOL)isLast { NSLog(@"onResults"); BOOL isSuccess = false; if (isLast) { if(results) { const char* chResult = [results bytes]; NSString* strResults = nil; NSStringEncoding encoding = CFStringConvertEncodingToNSStringEncoding(kCFStringEncodingGB_18030_2000); strResults = [[NSString alloc] initWithBytes:chResult length:[results length] encoding:encoding]; if(strResults != nil) { NSLog(@"onResults is success"); ISEResultXmlParser *parser = [ISEResultXmlParser alloc]; [parser setDelegate:self]; [parser parserXml:strResults]; isSuccess = YES; } } } if(isSuccess == NO){ NSLog(@"onResults is not success"); self.xunFeiOneVoice = false; [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; } } - (void)onCompleted:(IFlySpeechError *)errorCode { NSLog(@"onCompleted"); if (errorCode.errorCode != 0 || self.xunFeiOneVoice == true) { NSLog(@"onCompleted not error code"); self.xunFeiOneVoice = false; [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; } } - (void)onCancel { NSLog(@"取消录音"); } - (void)onBeginOfSpeech { NSLog(@"开始录音"); } - (void)onEndOfSpeech { NSLog(@"停止录音"); } - (void)onVolumeChanged:(int)volume buffer:(NSData *)buffer { NSLog(@"音量改变"); } #pragma mark - ISEResultXmlParser delegate -(void)onISEResultXmlParser:(NSXMLParser *)parser Error:(NSError*)error { NSLog(@"onISEResultXmlParser"); if (error.code != 0) { [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; } } -(void)onISEResultXmlParserResult:(ISEResult*)result { NSLog(@"onISEResultXmlParserResult"); if (result.is_rejected) { NSLog(@"onISEResultXmlParserResult reject"); self.xunFeiOneVoice = false; [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; } else { NSLog(@"onISEResultXmlParserResult not reject"); NSMutableDictionary *dic = [NSMutableDictionary dictionary]; [dic setValue:self.index forKey:@"index"]; [dic setValue:@(result.total_score) forKey:@"score"]; [dic setValue:@(result.accuracy_score) forKey:@"accuracy_score"]; [dic setValue:@(result.fluency_score) forKey:@"fluency_score"]; [dic setValue:@(result.integrity_score) forKey:@"integrity_score"]; NSString* words = [ISEResultTools formatDetailsForLanguageEN: result.sentences]; [dic setValue:words forKey:@"words"]; self.xunFeiOneVoice = false; [_channel invokeMethod:@"evaluatorResult" arguments: dic]; } } #pragma mark - chisheng回掉 /*评分引擎回调方法*/ int _aiengine_callback(const void *usrdata, const char *id, int type, const void *message, int size) { if (type == AIENGINE_MESSAGE_TYPE_JSON) { [(__bridge SpeechPlugin *)usrdata performSelectorOnMainThread:@selector(onChiResult:) withObject:[[NSString alloc] initWithUTF8String:(char *)message] waitUntilDone:NO]; } return 0; } //返回的内容 -(void) onChiResult:(NSString *) result{ if(result == nil){ [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; return; } NSLog(@"返回的结果:%@",result); NSData *jsonData = [result dataUsingEncoding:NSUTF8StringEncoding]; NSError *err = nil; NSMutableDictionary *dic = [NSJSONSerialization JSONObjectWithData:jsonData options:NSJSONReadingMutableContainers error:&err]; if(err) { NSLog(@"json解析失败:%@",err); [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; return ; } NSNumber *score = dic[@"result"][@"overall"]; NSNumber *integrity = dic[@"result"][@"integrity"]; NSNumber *accuracy = dic[@"result"][@"accuracy"]; NSNumber *fluency = dic[@"result"][@"fluency"][@"overall"]; NSArray* worksList = dic[@"result"][@"details"]; NSMutableDictionary *dict = [NSMutableDictionary dictionary]; dict[@"index"] = self.index; dict[@"score"] = @(score.doubleValue/20); dict[@"accuracy_score"] = @(accuracy.doubleValue/20); dict[@"fluency_score"] = @(fluency.doubleValue/20); dict[@"integrity_score"] = @(integrity.doubleValue/20); NSMutableArray * works =[NSMutableArray array]; for (NSDictionary* work in worksList) { NSMutableDictionary *workDict = [NSMutableDictionary dictionary]; workDict[@"score"] = work[@"score"]; workDict[@"content"] = work[@"char"]; [works addObject:workDict]; } NSData *data = [NSJSONSerialization dataWithJSONObject:works options:NSJSONWritingPrettyPrinted error:nil]; NSString *jsonStr = [[NSString alloc]initWithData:data encoding:NSUTF8StringEncoding]; [dict setValue:jsonStr forKey:@"words"]; [_channel invokeMethod:@"evaluatorResult" arguments: dict]; if (_engine) { aiengine_delete(_engine); _engine = NULL; } } #pragma mark - 腾讯会掉 - (void)oralEvaluation:(TAIOralEvaluation *)oralEvaluation onEvaluateData:(TAIOralEvaluationData *)data result:(TAIOralEvaluationRet *)result error:(TAIError *)error { //数据和结果回调(只有data.bEnd为YES,result有效) if (data.bEnd) { float pronFluency = result.pronFluency *5; float pronAccuracy = result.pronAccuracy; if(pronAccuracy == -1){ pronAccuracy = 0; }else { pronAccuracy = pronAccuracy/20; } float pronCompletion = result.pronCompletion * 5; float totalScore = (pronFluency + pronAccuracy + pronCompletion)/ 3; NSMutableArray* workList =[NSMutableArray array]; for (TAIOralEvaluationWord* work in result.words) { NSMutableDictionary *dic =[NSMutableDictionary dictionary]; float score = work.pronAccuracy; if(score == -1.0){ score = 0; }else{ score = score/20; } dic[@"score"]= [NSNumber numberWithFloat:score]; dic[@"content"] = work.word; [workList addObject:dic]; } NSMutableDictionary *dict = [NSMutableDictionary dictionary]; [dict setValue:self.index forKey:@"index"]; [dict setValue:@(totalScore) forKey:@"score"]; [dict setValue:@(pronAccuracy) forKey:@"accuracy_score"]; [dict setValue:@(pronFluency) forKey:@"fluency_score"]; [dict setValue:@(pronCompletion) forKey:@"integrity_score"]; NSData *data = [NSJSONSerialization dataWithJSONObject:workList options:NSJSONWritingPrettyPrinted error:nil]; NSString *jsonStr = [[NSString alloc]initWithData:data encoding:NSUTF8StringEncoding]; [dict setValue:workList forKey:@"words"]; [_channel invokeMethod:@"evaluatorResult" arguments: dict]; }else{ [_channel invokeMethod:@"evaluatorResult" arguments: [NSDictionary dictionaryWithObjectsAndKeys: self.index, @"index", [NSNull null], @"score", nil]]; } } @end