#include #include #include #include #include #include "fhhcp/sys.h" #include "fhhcp/npu.h" #include "types/vmm_api.h" #include "npu_com.h" #include "fh_vpu_mpi.h" #include "fh_tde_mpi.h" #include "types/bufCtrl.h" #include "common.h" //#include "npu_draw.h" #define RGB_CHN_DEFAULT 3 #define DEFAULT_YC_TIMEOUT 2000 //#define COLOR_FLASSIER 0xE083 #define COLOR_FLASSIER 0xFC00 #define DEFAULT_LINE_W 1 #define ACTURE_LINE_W 4 static pthread_t g_sample_npu_thrd = 0; static FH_BOOL g_npu_test_start = FH_FALSE; FH_UINT32 g_nna_FBPhyAddr = 0; //static FH_SINT32 g_nna_FBfd = -1; // extern bool g_bRunning; #if 0 static void printMdlDesc(T_TY_ModelDesc *desc){ INFO_LOG("model desc: version:%s", desc->info.version); INFO_LOG("model desc: compile:%s", desc->info.compileDate); INFO_LOG("model desc: input number:%d", desc->ioDesc.inputNum); uint32_t idx=0; for(;idxioDesc.inputNum; idx++){ INFO_LOG("input blob#%d", idx); printBlob(&desc->ioDesc.in[idx]); } INFO_LOG("model desc: output number:%d", desc->ioDesc.outputNum); idx=0; for(; idxioDesc.outputNum; idx++){ INFO_LOG("output blob#%d", idx); printBlob(&desc->ioDesc.out[idx]); } } #endif FH_UINT32 nna_ui_get_draw_addr() { return g_nna_FBPhyAddr; } static uint32_t npu_getMdlInputNum(T_TY_ModelDesc *desc){ return desc->ioDesc.inputNum; } static uint32_t npu_getMdlOutputNum(T_TY_ModelDesc *desc){ return desc->ioDesc.outputNum; } static void npu_freeTaskInOutBuf(T_TY_TaskInput inVec[], uint32_t inNum, T_TY_TaskOutput outVec[], uint32_t outNum){ uint32_t i=0; for(; iioDesc.inputNum == npuOp->inImgNum); assert(desc->ioDesc.inputNum == inNum); assert(desc->ioDesc.outputNum== outNum); int ret = FH_SUCCESS; uint32_t idx=0; for(; idxioDesc.inputNum; idx++){ T_TY_TaskInput *ptrTyIn = &inVec[idx]; stImage *ptrImg = &npuOp->inImageVec[idx]; T_TY_BlobDesc *ptrBlobDesc = &desc->ioDesc.in[idx]; ptrTyIn->descIn = *ptrBlobDesc; if(ptrTyIn->descIn.type == E_TY_BLOB_DATA){ ptrTyIn->dataIn.size = npu_getBlobSize(ptrBlobDesc); }else if(ptrTyIn->descIn.type == E_TY_BLOB_IMAGE_WITH_PRE_PROC){ ptrTyIn->descIn.type = E_TY_BLOB_IMAGE; ptrTyIn->descIn.img.picFormat = ptrImg->fmt; ptrTyIn->descIn.img.picWidth = ptrImg->width; ptrTyIn->descIn.img.picHeight = ptrImg->height; ptrTyIn->descIn.img.picWidthStride = ptrTyIn->descIn.img.picWidth; //根据图片实际stride填写 ptrTyIn->descIn.img.picHeightStride= ptrTyIn->descIn.img.picHeight; //根据图片实际stride填写 ptrTyIn->descIn.img.roi.x = 0; ptrTyIn->descIn.img.roi.y = 0; ptrTyIn->descIn.img.roi.width = ptrTyIn->descIn.img.picWidth; ptrTyIn->descIn.img.roi.height= ptrTyIn->descIn.img.picHeight; ptrTyIn->dataIn.size = npu_getImageSize(ptrTyIn->descIn.img.picWidth, ptrTyIn->descIn.img.picHeight, ptrTyIn->descIn.img.picFormat); }else{ assert(0); //TBD,暂时没有原始模型输入Image的情况 } ret = npu_mallocMem((void**)&ptrTyIn->dataIn.virAddr, &ptrTyIn->dataIn.phyAddr, ptrTyIn->dataIn.size, 8, E_TY_MEM_VMM_CACHED); if(ret != FH_SUCCESS) { return FH_FAILURE; } } idx=0; for(; idxioDesc.outputNum; idx++){ T_TY_TaskOutput *out = &outVec[idx]; out->dataOut.size = npu_getBlobSize(&desc->ioDesc.out[idx]); ret = npu_mallocMem((void**)&out->dataOut.virAddr, &out->dataOut.phyAddr, out->dataOut.size, 8, E_TY_MEM_VMM_CACHED); if(ret != FH_SUCCESS) { return FH_FAILURE; } memset((FH_VOID *)out->dataOut.virAddr, 0, out->dataOut.size); npu_flushVmmMem(out->dataOut.virAddr, out->dataOut.phyAddr, out->dataOut.size); } return ret; } int sample_npu_test_init(FY_NPU_INFO *npu_info) { int32_t tyRet; T_TY_TaskCfgParam tskCfg; T_TY_ModelCfgParam mdlCfg; printf("*******************[0]before TY_NPU_SysInit********************\n"); tyRet = TY_NPU_SysInit(); SAMPLE_NNA_CHECK_EXPR_RET(tyRet != TY_NPU_SUCCESS, FH_FAILURE, "TY_NPU_SysInit fail, errCode:%d", tyRet); #if 0 uint32_t loglvl = 4; tyRet = TY_SDK_SetLogLevel(loglvl); if(TY_NPU_SUCCESS != tyRet){ PRINTF_LOG(ERROR_LEVEL, "TY_SDK_SetLogLevel fail, errCode:%d", tyRet); TY_NPU_SysExit(); return FH_FAILURE; } #endif char *envMdlPath = "./res/"; if(NULL == envMdlPath){ SAMPLE_NNA_PRT("please set models path and data_in path"); TY_NPU_SysExit(); return FH_FAILURE; } /*========== 1. 分配加载模型文件的内存 =========*/ strncpy(npu_info->npuOp.mdlPath, envMdlPath, MAX_PATH_LEN); strncpy(npu_info->npuOp.mdlPath + strlen(npu_info->npuOp.mdlPath), NPU_MODEL_BIN, MAX_PATH_LEN - strlen(npu_info->npuOp.mdlPath)); npu_info->mdlMemInfo.segNum = 1; npu_info->mdlMemInfo.memInfo[0].allocInfo.alignByteSize = 128; npu_info->mdlMemInfo.memInfo[0].allocInfo.allocType = E_TY_MEM_VMM_CACHED; npu_info->mdlMemInfo.memInfo[0].allocInfo.shareType = E_MEM_EXCLUSIVED; npu_info->mdlMemInfo.memInfo[0].allocInfo.size = npu_GetFileSize(npu_info->npuOp.mdlPath); tyRet = npu_allocMemSegments(&npu_info->mdlMemInfo); if(FH_SUCCESS != tyRet){ SAMPLE_NNA_PRT("npu_allocMemSegments failed"); TY_NPU_SysExit(); return FH_FAILURE; } // 1.1 读取模型文件 tyRet = npu_readFile(&npu_info->mdlMemInfo.memInfo[0].mem, npu_info->npuOp.mdlPath); if(FH_SUCCESS != tyRet){ SAMPLE_NNA_PRT("read model file(%s) error", npu_info->npuOp.mdlPath); TY_NPU_SysExit(); npu_freeMemSegments(&npu_info->mdlMemInfo); return FH_FAILURE; } npu_flushVmmMem(npu_info->mdlMemInfo.memInfo[0].mem.virAddr, npu_info->mdlMemInfo.memInfo[0].mem.phyAddr, npu_info->mdlMemInfo.memInfo[0].mem.size); tyRet = TY_NPU_CreateModelFromPhyMem(&npu_info->mdlMemInfo, &mdlCfg, &npu_info->mdlMemInfo, &npu_info->mdlDesc, &npu_info->mdlHdl); if(TY_NPU_SUCCESS != tyRet){ SAMPLE_NNA_PRT("TY_NPU_CreateModel fail, errCode:%d", tyRet); TY_NPU_SysExit(); npu_freeMemSegments(&npu_info->mdlMemInfo); return FH_FAILURE; } /*========== 2. 获取task需要的内存信息,分配task内存并创建task =========*/ tyRet = TY_NPU_GetTaskMemSize(npu_info->mdlHdl, &tskCfg, &npu_info->tskMemInfo); if(TY_NPU_SUCCESS != tyRet){ SAMPLE_NNA_PRT("TY_NPU_GetTaskMemSize fail, errCode:%d", tyRet); TY_NPU_ReleaseModel(npu_info->mdlHdl); TY_NPU_SysExit(); npu_freeMemSegments(&npu_info->mdlMemInfo); return FH_FAILURE; } npu_allocMemSegments(&npu_info->tskMemInfo); tyRet = TY_NPU_CreateTask(npu_info->mdlHdl, &tskCfg, &npu_info->tskMemInfo, &npu_info->tskHdl); if(TY_NPU_SUCCESS != tyRet){ SAMPLE_NNA_PRT("TY_NPU_CreateTask fail, errCode:%d", tyRet); TY_NPU_ReleaseModel(npu_info->mdlHdl); TY_NPU_SysExit(); npu_freeMemSegments(&npu_info->tskMemInfo); npu_freeMemSegments(&npu_info->mdlMemInfo); return FH_FAILURE; } npu_info->npuOp.inImgNum = 1; npu_info->npuOp.inImageVec[0].width = NPU_INPUT_WIDTH; npu_info->npuOp.inImageVec[0].height= NPU_INPUT_HEIGHT; npu_info->npuOp.inImageVec[0].fmt = E_TY_PIXEL_FORMAT_RGB_888_PLANAR; uint32_t mdlInNum = npu_getMdlInputNum(&npu_info->mdlDesc); uint32_t mdlOutNum = npu_getMdlOutputNum(&npu_info->mdlDesc); npu_info->taskInVec = malloc(sizeof(T_TY_TaskInput) * mdlInNum); npu_info->taskOutVec = malloc(sizeof(T_TY_TaskOutput) * mdlOutNum); if((NULL == npu_info->taskInVec) || (NULL == npu_info->taskOutVec)){ SAMPLE_NNA_PRT( "malloc input and output error, inNum %d, outNum:%d", mdlInNum, mdlOutNum); TY_NPU_ReleaseTask(npu_info->tskHdl); TY_NPU_ReleaseModel(npu_info->mdlHdl); TY_NPU_SysExit(); npu_freeMemSegments(&npu_info->tskMemInfo); npu_freeMemSegments(&npu_info->mdlMemInfo); if(npu_info->taskInVec != NULL) free(npu_info->taskInVec); if(npu_info->taskOutVec!= NULL) free(npu_info->taskOutVec); return FH_FAILURE; } // prepareModelInOutCache(&mdlDesc, &npuOp, taskInVec, mdlInNum, taskOutVec, mdlOutNum); return 0; } FH_SINT32 dump_vpss_frame_data(FH_UINT8 *addr, int size) { FILE *file; SAMPLE_NNA_CHECK_EXPR_GOTO(addr == FH_NULL, exit, "address is null\n"); file = fopen("rgb_512x288.rgb", "wb"); if (file == NULL) { SAMPLE_NNA_PRT("open rgb_512x288.rgb err\n"); return FH_FAILURE; } fwrite(addr, size, 1, file); fflush(file); fclose(file); exit: return FH_FAILURE; } static FH_SINT32 npu_check_parmas(RECT_S *rect, int chn_width, int chn_height) { FH_SINT32 ret = 0; if (rect->s32X < 0 || rect->s32Y < 0 || rect->u32Width < 0 || rect->u32Height < 0) { ret = FH_FAILURE; } if ((rect->s32X + rect->u32Width) > chn_width) // fix over size { rect->u32Width = chn_width - rect->s32X - 1; } if ((rect->s32Y + rect->u32Height) > chn_height) { rect->u32Height = chn_height - rect->s32Y - 1; } return ret; } FH_SINT32 nna_draw_box(FH_UINT32 pAddr, FH_SINT32 mw, FH_SINT32 mh, RECT_S *box, FH_SINT32 box_num, FH_UINT32 color) { FH_SINT32 ret; TDE2_SURFACE_S stDst; TDE2_RECT_S stDstRect; FH_SINT32 tde_handle; tde_handle = FH_TDE2_BeginJob(); stDst.enColorFmt = TDE2_COLOR_FMT_ARGB1555; stDst.u32Width = mw; stDst.u32Height = mh; stDst.u32Stride = mw*2; stDst.u32PhyAddr = pAddr; stDst.bAlphaMax255 = 1; stDst.bAlphaExt1555 = 0; stDst.u8Alpha0 = 0; stDst.u8Alpha1 = 255; //clear screen stDstRect.s32Xpos = 0; stDstRect.s32Ypos = 0; stDstRect.u32Width = mw-1; stDstRect.u32Height = mh-1; printf("[fill 0]%d-%d-%d-%d\n",stDstRect.s32Xpos,stDstRect.s32Ypos,stDstRect.u32Width,stDstRect.u32Height); ret = FH_TDE2_QuickFill(tde_handle, &stDst, &stDstRect, 0x0000); if(box_num == 0) { printf("have no detection result\n"); goto exit; } box_num = box_num<7?box_num:7; if (ret) { goto error; } FH_SINT32 num = 0; for(; nummw) { x=mw; } if(y>mh) { y=mh; } if((x+w)>mw) { printf("[nna] width error(%d -- %d)\n",x,w); goto error; } else if((y+h)>mh) { h -= (y+h) - mh; } stDstRect.s32Xpos = x; stDstRect.s32Ypos = y; stDstRect.u32Width = w>ACTURE_LINE_W?ACTURE_LINE_W:DEFAULT_LINE_W; stDstRect.u32Height = h; printf("[fill 1]%d-%d-%d-%d\n",stDstRect.s32Xpos,stDstRect.s32Ypos,stDstRect.u32Width,stDstRect.u32Height); ret = FH_TDE2_QuickFill(tde_handle, &stDst, &stDstRect, color); if (ret) { goto error; } stDstRect.s32Xpos = x; stDstRect.s32Ypos = y; stDstRect.u32Width = w; stDstRect.u32Height = h>ACTURE_LINE_W?ACTURE_LINE_W:DEFAULT_LINE_W; printf("[fill 2]%d-%d-%d-%d\n",stDstRect.s32Xpos,stDstRect.s32Ypos,stDstRect.u32Width,stDstRect.u32Height); ret = FH_TDE2_QuickFill(tde_handle, &stDst, &stDstRect, color); if (ret) { goto error; } stDstRect.s32Xpos = w>ACTURE_LINE_W*2?((x+w)-ACTURE_LINE_W):((x+w)-DEFAULT_LINE_W); stDstRect.s32Ypos = y; stDstRect.u32Width = w>ACTURE_LINE_W*2?ACTURE_LINE_W:DEFAULT_LINE_W; stDstRect.u32Height = h; printf("[fill 3]%d-%d-%d-%d\n",stDstRect.s32Xpos,stDstRect.s32Ypos,stDstRect.u32Width,stDstRect.u32Height); ret = FH_TDE2_QuickFill(tde_handle, &stDst, &stDstRect, color); if (ret) { goto error; } stDstRect.s32Xpos = x; stDstRect.s32Ypos = h>ACTURE_LINE_W*2?((y+h)-ACTURE_LINE_W):((y+h)-DEFAULT_LINE_W); stDstRect.u32Width = w; stDstRect.u32Height = h>ACTURE_LINE_W*2?ACTURE_LINE_W:DEFAULT_LINE_W;; printf("[fill 4]%d-%d-%d-%d\n",stDstRect.s32Xpos,stDstRect.s32Ypos,stDstRect.u32Width,stDstRect.u32Height); ret = FH_TDE2_QuickFill(tde_handle, &stDst, &stDstRect, color); if (ret) { goto error; } } exit: ret = FH_TDE2_EndJob(tde_handle, 1, 1, 500); if (ret) { printf("FH_TDE2_EndJob failed! ret:0x%x(%d)\n",ret,ret); } return ret; error: printf("[nna] FH_TDE2_QuickFill failed! ret:0x%x(%d)\n",ret,ret); FH_TDE2_CancelJob(tde_handle); return FH_FAILURE; } int vpu_gosd_update_box(int grpid, int chn, int index, RECT_S *box, int box_num) { printf("vpu_gosd_update_box\n "); int ret = 0; //get the current osd FH_VPU_LOGOV2 logo; memset(&logo, 0, sizeof(logo)); logo.logo_idx = index; ret = FH_VPSS_GetChnGraphV2(grpid, chn, &logo); IPO_LOG_CHECK_GOTO(ret, exit, "FH_VPSS_GetChnGraphV2 failed! ret = 0x%x", ret); //allocate the memory, generate the bitmap int bitmap_w = logo.logo_cfg.logo_size.u32Width; /* 位图的宽 */ int bitmap_h = logo.logo_cfg.logo_size.u32Height; /* 位图的高 */ FH_UINT32 u32PhyAddr = 0; /* get video buffer block form common pool */ u32PhyAddr = logo.logo_addr; // printf("********* index = %d w = %d h = %d phyaddr = %d *********\n", logo.logo_idx, bitmap_w, bitmap_h, u32PhyAddr); if (u32PhyAddr == 0) { printf("ERR: SetUserPic FH_VPSS_GetChnGraphV2 logo get phyaddr failed!\n"); ret = -__LINE__; goto exit; } nna_draw_box(u32PhyAddr, bitmap_w, bitmap_h, box, box_num, COLOR_FLASSIER); exit: return ret; } static FH_SINT32 nna_draw_rect_ext(FY_NPU_INFO *nna_info, FH_UINT32 mdlOutNum, FH_SINT32 width, FH_SINT32 height, int grpid) { FH_SINT32 ret = 0; FH_SINT32 i = 0; FH_FLOAT *box_infor = NULL; FH_FLOAT *score = NULL; FH_UINT32 box_size; FH_SINT32 box_id = 0; RECT_S *box; RECT_S box_test; uint32_t nums = nna_info->taskOutVec[1].dataOut.size / sizeof(float); if (mdlOutNum == 2) { box_infor = (FH_FLOAT *)(FH_UINT32)(nna_info->taskOutVec[0].dataOut.virAddr); score = (FH_FLOAT *)(FH_UINT32)(nna_info->taskOutVec[1].dataOut.virAddr); box_size = (FH_UINT32)(&nna_info->taskOutVec[1].dataOut.size); box = malloc(sizeof(RECT_S) * box_size); box_id = 0; for (i = 0; i < nums; i++) { if (*score > 0.7) { box_test.s32X = ALIGNTO((FH_SINT32)(box_infor[1] * width), 2); box_test.s32Y = ALIGNTO((FH_SINT32)(box_infor[0] * height), 2); box_test.u32Width = ALIGNTO((FH_UINT32)((box_infor[3] - box_infor[1]) * width), 2); box_test.u32Height = ALIGNTO((FH_UINT32)((box_infor[2] - box_infor[0]) * height), 2); if (!npu_check_parmas(&box_test, width, height)) // over size check { SAMPLE_NNA_PRT("detect box idx[%d], pos[%d-%d-%d-%d], score=[%f]\n", box_id, box_test.s32X, box_test.s32Y, box_test.u32Width, box_test.u32Height, *score); box[box_id] = box_test; box_id++; } } box_infor += 4; // addr offset score += 1; } ret = vpu_gosd_update_box(grpid, 0, 2, box, box_id); free(box); } return ret; } FH_VOID* sample_npu_detect_thread(FH_VOID* pAtr) { FH_SINT32 ret; FH_SINT32 chn; FH_SINT32 vpssGrp; FH_UINT32 u32ChnNum; FY_NPU_INFO npu_info; E_TY_NpuID npuId = 0; FH_UINT32 mdlInNum = 0; FH_UINT32 mdlOutNum = 0; FH_UINT8 *pRGBaddr[3] = {NULL};//, *pGaddr = NULL, *pBaddr = NULL; FH_UINT32 u32VpssFrameWidth, u32VpssFrameHeight; FH_VPU_STREAM_ADV frameinfo; FH_UINT32 handle_lock; FH_UINT32 u32ComponentSize; FH_UINT32 u32Cnt = 0; FH_UINT32 i; // FH_UINT32 picnt = 0; memset(&npu_info, 0, sizeof(FY_NPU_INFO)); u32ChnNum = 1; ret = sample_npu_test_init(&npu_info); SAMPLE_NNA_CHECK_EXPR_GOTO(ret != FH_SUCCESS, exit_0, "sample_pp_npu_init failed\n"); mdlInNum = npu_getMdlInputNum(&npu_info.mdlDesc); mdlOutNum = npu_getMdlOutputNum(&npu_info.mdlDesc); ret = npu_prepareModelInOutCache(&npu_info.mdlDesc, &npu_info.npuOp, npu_info.taskInVec, mdlInNum, npu_info.taskOutVec, mdlOutNum); SAMPLE_NNA_CHECK_EXPR_GOTO(ret != FH_SUCCESS, exit_0, "npu_prepareModelInOutCache error\n"); printf("begin nna detection \n"); while(g_npu_test_start) { for(chn=0;chn