网站和数字界面设计师,佛山制作网站公司哪家好,网络营销的特点和职能,ppt模板免费下载古风ffmpeg 硬件解码
ffmpeg硬件解码可以使用最新的vulkan来做#xff0c;基本上来说#xff0c;不挑操作系统是比较重要的#xff0c;如果直接使用cuda也是非常好的选择。 AVPixelFormat sourcepf AV_PIX_FMT_NV12;// AV_PIX_FMT_NV12;// AV_PIX_FMT_YUV420P;AVPixelFormat d…ffmpeg 硬件解码
ffmpeg硬件解码可以使用最新的vulkan来做基本上来说不挑操作系统是比较重要的如果直接使用cuda也是非常好的选择。 AVPixelFormat sourcepf AV_PIX_FMT_NV12;// AV_PIX_FMT_NV12;// AV_PIX_FMT_YUV420P;AVPixelFormat destpf AV_PIX_FMT_YUV420P; //AV_PIX_FMT_BGR24AVBufferRef* hw_device_ctx NULL;下面的class 是我封装了一些功能只是示例读者可以自行修改读的是rtsp 流后面准备推流到gb28181上。 class c_AVDecoder:public TThreadRunable
{AVPixelFormat sourcepf AV_PIX_FMT_NV12;// AV_PIX_FMT_NV12;// AV_PIX_FMT_YUV420P;AVPixelFormat destpf AV_PIX_FMT_YUV420P; //AV_PIX_FMT_BGR24AVBufferRef* hw_device_ctx NULL;SDLDraw v_draw;int hw_decoder_init(AVCodecContext* ctx, const enum AVHWDeviceType type){int err 0;if ((err av_hwdevice_ctx_create(hw_device_ctx, type,NULL, NULL, 0)) 0) {fprintf(stderr, Failed to create specified HW device.\n);return err;}ctx-hw_device_ctx av_buffer_ref(hw_device_ctx);return err;}//SDLDraw g_draw;struct SwsContext* img_convert_ctx NULL;AVFormatContext* input_ctx NULL;int video_stream, ret;AVStream* video NULL;AVCodecContext* decoder_ctx NULL;AVCodec* decoder NULL;AVPacket packet;enum AVHWDeviceType type;//转换成yuv420 或者rgbint decode_write(AVCodecContext* avctx, AVPacket* packet){AVFrame* frame NULL, * sw_frame NULL;AVFrame* tmp_frame NULL;//AVFrame* pFrameDst NULL;unsigned char* out_buffer NULL;int ret 0;ret avcodec_send_packet(avctx, packet);if (ret 0) {fprintf(stderr, Error during decoding\n);return ret;}if (img_convert_ctx NULL){img_convert_ctx sws_getContext(avctx-width, avctx-height, sourcepf,WIDTH, HEIGHT, destpf, SWS_FAST_BILINEAR, NULL, NULL, NULL);}static int i 0;while (1) {if (!(frame av_frame_alloc()) || !(sw_frame av_frame_alloc())) {fprintf(stderr, Can not alloc frame\n);ret AVERROR(ENOMEM);goto fail;}//avctx-get_buffer2ret avcodec_receive_frame(avctx, frame);if (ret AVERROR(EAGAIN) || ret AVERROR_EOF) {av_frame_free(frame);av_frame_free(sw_frame);return 0;}else if (ret 0) {fprintf(stderr, Error while decoding\n);goto fail;}if (frame-format hw_pix_fmt) {/* retrieve data from GPU to CPU */sw_frame-format sourcepf; // AV_PIX_FMT_NV12;// AV_PIX_FMT_YUV420P;// AV_PIX_FMT_NV12;// AV_PIX_FMT_BGR24;// AV_PIX_FMT_YUV420P;if ((ret av_hwframe_transfer_data(sw_frame, frame, 0)) 0) {fprintf(stderr, Error transferring the data to system memory\n);goto fail;}//av_frame_copy_props(sw_frame, frame);tmp_frame sw_frame;}elsetmp_frame frame;#if 0if (out_buffer NULL){out_buffer (unsigned char*)av_malloc(av_image_get_buffer_size(destpf,WIDTH,HEIGHT,1));//av_image_alloc()av_image_fill_arrays(pFrameDst.data, pFrameDst.linesize, out_buffer,destpf, WIDTH, HEIGHT, 1);}#endifAVFrame* pFrameDst av_frame_alloc();av_image_alloc(pFrameDst-data, pFrameDst-linesize, WIDTH, HEIGHT, destpf, 1);
#if 1sws_scale(img_convert_ctx, tmp_frame-data, tmp_frame-linesize,0, avctx-height, pFrameDst-data, pFrameDst-linesize);#endif//cout into i endl;#if 1if (!v_draw.push(pFrameDst)){av_freep(pFrameDst-data[0]);av_frame_free(pFrameDst);}#endif
#if 0size av_image_get_buffer_size((AVPixelFormat)tmp_frame-format, tmp_frame-width,tmp_frame-height, 1);buffer (uint8_t*)av_malloc(size);if (!buffer) {fprintf(stderr, Can not alloc buffer\n);ret AVERROR(ENOMEM);goto fail;}ret av_image_copy_to_buffer(buffer, size,(const uint8_t* const*)tmp_frame-data,(const int*)tmp_frame-linesize, (AVPixelFormat)tmp_frame-format,tmp_frame-width, tmp_frame-height, 1);if (ret 0) {fprintf(stderr, Can not copy image to buffer\n);goto fail;}if ((ret (int)fwrite(buffer, 1, size, output_file)) 0) {fprintf(stderr, Failed to dump raw data.\n);goto fail;}
#endiffail:av_frame_free(frame);av_frame_free(sw_frame);//av_freep(buffer);if (ret 0)return ret;}}int func_init(const char* url){const char* stype dxva2;type av_hwdevice_find_type_by_name(stype);if (type AV_HWDEVICE_TYPE_NONE) {fprintf(stderr, Device type %s is not supported.\n, stype);fprintf(stderr, Available device types:);while ((type av_hwdevice_iterate_types(type)) ! AV_HWDEVICE_TYPE_NONE)fprintf(stderr, %s, av_hwdevice_get_type_name(type));fprintf(stderr, \n);return -1;}/* open the input file *///const char * filename h:/video/a.mp4;AVDictionary* opts NULL;av_dict_set(opts, rtsp_transport, tcp, 0);av_dict_set(opts, buffer_size, 1048576, 0);av_dict_set(opts, fpsprobesize, 5, 0);av_dict_set(opts, analyzeduration, 5000000, 0);if (avformat_open_input(input_ctx, url, NULL, opts) ! 0) {fprintf(stderr, Cannot open input file %s\n, url);return -1;}if (avformat_find_stream_info(input_ctx, NULL) 0) {fprintf(stderr, Cannot find input stream information.\n);return -1;}/* find the video stream information */ret av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, decoder, 0);if (ret 0) {fprintf(stderr, Cannot find a video stream in the input file\n);return -1;}video_stream ret;AVStream* stream input_ctx-streams[video_stream];float frame_rate stream-avg_frame_rate.num / stream-avg_frame_rate.den;//每秒多少帧std::cout frame_rate is: frame_rate std::endl;//优化直接定死/*for (int i 0;; i) {const AVCodecHWConfig* config avcodec_get_hw_config(decoder, i);if (!config) {fprintf(stderr, Decoder %s does not support device type %s.\n,decoder-name, av_hwdevice_get_type_name(type));return -1;}if (config-methods AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX config-device_type type) {hw_pix_fmt config-pix_fmt;break;}}*/if (!(decoder_ctx avcodec_alloc_context3(decoder)))return AVERROR(ENOMEM);video input_ctx-streams[video_stream];if (avcodec_parameters_to_context(decoder_ctx, video-codecpar) 0)return -1;decoder_ctx-get_format get_hw_format;if (hw_decoder_init(decoder_ctx, type) 0)return -1;if ((ret avcodec_open2(decoder_ctx, decoder, NULL)) 0) {fprintf(stderr, Failed to open codec for stream #%u\n, video_stream);return -1;}/* open the file to dump raw data *///output_file fopen(argv[3], w);v_draw.init(1280, 720, (int)frame_rate);v_draw.Start();/* actual decoding and dump the raw data */while (ret 0) {if ((ret av_read_frame(input_ctx, packet)) 0)break;if (video_stream packet.stream_index){//这里需要更加精确的计算ret decode_write(decoder_ctx, packet);//SDL_Delay(1000 / frame_rate);}av_packet_unref(packet);if (IsStop())break;}/* flush the decoder */packet.data NULL;packet.size 0;ret decode_write(decoder_ctx, packet);av_packet_unref(packet);avcodec_free_context(decoder_ctx);avformat_close_input(input_ctx);av_buffer_unref(hw_device_ctx);return 0;}public:void Run(){while (1){if(!IsStop())func_init(rtspurl);}}
};使用sdl来绘制画面
首先声明使用sdl来渲染画面并不是有多好如果可以可以使用新的绘制方式比如直接使用opengl直接使用vulkan来绘制是更好的sdl封装了opengld3d等绘制方式但是也失去了灵活性当然你从头到尾读遍了源码另当别论如果要使用均值处理函数来做消除百叶窗效果摩尔纹还是直接使用opengl 渲染更为简单使用glsl语言就行了甚至要使用非线性函数来增亮图形最好也是直接使用opengl来绘制。
#pragma once#includechrono
#define __STDC_CONSTANT_MACROS
#define SDL_MAIN_HANDLED
extern C
{
#include libavcodec/avcodec.h
#include libavformat/avformat.h
#include libavutil/pixdesc.h
#include libavutil/hwcontext.h
#include libavutil/opt.h
#include libavutil/avassert.h
#include libavutil/imgutils.h
#include libswscale/swscale.h
#include SDL2\SDL.h
};#include c_ringbuffer.h
#include TThreadRunable.h#include TYUVMerge.h#define SFM_REFRESH_EVENT (SDL_USEREVENT 1)
#define SFM_BREAK_EVENT (SDL_USEREVENT 2)#define WIDTH 640
#define HEIGHT 360typedef struct
{int g_fps 40;int thread_exit 0;int thread_pause 0;
}s_param;
static int sfp_refresh_thread(void *opaque)
{s_param * param (s_param*)opaque;param-thread_exit 0;param-thread_pause 0;while (!param-thread_exit){if (!param-thread_pause){SDL_Event event;event.type SFM_REFRESH_EVENT;SDL_PushEvent(event);}SDL_Delay(1000 / param-g_fps);}param-thread_exit 0;param-thread_pause 0;//BreakSDL_Event event;event.type SFM_BREAK_EVENT;SDL_PushEvent(event);return 0;
}class TicToc
{
public:TicToc(){tic();}void tic(){start std::chrono::system_clock::now();}double toc(){end std::chrono::system_clock::now();std::chrono::durationdouble elapsed_seconds end - start;return elapsed_seconds.count() * 1000;}private:std::chrono::time_pointstd::chrono::system_clock start, end;
};class SDLDraw :public TThreadRunable//实际上这里可以编码发送出去
{int m_w 0, m_h 0;SDL_Window *screen NULL;SDL_Renderer *sdlRenderer NULL;SDL_Texture *sdlTexture NULL;SDL_Rect sdlRect;SDL_Thread *video_tid;SDL_Event event;//struct SwsContext *img_convert_ctx NULL;bool m_window_init false;lfringqueueAVFrame, 20 v_frames;lfringqueueAVFrame, 20 v_frames_1;lfringqueueAVFrame, 20 v_frames_2;s_param v_param;//画布AVFrame * v_canvas_frame NULL;
public:void init(int w, int h,int fps){if (w ! m_w || h ! m_h){m_w w;m_h h;if (v_canvas_frame ! NULL){func_uninit();}}v_param.g_fps fps;//这是背景画布if (v_canvas_frame NULL){v_canvas_frame av_frame_alloc();av_image_alloc(v_canvas_frame-data, v_canvas_frame-linesize, 1280, 720, AV_PIX_FMT_YUV420P, 1);}}void func_uninit(){av_freep(v_canvas_frame-data[0]);av_frame_free(v_canvas_frame);}bool push(AVFrame * frame){//尝试三次插入return v_frames.enqueue(frame,3);}protected:int draw_init(/*HWND hWnd,*/ ){//m_w 1280;//m_h 720;if (m_window_init false){SDL_Init(SDL_INIT_VIDEO);SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, 1);screen SDL_CreateWindow(FF, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED,1920, 1000, SDL_WINDOW_SHOWN/* SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE*/);//screen SDL_CreateWindowFrom((void *)(hWnd));for (int i 0; i SDL_GetNumRenderDrivers(); i){SDL_RendererInfo rendererInfo {};SDL_GetRenderDriverInfo(i, rendererInfo);cout i rendererInfo.name endl;//if (rendererInfo.name ! std::string(direct3d11))//{// continue;//}}if (screen NULL){//printf(Window could not be created! SDL_Error: %s\n, SDL_GetError());return -1;}sdlRenderer SDL_CreateRenderer(screen,0, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);//sdlRenderer SDL_CreateRenderer(screen, -1, SDL_RENDERER_ACCELERATED);//SDL_SetHint(SDL_HINT_RENDER_DRIVER, opengl);m_window_init true;SDL_Thread *video_tid SDL_CreateThread(sfp_refresh_thread, NULL, v_param);}return 0;}void draw(uint8_t *data[], int linesize[]){if (sdlTexture ! NULL){SDL_DestroyTexture(sdlTexture);sdlTexture NULL;}//m_w w;//m_h h;if (sdlTexture NULL){sdlTexture SDL_CreateTexture(sdlRenderer, SDL_PIXELFORMAT_IYUV,SDL_TEXTUREACCESS_STREAMING, m_w, m_h);sdlRect.x 0;sdlRect.y 0;sdlRect.w m_w;sdlRect.h m_h;// nh;}SDL_UpdateYUVTexture(sdlTexture, sdlRect,data[0], linesize[0],data[1], linesize[1],data[2], linesize[2]);//SDL_RenderClear(sdlRenderer);SDL_RenderCopy(sdlRenderer, sdlTexture, NULL, NULL);SDL_RenderPresent(sdlRenderer);//video_tid SDL_CreateThread(sfp_refresh_thread, NULL, NULL);}
public:void Run(){draw_init();AVFrame * frame NULL;//while (!IsStop())//{// //解码播放 直接播放// if (v_frames.dequeue(frame))// {// //播放// draw(frame-data, frame-linesize,WIDTH,HEIGHT);// av_frame_free(frame);// SDL_Delay(10);// }// else// {// SDL_Delay(10);// }//}#if 1/*SDL_Thread *video_tid SDL_CreateThread(sfp_refresh_thread, NULL, v_param);*/int tick 0;double x 0.0f;for (;;){if (IsStop()){v_param.thread_exit 1;//break;}SDL_WaitEvent(event);if (event.type SFM_REFRESH_EVENT){if (v_frames.dequeue(frame)){tick;TicToc tt;//MergeYUV(v_canvas_frame-data[0], 1280, 720,// frame-data[0], WIDTH, HEIGHT, 1, 10, 10);MergeYUV_S(v_canvas_frame-data[0], 1280, 720,frame-data[0], WIDTH, HEIGHT, 10, 10);x tt.toc();//播放draw(v_canvas_frame-data, v_canvas_frame-linesize);av_freep(frame-data[0]);av_frame_free(frame);if (tick 10){tick 0;cout x endl;x 0.0f;}}}else if (event.type SDL_KEYDOWN){if (event.key.keysym.sym SDLK_SPACE)v_param.thread_pause !v_param.thread_pause;}else if (event.type SDL_QUIT){v_param.thread_exit 1;}else if (event.type SFM_BREAK_EVENT){break;}}
#endif}SDLDraw(){}~SDLDraw(){func_uninit();}};//SDL_FillRect(gScreenSurface, NULL, SDL_MapRGB(gScreenSurface-format, 0xFF, 0x00, 0x00));
ffmpeg 深度学习处理filter dnn_processing从2018年开始就已经是FFmpeg中的一个视频filter支持所有基于深度学习模型的图像处理算法即输入和输出都是AVFrame而处理过程使用的是深度学习模型。为什么要开发这样一个filter因为作为FFmpeg DNN模块的maintainerdnn_processing就是一个很好的使用者入手功能读ffmpeg的代码就可以知道其实自己写这些功能就行了至于支持视频分析功能的filter先不进行编码主要考虑如何支持异步建立流水线如何启用batch size从而最大化的用好系统的并行计算能力本来ffmpeg硬件解码后如何直接在gpu中进行swscale其实是不支持的这一部分要自己写代码来支持这是另外一回话我们先使用dnn_processing模块再说 这个模块可以完成针对灰度图的sobel算子的调用其输入输出的格式是grayf32。除了这个dnn_processing还可以完成sr超分辨率和derain去除雨点filter的功能下面使用ffmpeg命令演示对yuv和rgb格式的支持
./ffmpeg -i night.jpg -vf scaleiw*2:ih*2,formatyuv420p,dnn_processingdnn_backendtensorflow:model./srcnn.pb:inputx:outputy srcnn.jpg./ffmpeg -i small.jpg -vf formatyuv420p,scaleiw*2:ih*2,dnn_processingdnn_backendnative:model./srcnn.model:inputx:outputy -q:v 2 small.jpgsrcnn.jpg./ffmpeg -i small.jpg -vf formatyuv420p,dnn_processingdnn_backendnative:model./espcn.model:inputx:outputy small_b.jpg./ffmpeg -i rain.jpg -vf formatrgb24,dnn_processingdnn_backendnative:model./can.model:inputx:outputy derain.jpg下图可以看出放大两倍以后还是可以的不是很模糊其实是因为我们的训练模型很浅还没有好好做训练即使如此我仔细查看过线性差值比这个图像要差。
连贯执行
后面第二篇就要进行硬件解码到提升质量filter到输出编码了请等待第二篇
