javacpp-ffmpeg系列:
javacpp-FFmpeg系列之3： 图像数据转换（BGR与BufferdImage互转，RGB与BufferdImage互转）
前言：
第一篇中视频解码成YUVJ420P图像像素数据（以下简称YUV或YUV数据），只是YUV在流媒体协议中用的较多（数据少，节省流量带宽），在图像处理应用较多的是BGR和RGB像素数据。我们已经获取到了YUV数据，那么把YUV转成BGR或者RGB需要再进行一次转换，显然性能上的表现并不是很好，所以本篇会通过编写一个通用转换器来介绍如何使用ffmpeg解码转出BGR、RGB、YUV等像素数据。
补充：
（1）为什么暂时没有用python实现，主要是先熟悉ffmpeg的API，后面会出的
（2）为什么要转成BGR、RGB像素数据，因为有了这俩其中一个就可以直接生成java的BufferImage了啊，最重要的是我们本意不是要转成BufferImage，而是直接编码成base64的图像数据啊
（3）演示demo见下一章
一、功能设计
第一篇写的很简略（实际上是那一大坨代码，自己实在看不下去了qaq），直接参考ffmpeg原生C的API，不符合java语言编写习惯，所以本篇会对上篇代码进行一些简单的封装复用。
功能上，会支持多种格式的像素数据(BGR、RGB、YUV等等)；代码上，会对各个流程进行阐述分析。
二、功能实现
（1）初始化
加载ffmpeg的网络库和编解码库，不初始化就没法用，适合放在静态块中进行加载
static {
// Register all formats and codecs av_register_all(); avformat_network_init(); }（2）打开视频流
初始化AVFormatContext，主要就是根据url创建InputStream，并且会根据不同协议（rtmp/rtsp/hls/文件等）尝试读取一些文件头数据（流媒体头数据）。
补充：FileNotOpenException是继承自RuntimeException的自定义异常类，只是加个名字方便标识异常而已，下面还会有几个异常，都是继承自RuntimeException的自定义异常类，以下不会再提
/**
* 打开视频流 * @param url -url * @return * @throws FileNotOpenException */ protected AVFormatContext openInput(String url) throws FileNotOpenException{ AVFormatContext pFormatCtx = new AVFormatContext(null); if(avformat_open_input(pFormatCtx, url, null, null)==0) { return pFormatCtx; } throw new FileNotOpenException("Didn't open video file"); }
（3）检索流信息
上一步获得了AVFormatContext，这一步继续根据AVFormatContext读取一部分视音频数据并且获得一些相关的信息
/**
* 检索流信息 * @param pFormatCtx * @return */ protected AVFormatContext findStreamInfo(AVFormatContext pFormatCtx) throws StreamInfoNotFoundException{ if (avformat_find_stream_info(pFormatCtx, (PointerPointer<?>) null)>= 0) { return pFormatCtx; } throw new StreamInfoNotFoundException("Didn't retrieve stream information"); }
（3）获取视频帧
上面两步确定了媒体文件/流的上下文，这一步尝试读取一帧视频帧。
分成两个方法，先获取视频帧位置，然后根据位置获取视频帧，当然也可以合成一个方法使用。
/**
* 获取第一帧视频位置 * @param pFormatCtx * @return */ protected int findVideoStreamIndex(AVFormatContext pFormatCtx) { int i = 0, videoStream = -1; for (i = 0; i < pFormatCtx.nb_streams(); i++) { AVStream stream=pFormatCtx.streams(i); AVCodecContext codec=stream.codec(); if (codec.codec_type() == AVMEDIA_TYPE_VIDEO) { videoStream = i; break; } } return videoStream; }
/**
* 指定视频帧位置获取对应视频帧 * @param pFormatCtx * @param videoStream * @return */ protected AVCodecContext findVideoStream(AVFormatContext pFormatCtx ,int videoStream)throws StreamNotFoundException { if(videoStream >=0) { // Get a pointer to the codec context for the video stream AVStream stream=pFormatCtx.streams(videoStream); AVCodecContext pCodecCtx = stream.codec(); return pCodecCtx; } throw new StreamNotFoundException("Didn't open video file"); }
（4）查找编解码器
其实底层调用的还是find_encdec()，遍历AVCodec链表查找有没有对应的编解码器，如果没有找到直接抛出异常，如果已经确定编解码，也可以指定codec_id
/**
* 查找并尝试打开解码器 * @return */ protected AVCodecContext findAndOpenCodec(AVCodecContext pCodecCtx) { // Find the decoder for the video stream AVCodec pCodec = avcodec_find_decoder(pCodecCtx.codec_id()); if (pCodec == null) { System.err.println("Codec not found"); throw new CodecNotFoundExpception("Codec not found"); } AVDictionary optionsDict = null; // Open codec if (avcodec_open2(pCodecCtx, pCodec, optionsDict) < 0) { System.err.println("Could not open codec"); throw new CodecNotFoundExpception("Could not open codec"); // Could not open codec } return pCodecCtx; }
（5.1）循环读取视频帧并解码成yuv
这个没什么好讲的了，前面的准备任务做完，就是一直循环读取视频帧，最后解码出来的图像帧都是yuv像素数据，这个显然不是我们想要的，所以要对这里进行改动
// Allocate video frame
AVFrame pFrame = av_frame_alloc();AVPacket packet = new AVPacket();
// Read frames and save first five frames to disk
while (av_read_frame(pFormatCtx, packet) >= 0) { // Is this a packet from the video stream? if (packet.stream_index() == videoStream) { // Decode video frame avcodec_decode_video2(pCodecCtx, pFrame, frameFinished, packet); // Did we get a video frame? if (frameFinished != null&&frameFinished[0] != 0) {//ffmpeg默认解码出来的是yuv数据
System.err.println(pFrame.data()); } } // Free the packet that was allocated by av_read_frame av_free_packet(packet); }
（5.2）循环读取视频帧并转换成RGB或BGR图像像素数据
// Allocate video frame
AVFrame pFrame = av_frame_alloc(); //Allocate an AVFrame structure AVFrame pFrameRGB = av_frame_alloc();width = pCodecCtx.width();
height = pCodecCtx.height(); pFrameRGB.width(width); pFrameRGB.height(height); pFrameRGB.format(fmt);// Determine required buffer size and allocate buffer
int numBytes = avpicture_get_size(fmt, width, height);SwsContext sws_ctx = sws_getContext(width, height, pCodecCtx.pix_fmt(), width, height,fmt, SWS_BILINEAR, null, null, (DoublePointer) null);
BytePointer buffer = new BytePointer(av_malloc(numBytes));
// Assign appropriate parts of buffer to image planes in pFrameRGB // Note that pFrameRGB is an AVFrame, but AVFrame is a superset // of AVPicture avpicture_fill(new AVPicture(pFrameRGB), buffer, fmt, width, height); AVPacket packet = new AVPacket(); int[] frameFinished = new int[1]; // Read frames and save first five frames to disk while (av_read_frame(pFormatCtx, packet) >= 0) { // Is this a packet from the video stream? if (packet.stream_index() == videoStream) { // Decode video frame avcodec_decode_video2(pCodecCtx, pFrame, frameFinished, packet); // Did we get a video frame? if (frameFinished != null&&frameFinished[0] != 0) { // Convert the image from its native format to BGR sws_scale(sws_ctx, pFrame.data(), pFrame.linesize(), 0, height, pFrameRGB.data(),pFrameRGB.linesize()); //Convert BGR to ByteBuffer//保存RGB或BGR数据
return saveFrame(pFrameRGB, width, height); } } // Free the packet that was allocated by av_read_frame av_free_packet(packet); }
三、完整代码
package cc.eguid.cv.corelib.videoimageshot.grabber;
import static org.bytedeco.javacpp.avcodec.*;
import static org.bytedeco.javacpp.avformat.*; import static org.bytedeco.javacpp.avutil.*; import static org.bytedeco.javacpp.swscale.*; import java.io.IOException; import java.nio.ByteBuffer;import org.bytedeco.javacpp.BytePointer;
import org.bytedeco.javacpp.DoublePointer; import org.bytedeco.javacpp.PointerPointer;import cc.eguid.cv.corelib.videoimageshot.exception.CodecNotFoundExpception;
import cc.eguid.cv.corelib.videoimageshot.exception.FileNotOpenException; import cc.eguid.cv.corelib.videoimageshot.exception.StreamInfoNotFoundException; import cc.eguid.cv.corelib.videoimageshot.exception.StreamNotFoundException;public abstract class GrabberTmplate {
static {
// Register all formats and codecs av_register_all(); avformat_network_init(); } //保留，暂不用 protected int width;//宽度 protected int height;//高度 /** * 打开视频流 * @param url -url * @return * @throws FileNotOpenException */ protected AVFormatContext openInput(String url) throws FileNotOpenException{ AVFormatContext pFormatCtx = new AVFormatContext(null); if(avformat_open_input(pFormatCtx, url, null, null)==0) { return pFormatCtx; } throw new FileNotOpenException("Didn't open video file"); } /** * 检索流信息 * @param pFormatCtx * @return */ protected AVFormatContext findStreamInfo(AVFormatContext pFormatCtx) throws StreamInfoNotFoundException{ if (avformat_find_stream_info(pFormatCtx, (PointerPointer<?>) null)>= 0) { return pFormatCtx; } throw new StreamInfoNotFoundException("Didn't retrieve stream information"); } /** * 获取第一帧视频位置 * @param pFormatCtx * @return */ protected int findVideoStreamIndex(AVFormatContext pFormatCtx) { int i = 0, videoStream = -1; for (i = 0; i < pFormatCtx.nb_streams(); i++) { AVStream stream=pFormatCtx.streams(i); AVCodecContext codec=stream.codec(); if (codec.codec_type() == AVMEDIA_TYPE_VIDEO) { videoStream = i; break; } } return videoStream; } /** * 指定视频帧位置获取对应视频帧 * @param pFormatCtx * @param videoStream * @return */ protected AVCodecContext findVideoStream(AVFormatContext pFormatCtx ,int videoStream)throws StreamNotFoundException { if(videoStream >=0) { // Get a pointer to the codec context for the video stream AVStream stream=pFormatCtx.streams(videoStream); AVCodecContext pCodecCtx = stream.codec(); return pCodecCtx; } throw new StreamNotFoundException("Didn't open video file"); } /** * 查找并尝试打开解码器 * @return */ protected AVCodecContext findAndOpenCodec(AVCodecContext pCodecCtx) { // Find the decoder for the video stream AVCodec pCodec = avcodec_find_decoder(pCodecCtx.codec_id()); if (pCodec == null) { System.err.println("Codec not found"); throw new CodecNotFoundExpception("Codec not found"); } AVDictionary optionsDict = null; // Open codec if (avcodec_open2(pCodecCtx, pCodec, optionsDict) < 0) { System.err.println("Could not open codec"); throw new CodecNotFoundExpception("Could not open codec"); // Could not open codec } return pCodecCtx; }/**
* 抓取视频帧（默认跳过音频帧和空帧） * @param url -视频源（rtsp/rtmp/hls/文件等等） * @param fmt - 像素格式，比如AV_PIX_FMT_BGR24 * @return * @throws IOException */ public ByteBuffer grabVideoFrame(String url,int fmt) throws IOException { // Open video file AVFormatContext pFormatCtx=openInput(url);// Retrieve stream information
pFormatCtx=findStreamInfo(pFormatCtx);// Dump information about file onto standard error
//av_dump_format(pFormatCtx, 0, url, 0);//Find a video stream
int videoStream=findVideoStreamIndex(pFormatCtx); AVCodecContext pCodecCtx =findVideoStream(pFormatCtx,videoStream); // Find the decoder for the video stream pCodecCtx= findAndOpenCodec(pCodecCtx);// Allocate video frame
AVFrame pFrame = av_frame_alloc(); //Allocate an AVFrame structure AVFrame pFrameRGB = av_frame_alloc();width = pCodecCtx.width();
height = pCodecCtx.height(); pFrameRGB.width(width); pFrameRGB.height(height); pFrameRGB.format(fmt);// Determine required buffer size and allocate buffer
int numBytes = avpicture_get_size(fmt, width, height);SwsContext sws_ctx = sws_getContext(width, height, pCodecCtx.pix_fmt(), width, height,fmt, SWS_BILINEAR, null, null, (DoublePointer) null);
BytePointer buffer = new BytePointer(av_malloc(numBytes));
// Assign appropriate parts of buffer to image planes in pFrameRGB // Note that pFrameRGB is an AVFrame, but AVFrame is a superset // of AVPicture avpicture_fill(new AVPicture(pFrameRGB), buffer, fmt, width, height); AVPacket packet = new AVPacket(); int[] frameFinished = new int[1]; try { // Read frames and save first five frames to disk while (av_read_frame(pFormatCtx, packet) >= 0) { // Is this a packet from the video stream? if (packet.stream_index() == videoStream) { // Decode video frame avcodec_decode_video2(pCodecCtx, pFrame, frameFinished, packet); // Did we get a video frame? if (frameFinished != null&&frameFinished[0] != 0) { // Convert the image from its native format to BGR sws_scale(sws_ctx, pFrame.data(), pFrame.linesize(), 0, height, pFrameRGB.data(),pFrameRGB.linesize()); //Convert BGR to ByteBuffer return saveFrame(pFrameRGB, width, height); } } // Free the packet that was allocated by av_read_frame av_free_packet(packet); } return null; }finally { //Don't free buffer // av_free(buffer); av_free(pFrameRGB);// Free the RGB image av_free(pFrame);// Free the YUV frame sws_freeContext(sws_ctx);//Free SwsContext avcodec_close(pCodecCtx);// Close the codec avformat_close_input(pFormatCtx);// Close the video file } } /** * BGR图像帧转字节缓冲区（BGR结构） * * @param pFrame * -bgr图像帧 * @param width * -宽度 * @param height * -高度 * @return * @throws IOException */ abstract ByteBuffer saveFrame(AVFrame pFrameRGB, int width, int height); }
四、小结
本章主要详解ffmpeg拉流解码的各个流程，可以通过本章代码可以轻松实现不限于RGB/BGR/YUV的FFmpeg所支持的多种像素格式转换