Hello, in the last few days, I’ve been trying to find a way to decode h264 from appsrc, that uses frames which will be passed from the media of the webrtc crate.
I tried to test decoding raw h264 file was generated using ffmpeg with the following command:
ffmpeg -i video.mp4 -an -c:v libx264 -bsf:v h264_mp4toannexb -b:v 2M -max_delay 0 -bf 0 output.h264
Using the following pipeline to output video works:
gst-launch-1.0 filesrc location=output.h264 ! h264parse ! avdec_h264 ! videoconvert ! ximagesink
What I wrote is the following:
use gstreamer as gst;
use gst::prelude::*;
use std::fs::File;
use std::io::BufReader;
use webrtc::media::io::h264_reader::H264Reader;
fn main() {
gst::init().unwrap();
// declaring pipeline
let pipeline = gst::Pipeline::new(None);
let src = gst::ElementFactory::make("appsrc", None).unwrap();
let parse = gst::ElementFactory::make("h264parse", None).unwrap();
let decode = gst::ElementFactory::make("avdec_h264", None).unwrap();
let glup = gst::ElementFactory::make("videoconvert", None).unwrap();
let sink = gst::ElementFactory::make("ximagesink", None).unwrap();
// attaching pipeline elements
pipeline.add_many(&[&src, &parse, &decode, &glup, &sink]).unwrap();
gst::Element::link_many(&[&src, &parse, &decode, &glup, &sink]).unwrap();
let appsrc = src
.dynamic_cast::<gst_app::AppSrc>()
.expect("Source element is expected to be an appsrc!");
// Closure that feeds frames to pipeline
let mut i = 0;
let file = File::open("./output.h264").unwrap();
let reader = BufReader::new(file);
// A reader that groups by nal units
let mut nalreader = H264Reader::new(reader);
appsrc.set_callbacks(
gst_app::AppSrcCallbacks::builder()
.need_data(move |appsrc, _| {
println!("Producing frame {}", i);
match nalreader.next_nal() {
Ok(frame) => {
// giving the buffer the data without header
let buffer = gst::Buffer::from_slice(frame.data.freeze());
i += 1;
// appsrc already handles the error here
let _ = appsrc.push_buffer(buffer);
},
Err(err) => {
println!("All video frames parsed and sent: {}", err);
}
}
})
.build(),
);
pipeline.set_state(gst::State::Playing).unwrap();
let bus = pipeline
.bus()
.expect("Pipeline without bus. Shouldn't happen!");
for msg in bus.iter_timed(gst::ClockTime::NONE) {
use gst::MessageView;
match msg.view() {
MessageView::Eos(_) => break,
MessageView::Error(_) => break,
_ => (),
}
}
pipeline.set_state(gst::State::Null).unwrap();
}
I did also try to decode vp9 frames and it did work with the following code:
use gstreamer as gst;
use gst::prelude::*;
use webrtc::media::io::ivf_reader::IVFReader;
use std::fs::File;
use std::io::BufReader;
use webrtc::media::io::h264_reader::H264Reader;
fn main() {
gst::init().unwrap();
let pipeline = gst::Pipeline::new(None);
let src = gst::ElementFactory::make("appsrc", None).unwrap();
let parse = gst::ElementFactory::make("vp9parse", None).unwrap();
let decode = gst::ElementFactory::make("vp9dec", None).unwrap();
let glup = gst::ElementFactory::make("videoconvert", None).unwrap();
let sink = gst::ElementFactory::make("ximagesink", None).unwrap();
pipeline.add_many(&[&src, &parse, &decode, &glup, &sink]).unwrap();
gst::Element::link_many(&[&src, &parse, &decode, &glup, &sink]).unwrap();
let appsrc = src
.dynamic_cast::<gst_app::AppSrc>()
.expect("Source element is expected to be an appsrc!");
let mut i = 0;
let file = File::open("./output_vp9.ivf").unwrap();
let reader = BufReader::new(file);
let (mut ivf, header) = IVFReader::new(reader).unwrap();
appsrc.set_callbacks(
gst_app::AppSrcCallbacks::builder()
.need_data(move |appsrc, _| {
println!("Producing frame {}", i);
match ivf.parse_next_frame() {
Ok((frame, _)) => {
let buffer = gst::Buffer::from_slice(frame.freeze());
i += 1;
// appsrc already handles the error here
let _ = appsrc.push_buffer(buffer);
},
Err(err) => {
println!("All video frames parsed and sent: {}", err);
}
}
})
.build(),
);
pipeline.set_state(gst::State::Playing).unwrap();
let bus = pipeline
.bus()
.expect("Pipeline without bus. Shouldn't happen!");
for msg in bus.iter_timed(gst::ClockTime::NONE) {
use gst::MessageView;
match msg.view() {
MessageView::Eos(_) => break,
MessageView::Error(_) => break,
_ => (),
}
}
pipeline.set_state(gst::State::Null).unwrap();
}
