Use FFMPEG mux flv and send rtmp on IOS - ios

Question

I would like to use iphone camera & microphone to capture information pushed out through FFMPEG RTMP Streaming
The following Function capture information on IOS
- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
{
if (connection == videoCaptureConnection)
{
[manager264 encoderToH264:sampleBuffer];
}
else if (connection == audioCaptureConnection)
{
[manager264 encoderToMP3:sampleBuffer];
}
}
Initialization FFMPEG
- (int)setX264Resource
{
Global_Variables_VVV = (AppDelegate *)[[UIApplication sharedApplication] delegate];
avformat_network_init();
av_register_all();
pFormatCtx = avformat_alloc_context();
avformat_alloc_output_context2(&pFormatCtx, NULL, "flv", out_file);
fmt = pFormatCtx->oformat;
//Open output URL
if (avio_open(&pFormatCtx->pb, out_file, AVIO_FLAG_READ_WRITE) < 0)
{
printf("Failed to open output file! \n");
return -1;
}
/* Add the audio and video streams using the default format codecs
* and initialize the codecs. */
video_st = NULL;
audio_st = NULL;
if (fmt->video_codec != AV_CODEC_ID_NONE) {
video_st = add_stream(pFormatCtx, &pCodec, AV_CODEC_ID_H264);
}
if (fmt->audio_codec != AV_CODEC_ID_NONE) {
audio_st = add_stream(pFormatCtx, &aCodec, AV_CODEC_ID_MP3);
}
/* Now that all the parameters are set, we can open the audio and
* video codecs and allocate the necessary encode buffers. */
if (video_st)
[self open_video:pFormatCtx avcodec:pCodec avstream:video_st];
if (audio_st)
[self open_audio:pFormatCtx avcodec:aCodec avstream:audio_st];
// Show some Information
av_dump_format(pFormatCtx, 0, out_file, 1);
//Write File Header
avformat_write_header(pFormatCtx, NULL);
av_new_packet(&pkt, picture_size);
av_new_packet(&pkt2, picture_size);
AVCodecContext *c = video_st->codec;
y_size = c->width * c->height;
if (pFrame)
pFrame->pts = 0;
if(aFrame)
{
aFrame->pts = 0;
}
return 0;
}
static AVStream *add_stream(AVFormatContext *oc, AVCodec **codec, enum AVCodecID codec_id)
{
AVCodecContext *c;
AVStream *st;
/* find the encoder */
*codec = avcodec_find_encoder(codec_id);
if (!(*codec))
{
NSLog(#"Could not find encoder for '%s'\n",
avcodec_get_name(codec_id));
}
st = avformat_new_stream(oc, *codec);
if (!st)
{
NSLog(#"Could not allocate stream\n");
}
st->id = oc->nb_streams-1;
c = st->codec;
switch ((*codec)->type)
{
case AVMEDIA_TYPE_AUDIO:
c->codec_id = AV_CODEC_ID_MP3;
c->codec_type = AVMEDIA_TYPE_AUDIO;
c->channels = 1;
c->sample_fmt = AV_SAMPLE_FMT_S16P;
c->bit_rate = 128000;
c->sample_rate = 44100;
c->channel_layout = AV_CH_LAYOUT_MONO;
break;
case AVMEDIA_TYPE_VIDEO:
c->codec_id = AV_CODEC_ID_H264;
c->codec_type=AVMEDIA_TYPE_VIDEO;
/* Resolution must be a multiple of two. */
c->width = 720;
c->height = 1280;
/* timebase: This is the fundamental unit of time (in seconds) in terms
* of which frame timestamps are represented. For fixed-fps content,
* timebase should be 1/framerate and timestamp increments should be
* identical to 1. */
c->time_base.den = 30;
c->time_base.num = 1;
c->gop_size = 15; /* emit one intra frame every twelve frames at most */
c->pix_fmt = PIX_FMT_YUV420P;
c->max_b_frames = 0;
c->bit_rate = 3000000;
c->qmin = 10;
c->qmax = 51;
break;
default:
break;
}
/* Some formats want stream headers to be separate. */
if (oc->oformat->flags & AVFMT_GLOBALHEADER)
c->flags |= CODEC_FLAG_GLOBAL_HEADER;
return st;
}
SampleBuffer will turn into H264 and pushed out RTMP Streaming
- (void)encoderToH264:(CMSampleBufferRef)sampleBuffer
{
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (CVPixelBufferLockBaseAddress(imageBuffer, 0) == kCVReturnSuccess)
{
UInt8 *bufferbasePtr = (UInt8 *)CVPixelBufferGetBaseAddress(imageBuffer);
UInt8 *bufferPtr = (UInt8 *)CVPixelBufferGetBaseAddressOfPlane(imageBuffer,0);
UInt8 *bufferPtr1 = (UInt8 *)CVPixelBufferGetBaseAddressOfPlane(imageBuffer,1);
size_t buffeSize = CVPixelBufferGetDataSize(imageBuffer);
size_t width = CVPixelBufferGetWidth(imageBuffer);
size_t height = CVPixelBufferGetHeight(imageBuffer);
size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
size_t bytesrow0 = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer,0);
size_t bytesrow1 = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer,1);
size_t bytesrow2 = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer,2);
UInt8 *yuv420_data = (UInt8 *)malloc(width * height *3/ 2); // buffer to store YUV with layout YYYYYYYYUUVV
/* convert NV12 data to YUV420*/
UInt8 *pY = bufferPtr ;
UInt8 *pUV = bufferPtr1;
UInt8 *pU = yuv420_data + width*height;
UInt8 *pV = pU + width*height/4;
for(int i =0;i<height;i++)
{
memcpy(yuv420_data+i*width,pY+i*bytesrow0,width);
}
for(int j = 0;j<height/2;j++)
{
for(int i =0;i<width/2;i++)
{
*(pU++) = pUV[i<<1];
*(pV++) = pUV[(i<<1) + 1];
}
pUV+=bytesrow1;
}
//Read raw YUV data
picture_buf = yuv420_data;
pFrame->data[0] = picture_buf; // Y
pFrame->data[1] = picture_buf+ y_size; // U
pFrame->data[2] = picture_buf+ y_size*5/4; // V
int got_picture = 0;
// Encode
pFrame->width = 720;
pFrame->height = 1280;
pFrame->format = PIX_FMT_YUV420P;
AVCodecContext *c = video_st->codec;
int ret = avcodec_encode_video2(c, &pkt, pFrame, &got_picture);
if(ret < 0)
{
printf("Failed to encode! \n");
}
if (got_picture==1)
{
/* Compute current audio and video time. */
video_time = video_st ? video_st->pts.val * av_q2d(video_st->time_base) : 0.0;
pFrame->pts += av_rescale_q(1, video_st->codec->time_base, video_st->time_base);
if(pkt.size != 0)
{
printf("Succeed to encode frame: %5lld\tsize:%5d\n", pFrame->pts, pkt.size);
pkt.stream_index = video_st->index;
ret = av_write_frame(pFormatCtx, &pkt);
av_free_packet(&pkt);
}
}
free(yuv420_data);
}
CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
}
SampleBuffer will turn into MP3 and pushed out RTMP Streaming
-(void)encoderToMP3:(CMSampleBufferRef)sampleBuffer
{
CMSampleTimingInfo timing_info;
CMSampleBufferGetSampleTimingInfo(sampleBuffer, 0, &timing_info);
double pts=0;
double dts=0;
AVCodecContext *c;
int got_packet, ret;
c = audio_st->codec;
CMItemCount numSamples = CMSampleBufferGetNumSamples(sampleBuffer);
NSUInteger channelIndex = 0;
CMBlockBufferRef audioBlockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
size_t audioBlockBufferOffset = (channelIndex * numSamples * sizeof(SInt16));
size_t lengthAtOffset = 0;
size_t totalLength = 0;
SInt16 *samples = NULL;
CMBlockBufferGetDataPointer(audioBlockBuffer, audioBlockBufferOffset, &lengthAtOffset, &totalLength, (char **)(&samples));
const AudioStreamBasicDescription *audioDescription = CMAudioFormatDescriptionGetStreamBasicDescription(CMSampleBufferGetFormatDescription(sampleBuffer));
SwrContext *swr = swr_alloc();
int in_smprt = (int)audioDescription->mSampleRate;
av_opt_set_int(swr, "in_channel_layout", AV_CH_LAYOUT_MONO, 0);
av_opt_set_int(swr, "out_channel_layout", audio_st->codec->channel_layout, 0);
av_opt_set_int(swr, "in_channel_count", audioDescription->mChannelsPerFrame, 0);
av_opt_set_int(swr, "out_channel_count", 1, 0);
av_opt_set_int(swr, "out_channel_layout", audio_st->codec->channel_layout, 0);
av_opt_set_int(swr, "in_sample_rate", audioDescription->mSampleRate,0);
av_opt_set_int(swr, "out_sample_rate", audio_st->codec->sample_rate,0);
av_opt_set_sample_fmt(swr, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0);
av_opt_set_sample_fmt(swr, "out_sample_fmt", audio_st->codec->sample_fmt, 0);
swr_init(swr);
uint8_t **input = NULL;
int src_linesize;
int in_samples = (int)numSamples;
ret = av_samples_alloc_array_and_samples(&input, &src_linesize, audioDescription->mChannelsPerFrame, in_samples, AV_SAMPLE_FMT_S16P, 0);
*input=(uint8_t*)samples;
uint8_t *output=NULL;
int out_samples = av_rescale_rnd(swr_get_delay(swr, in_smprt) +in_samples, (int)audio_st->codec->sample_rate, in_smprt, AV_ROUND_UP);
av_samples_alloc(&output, NULL, audio_st->codec->channels, out_samples, audio_st->codec->sample_fmt, 0);
in_samples = (int)numSamples;
out_samples = swr_convert(swr, &output, out_samples, (const uint8_t **)input, in_samples);
aFrame->nb_samples =(int) out_samples;
ret = avcodec_fill_audio_frame(aFrame, audio_st->codec->channels, audio_st->codec->sample_fmt,
(uint8_t *)output,
(int) out_samples *
av_get_bytes_per_sample(audio_st->codec->sample_fmt) *
audio_st->codec->channels, 1);
if (ret < 0)
{
fprintf(stderr, "Error fill audio frame: %s\n", av_err2str(ret));
}
aFrame->channel_layout = audio_st->codec->channel_layout;
aFrame->channels=audio_st->codec->channels;
aFrame->sample_rate= audio_st->codec->sample_rate;
if (timing_info.presentationTimeStamp.timescale!=0)
pts=(double) timing_info.presentationTimeStamp.value/timing_info.presentationTimeStamp.timescale;
aFrame->pts = pts*audio_st->time_base.den;
aFrame->pts = av_rescale_q(aFrame->pts, audio_st->time_base, audio_st->codec->time_base);
ret = avcodec_encode_audio2(c, &pkt2, aFrame, &got_packet);
if (ret < 0)
{
fprintf(stderr, "Error encoding audio frame: %s\n", av_err2str(ret));
}
swr_free(&swr);
if (got_packet)
{
pkt2.stream_index = audio_st->index;
// Write the compressed frame to the media file.
ret = av_interleaved_write_frame(pFormatCtx, &pkt2);
if (ret != 0)
{
fprintf(stderr, "Error while writing audio frame: %s\n", av_err2str(ret));
av_free_packet(&pkt2);
}
}
}
Soon "Broken pipe" problem occurs after execution.
PTS is currently feeling is not adjusted, but do not know how to adjust the PTS.
2016-03-09 16:57:41.058 PoliceCamPlayer[1004:193465] recordVideo....
[libx264 # 0x12f8b6e00] using cpu capabilities: ARMv8 NEON
[libx264 # 0x12f8b6e00] profile Constrained Baseline, level 3.1
[libx264 # 0x12f8b6e00] 264 - core 148 - H.264/MPEG-4 AVC codec - Copyleft 2003-2016 - http://www.videolan.org/x264.html - options: cabac=0 ref=1 deblock=1:0:0 analyse=0x1:0x111 me=hex subme=2 psy=1 psy_rd=1.00:0.00 mixed_ref=0 me_range=16 chroma_me=1 trellis=0 8x8dct=0 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=0 threads=2 lookahead_threads=2 sliced_threads=1 slices=2 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=0 weightp=0 keyint=15 keyint_min=1 scenecut=40 intra_refresh=0 rc=abr mbtree=0 bitrate=3000 ratetol=1.0 qcomp=0.60 qpmin=25 qpmax=51 qpstep=4 ip_ratio=1.40 aq=1:1.00
Output #0, flv, to 'rtmp://XXX.XX.XXX.XX/myapp/jackal':
Stream #0:0: Video: h264 (libx264), yuv420p, 720x1280, q=25-51, 3000 kb/s, 23 tbc
Stream #0:1: Audio: mp3 (libmp3lame), 44100 Hz, mono, s16p, 64 kb/s
[flv # 0x12f8b5400] Using AVStream.codec.time_base as a timebase hint to the muxer is deprecated. Set AVStream.time_base instead.
[flv # 0x12f8b5400] Using AVStream.codec.time_base as a timebase hint to the muxer is deprecated. Set AVStream.time_base instead.
[libx264 # 0x12f8b6e00] Provided packet is too small, needs to be 33468
Failed to encode!
Audio_pts:4154432515 pts_time:4.15443e+06 dts:4154432515 dts_time:4.15443e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:43 pts_time:0.043 dts:43 dts_time:0.043 duration:0 duration_time:0 stream_index:0
Audio_pts:4154433667 pts_time:4.15443e+06 dts:4154433667 dts_time:4.15443e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154434854 pts_time:4.15443e+06 dts:4154434854 dts_time:4.15443e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:86 pts_time:0.086 dts:86 dts_time:0.086 duration:0 duration_time:0 stream_index:0
Audio_pts:4154435996 pts_time:4.15444e+06 dts:4154435996 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154437138 pts_time:4.15444e+06 dts:4154437138 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:129 pts_time:0.129 dts:129 dts_time:0.129 duration:0 duration_time:0 stream_index:0
Audio_pts:4154438281 pts_time:4.15444e+06 dts:4154438281 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:172 pts_time:0.172 dts:172 dts_time:0.172 duration:0 duration_time:0 stream_index:0
Audio_pts:4154439467 pts_time:4.15444e+06 dts:4154439467 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:215 pts_time:0.215 dts:215 dts_time:0.215 duration:0 duration_time:0 stream_index:0
Audio_pts:4154440609 pts_time:4.15444e+06 dts:4154440609 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154441752 pts_time:4.15444e+06 dts:4154441752 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:258 pts_time:0.258 dts:258 dts_time:0.258 duration:0 duration_time:0 stream_index:0
Audio_pts:4154442884 pts_time:4.15444e+06 dts:4154442884 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154444071 pts_time:4.15444e+06 dts:4154444071 dts_time:4.15444e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:301 pts_time:0.301 dts:301 dts_time:0.301 duration:0 duration_time:0 stream_index:0
Audio_pts:4154445213 pts_time:4.15445e+06 dts:4154445213 dts_time:4.15445e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154446355 pts_time:4.15445e+06 dts:4154446355 dts_time:4.15445e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:344 pts_time:0.344 dts:344 dts_time:0.344 duration:0 duration_time:0 stream_index:0
Audio_pts:4154447498 pts_time:4.15445e+06 dts:4154447498 dts_time:4.15445e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:387 pts_time:0.387 dts:387 dts_time:0.387 duration:0 duration_time:0 stream_index:0
Audio_pts:4154448640 pts_time:4.15445e+06 dts:4154448640 dts_time:4.15445e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154449826 pts_time:4.15445e+06 dts:4154449826 dts_time:4.15445e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:430 pts_time:0.43 dts:430 dts_time:0.43 duration:0 duration_time:0 stream_index:0
Audio_pts:4154450969 pts_time:4.15445e+06 dts:4154450969 dts_time:4.15445e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154452101 pts_time:4.15445e+06 dts:4154452101 dts_time:4.15445e+06 duration:1152 duration_time:1.152 stream_index:1
...................
...................
...................
Video_pts:4343 pts_time:4.343 dts:4343 dts_time:4.343 duration:0 duration_time:0 stream_index:0
Audio_pts:4154622619 pts_time:4.15462e+06 dts:4154622619 dts_time:4.15462e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:4386 pts_time:4.386 dts:4386 dts_time:4.386 duration:0 duration_time:0 stream_index:0
Audio_pts:4154623761 pts_time:4.15462e+06 dts:4154623761 dts_time:4.15462e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154624903 pts_time:4.15462e+06 dts:4154624903 dts_time:4.15462e+06 duration:1152 duration_time:1.152 stream_index:1
Audio_pts:4154626090 pts_time:4.15463e+06 dts:4154626090 dts_time:4.15463e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:4429 pts_time:4.429 dts:4429 dts_time:4.429 duration:0 duration_time:0 stream_index:0
Audio_pts:4154627222 pts_time:4.15463e+06 dts:4154627222 dts_time:4.15463e+06 duration:1152 duration_time:1.152 stream_index:1
Video_pts:4472 pts_time:4.472 dts:4472 dts_time:4.472 duration:0 duration_time:0 stream_index:0
Error while writing audio frame: Broken pipe
Audio_pts:4154628365 pts_time:4.15463e+06 dts:4154628365 dts_time:4.15463e+06 duration:1152 duration_time:1.152 stream_index:1
Error while writing audio frame: Broken pipe
Audio_pts:4154629507 pts_time:4.15463e+06 dts:4154629507 dts_time:4.15463e+06 duration:1152 duration_time:1.152 stream_index:1
Error while writing audio frame: Broken pipe
Audio_pts:4154630693 pts_time:4.15463e+06 dts:4154630693 dts_time:4.15463e+06 duration:1152 duration_time:1.152 stream_index:1
Error while writing audio frame: Broken pipe
Audio_pts:4154631836 pts_time:4.15463e+06 dts:4154631836 dts_time:4.15463e+06 duration:1152 duration_time:1.152 stream_index:1
Error while writing audio frame: Broken pipe
Audio_pts:4154632978 pts_time:4.15463e+06 dts:4154632978 dts_time:4.15463e+06 duration:1152 duration_time:1.152 stream_index:1
.......................
.......................
.......................
2016-03-09 16:57:49.345 PoliceCamPlayer[1004:193465] stopRecord!!!
Video_pts:7783 pts_time:7.783 dts:7783 dts_time:7.783 duration:0 duration_time:0 stream_index:0
[flv # 0x12f8b5400] Failed to update header with correct duration.
[flv # 0x12f8b5400] Failed to update header with correct filesize.
[libx264 # 0x12f8b6e00] frame I:28 Avg QP:25.36 size: 24181
[libx264 # 0x12f8b6e00] frame P:154 Avg QP:25.34 size: 6603
[libx264 # 0x12f8b6e00] mb I I16..4: 80.9% 0.0% 19.1%
[libx264 # 0x12f8b6e00] mb P I16..4: 5.9% 0.0% 0.2% P16..4: 28.2% 4.4% 1.0% 0.0% 0.0% skip:60.2%
[libx264 # 0x12f8b6e00] final ratefactor: 16.70
[libx264 # 0x12f8b6e00] coded y,uvDC,uvAC intra: 35.8% 9.3% 0.4% inter: 8.8% 1.6% 0.0%
[libx264 # 0x12f8b6e00] i16 v,h,dc,p: 28% 26% 26% 21%
[libx264 # 0x12f8b6e00] i4 v,h,dc,ddl,ddr,vr,hd,vl,hu: 13% 26% 25% 3% 7% 4% 5% 3% 13%
[libx264 # 0x12f8b6e00] i8c dc,h,v,p: 85% 9% 5% 0%
[libx264 # 0x12f8b6e00] kb/s:1712.63

Answer 1

Increase PTS by amount of samples that you are sending to encoder.
Also, don't forget to rescale timings from your audio stream to output format context.
So, the fixes are:
audioFrame->pts = audioSamplesCounter; // starting from zero
Then after encoding (avcodec_encode_audio2) increase counter by amount of samples in frame that you have sent to encoder (in your case this will be not the amount you've got from CMSampleBuffer, but the one after resampling by SWR - "out_samples"):
audioSamplesCounter += audioFrame->nb_samples;
And right before writing to an media output file rescale timings:
av_packet_rescale_ts(&audioPacket,
audioStream->codec->time_base,
outputFormatContext->streams[audioStream->index]->time_base);
Also, I'd like recommend you to optimize approaches you using device's resources.
Create contexts for rescaling/resampling once and reuse them.
Allocate buffers for audio and video as soon as your stream starts or when first CMSampleBufferRef arrives. The size will not change until restarting stream/session. It will be huge improvement for performance & memory consumption.
Use hardware acceleration when it's possible.
Do not forget to free any allocated arrays & contexts.
Hope it helps you :)