Skip to content

Commit

Permalink
Re #1762: Add I420/IYUV output format to capture devices.
Browse files Browse the repository at this point in the history 
git-svn-id: http://svn.pjsip.org/repos/pjproject/trunk@4836 74dad513-b988-da41-8d7b-12977e46ad98
  • Loading branch information
nanang committed May 6, 2014
1 parent 13eb6bf commit b5101f4
Showing 1 changed file with 77 additions and 36 deletions.
113 changes: 77 additions & 36 deletions pjmedia/src/pjmedia-videodev/ios_dev.m
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@

static ios_fmt_info ios_fmts[] =
{
{ PJMEDIA_FORMAT_BGRA, kCVPixelFormatType_32BGRA }
{ PJMEDIA_FORMAT_BGRA, kCVPixelFormatType_32BGRA },
{ PJMEDIA_FORMAT_I420, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange }
};

/* qt device info */
Expand Down Expand Up @@ -86,14 +87,15 @@ @interface VOutDelegate: NSObject
void *user_data; /**< Application data */

pjmedia_rect_size size;
pj_uint8_t bpp;
unsigned bytes_per_row;
unsigned frame_size;
unsigned frame_size; /**< Frame size (bytes)*/
pj_bool_t is_planar;

AVCaptureSession *cap_session;
AVCaptureDeviceInput *dev_input;
AVCaptureVideoDataOutput *video_output;
VOutDelegate *vout_delegate;
void *capture_buf;

void *render_buf;
pj_size_t render_buf_size;
Expand Down Expand Up @@ -209,7 +211,6 @@ static pj_status_t ios_factory_init(pjmedia_vid_dev_factory *f)
pj_ansi_strncpy(qdi->info.driver, "iOS", sizeof(qdi->info.driver));
qdi->info.dir = PJMEDIA_DIR_RENDER;
qdi->info.has_callback = PJ_FALSE;
qdi->info.caps = PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW;

/* Init input device */
first_idx = qf->dev_count;
Expand All @@ -229,14 +230,13 @@ static pj_status_t ios_factory_init(pjmedia_vid_dev_factory *f)

qdi = &qf->dev_info[qf->dev_count++];
pj_bzero(qdi, sizeof(*qdi));
pj_ansi_strncpy(qdi->info.name, [[device localizedName] UTF8String],
pj_ansi_strncpy(qdi->info.name, [device.localizedName UTF8String],
sizeof(qdi->info.name));
pj_ansi_strncpy(qdi->info.driver, "iOS", sizeof(qdi->info.driver));
qdi->info.dir = PJMEDIA_DIR_CAPTURE;
qdi->info.has_callback = PJ_TRUE;
qdi->info.caps = PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW |
PJMEDIA_VID_DEV_CAP_SWITCH |
PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW;
PJMEDIA_VID_DEV_CAP_SWITCH;
qdi->dev = device;
}
}
Expand All @@ -251,15 +251,24 @@ static pj_status_t ios_factory_init(pjmedia_vid_dev_factory *f)
/* Set supported formats */
for (i = 0; i < qf->dev_count; i++) {
qdi = &qf->dev_info[i];
qdi->info.fmt_cnt = PJ_ARRAY_SIZE(ios_fmts);
qdi->info.caps |= PJMEDIA_VID_DEV_CAP_FORMAT |
PJMEDIA_VID_DEV_CAP_OUTPUT_WINDOW |
PJMEDIA_VID_DEV_CAP_OUTPUT_RESIZE |
PJMEDIA_VID_DEV_CAP_OUTPUT_POSITION |
PJMEDIA_VID_DEV_CAP_OUTPUT_HIDE |
PJMEDIA_VID_DEV_CAP_ORIENTATION;

for (l = 0; l < PJ_ARRAY_SIZE(ios_fmts); l++) {
pjmedia_format *fmt = &qdi->info.fmt[l];
pjmedia_format *fmt;

/* Simple renderer UIView only supports BGRA */
if (qdi->info.dir == PJMEDIA_DIR_RENDER &&
ios_fmts[l].pjmedia_format != PJMEDIA_FORMAT_BGRA)
{
continue;
}

fmt = &qdi->info.fmt[qdi->info.fmt_cnt++];
pjmedia_format_init_video(fmt,
ios_fmts[l].pjmedia_format,
DEFAULT_WIDTH,
Expand Down Expand Up @@ -377,23 +386,48 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
fromConnection:(AVCaptureConnection *)connection
{
pjmedia_frame frame;
CVImageBufferRef imageBuffer;
pjmedia_frame frame = {0};
CVImageBufferRef img;

if (!sampleBuffer)
return;

/* Get a CMSampleBuffer's Core Video image buffer for the media data */
imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
img = CMSampleBufferGetImageBuffer(sampleBuffer);

/* Lock the base address of the pixel buffer */
CVPixelBufferLockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
CVPixelBufferLockBaseAddress(img, kCVPixelBufferLock_ReadOnly);

frame.type = PJMEDIA_FRAME_TYPE_VIDEO;
frame.buf = CVPixelBufferGetBaseAddress(imageBuffer);
frame.size = stream->frame_size;
frame.bit_info = 0;
frame.timestamp.u64 = stream->frame_ts.u64;

if (stream->is_planar && stream->capture_buf) {
if (stream->param.fmt.id == PJMEDIA_FORMAT_I420) {
/* kCVPixelFormatType_420YpCbCr8BiPlanar* is NV12 */
pj_uint8_t *p, *p_end, *Y, *U, *V;
pj_size_t p_len;

p = (pj_uint8_t*)CVPixelBufferGetBaseAddressOfPlane(img, 0);
p_len = stream->size.w * stream->size.h;
Y = (pj_uint8_t*)stream->capture_buf;
U = Y + p_len;
V = U + p_len/4;
pj_memcpy(Y, p, p_len);

p = (pj_uint8_t*)CVPixelBufferGetBaseAddressOfPlane(img, 1);
p_len >>= 1;
p_end = p + p_len;
while (p < p_end) {
*U++ = *p++;
*V++ = *p++;
}

frame.buf = stream->capture_buf;
}
} else {
frame.buf = CVPixelBufferGetBaseAddress(img);
}

if (stream->vid_cb.capture_cb) {
if (stream->thread_initialized == 0 || !pj_thread_is_registered())
Expand All @@ -410,7 +444,7 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput
stream->frame_ts.u64 += stream->ts_inc;

/* Unlock the pixel buffer */
CVPixelBufferUnlockBaseAddress(imageBuffer, kCVPixelBufferLock_ReadOnly);
CVPixelBufferUnlockBaseAddress(img, kCVPixelBufferLock_ReadOnly);
}
@end

Expand Down Expand Up @@ -507,10 +541,10 @@ static pj_status_t ios_factory_create_stream(

vfd = pjmedia_format_get_video_format_detail(&strm->param.fmt, PJ_TRUE);
pj_memcpy(&strm->size, &vfd->size, sizeof(vfd->size));
strm->bpp = vfi->bpp;
strm->bytes_per_row = strm->size.w * strm->bpp / 8;
strm->bytes_per_row = strm->size.w * vfi->bpp / 8;
strm->frame_size = strm->bytes_per_row * strm->size.h;
strm->ts_inc = PJMEDIA_SPF2(param->clock_rate, &vfd->fps, 1);
strm->is_planar = vfi->plane_cnt > 1;

if (param->dir & PJMEDIA_DIR_CAPTURE) {
/* Create capture stream here */
Expand All @@ -525,7 +559,7 @@ static pj_status_t ios_factory_create_stream(
vfd->size.w = 352;
vfd->size.h = 288;
strm->size = vfd->size;
strm->bytes_per_row = strm->size.w * strm->bpp / 8;
strm->bytes_per_row = strm->size.w * vfi->bpp / 8;
strm->frame_size = strm->bytes_per_row * strm->size.h;

/* Update param as output */
Expand Down Expand Up @@ -577,6 +611,10 @@ static pj_status_t ios_factory_create_stream(
[NSNumber numberWithInt:ifi->ios_format],
kCVPixelBufferPixelFormatTypeKey, nil];

/* Prepare capture buffer if it's planar format */
if (strm->is_planar)
strm->capture_buf = pj_pool_alloc(strm->pool, strm->frame_size);

/* Native preview */
if (param->flags & PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW) {
ios_stream_set_cap(&strm->base, PJMEDIA_VID_DEV_CAP_INPUT_PREVIEW,
Expand Down Expand Up @@ -676,21 +714,23 @@ static pj_status_t ios_stream_set_cap(pjmedia_vid_dev_stream *s,
/* Create view */
ios_init_view(strm);

CALayer *view_layer = strm->render_view.layer;
CGRect r = strm->render_view.bounds;

/* Preview layer instantiation should be in main thread! */
dispatch_async(dispatch_get_main_queue(), ^{
/* Create preview layer */
AVCaptureVideoPreviewLayer *previewLayer =
[AVCaptureVideoPreviewLayer layerWithSession:strm->cap_session];
AVCaptureVideoPreviewLayer *prev_layer =
[AVCaptureVideoPreviewLayer
layerWithSession:strm->cap_session];

/* Attach preview layer to a UIView */
CGRect r = strm->render_view.bounds;
previewLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
previewLayer.frame = r;
[[strm->render_view layer] addSublayer:previewLayer];
prev_layer.videoGravity = AVLayerVideoGravityResizeAspectFill;
prev_layer.frame = r;
[view_layer addSublayer:prev_layer];
PJ_LOG(4, (THIS_FILE, "Native preview initialized"));
});

NSLog(@"Native preview initialized.");

return PJ_SUCCESS;
}

Expand Down Expand Up @@ -720,9 +760,6 @@ static pj_status_t ios_stream_set_cap(pjmedia_vid_dev_stream *s,

/* Ok, let's do the switch */
AVCaptureDeviceInput *cur_dev_input = strm->dev_input;
//[AVCaptureDeviceInput
// deviceInputWithDevice:di[strm->param.cap_id].dev
// error:&error];
AVCaptureDeviceInput *new_dev_input =
[AVCaptureDeviceInput
deviceInputWithDevice:di[p->target_id].dev
Expand All @@ -749,23 +786,26 @@ static pj_status_t ios_stream_set_cap(pjmedia_vid_dev_stream *s,
if (!(ifi = get_ios_format_info(fmt->id)))
return PJMEDIA_EVID_BADFORMAT;

vfi = pjmedia_get_video_format_info(pjmedia_video_format_mgr_instance(),
fmt->id);
vfi = pjmedia_get_video_format_info(
pjmedia_video_format_mgr_instance(),
fmt->id);
if (!vfi)
return PJMEDIA_EVID_BADFORMAT;

pjmedia_format_copy(&strm->param.fmt, fmt);

vfd = pjmedia_format_get_video_format_detail(fmt, PJ_TRUE);
pj_memcpy(&strm->size, &vfd->size, sizeof(vfd->size));
strm->bytes_per_row = strm->size.w * strm->bpp / 8;
strm->bytes_per_row = strm->size.w * vfi->bpp / 8;
strm->frame_size = strm->bytes_per_row * strm->size.h;
if (strm->render_buf_size < strm->frame_size) {
/* Realloc only when needed */
strm->render_buf = pj_pool_alloc(strm->pool, strm->frame_size);
strm->render_buf_size = strm->frame_size;
CGDataProviderRelease(strm->render_data_provider);
strm->render_data_provider = CGDataProviderCreateWithData(NULL,
strm->render_buf, strm->frame_size,
strm->render_buf,
strm->frame_size,
NULL);
}

Expand All @@ -776,8 +816,9 @@ static pj_status_t ios_stream_set_cap(pjmedia_vid_dev_stream *s,
{
UIView *view = (UIView *)pval;
strm->param.window.info.ios.window = (void *)pval;
dispatch_async(dispatch_get_main_queue(),
^{[view addSubview:strm->render_view];});
dispatch_async(dispatch_get_main_queue(), ^{
[view addSubview:strm->render_view];
});
return PJ_SUCCESS;
}

Expand Down

0 comments on commit b5101f4

Please sign in to comment.