Blame libfreerdp/codec/h264_ffmpeg.c

Packit 1fb8d4
/**
Packit 1fb8d4
 * FreeRDP: A Remote Desktop Protocol Implementation
Packit 1fb8d4
 * H.264 Bitmap Compression
Packit 1fb8d4
 *
Packit 1fb8d4
 * Copyright 2015 Marc-André Moreau <marcandre.moreau@gmail.com>
Packit 1fb8d4
 * Copyright 2014 Mike McDonald <Mike.McDonald@software.dell.com>
Packit 1fb8d4
 * Copyright 2014 erbth <t.erbesdobler@team103.com>
Packit 1fb8d4
 *
Packit 1fb8d4
 * Licensed under the Apache License, Version 2.0 (the "License");
Packit 1fb8d4
 * you may not use this file except in compliance with the License.
Packit 1fb8d4
 * You may obtain a copy of the License at
Packit 1fb8d4
 *
Packit 1fb8d4
 *     http://www.apache.org/licenses/LICENSE-2.0
Packit 1fb8d4
 *
Packit 1fb8d4
 * Unless required by applicable law or agreed to in writing, software
Packit 1fb8d4
 * distributed under the License is distributed on an "AS IS" BASIS,
Packit 1fb8d4
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Packit 1fb8d4
 * See the License for the specific language governing permissions and
Packit 1fb8d4
 * limitations under the License.
Packit 1fb8d4
 */
Packit 1fb8d4
#ifdef HAVE_CONFIG_H
Packit 1fb8d4
#include "config.h"
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
#include <winpr/wlog.h>
Packit 1fb8d4
#include <freerdp/log.h>
Packit 1fb8d4
#include <freerdp/codec/h264.h>
Packit 1fb8d4
#include <libavcodec/avcodec.h>
Packit 1fb8d4
#include <libavutil/opt.h>
Packit 1fb8d4
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 9, 0)
Packit 1fb8d4
#include <libavutil/hwcontext.h>
Packit 1fb8d4
#else
Packit 1fb8d4
#pragma warning You have asked for VA-API decoding, but your version of libavutil is too old! Disabling.
Packit 1fb8d4
#undef WITH_VAAPI
Packit 1fb8d4
#endif
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
/* Fallback support for older libavcodec versions */
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(54, 59, 100)
Packit 1fb8d4
#define AV_CODEC_ID_H264 CODEC_ID_H264
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(56, 34, 2)
Packit 1fb8d4
#define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER
Packit 1fb8d4
#define AV_CODEC_CAP_TRUNCATED CODEC_CAP_TRUNCATED
Packit 1fb8d4
#define AV_CODEC_FLAG_TRUNCATED CODEC_FLAG_TRUNCATED
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
#if LIBAVUTIL_VERSION_MAJOR < 52
Packit 1fb8d4
#define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
/* Ubuntu 14.04 ships without the functions provided by avutil,
Packit 1fb8d4
 * so define error to string methods here. */
Packit 1fb8d4
#if !defined(av_err2str)
Packit 1fb8d4
static inline char* error_string(char* errbuf, size_t errbuf_size, int errnum)
Packit 1fb8d4
{
Packit 1fb8d4
	av_strerror(errnum, errbuf, errbuf_size);
Packit 1fb8d4
	return errbuf;
Packit 1fb8d4
}
Packit 1fb8d4
Packit 1fb8d4
#define av_err2str(errnum) \
Packit 1fb8d4
	error_string((char[64]){0}, 64, errnum)
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
#define VAAPI_DEVICE "/dev/dri/renderD128"
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
struct _H264_CONTEXT_LIBAVCODEC
Packit 1fb8d4
{
Packit 1fb8d4
	AVCodec* codecDecoder;
Packit 1fb8d4
	AVCodecContext* codecDecoderContext;
Packit 1fb8d4
	AVCodec* codecEncoder;
Packit 1fb8d4
	AVCodecContext* codecEncoderContext;
Packit 1fb8d4
	AVCodecParserContext* codecParser;
Packit 1fb8d4
	AVFrame* videoFrame;
Packit 1fb8d4
	AVPacket packet;
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
	AVBufferRef* hwctx;
Packit 1fb8d4
	AVFrame* hwVideoFrame;
Packit 1fb8d4
	enum AVPixelFormat hw_pix_fmt;
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
Packit 1fb8d4
	AVBufferRef* hw_frames_ctx;
Packit 1fb8d4
#endif
Packit 1fb8d4
#endif
Packit 1fb8d4
};
Packit 1fb8d4
typedef struct _H264_CONTEXT_LIBAVCODEC H264_CONTEXT_LIBAVCODEC;
Packit 1fb8d4
Packit 1fb8d4
static void libavcodec_destroy_encoder(H264_CONTEXT* h264)
Packit 1fb8d4
{
Packit 1fb8d4
	H264_CONTEXT_LIBAVCODEC* sys;
Packit 1fb8d4
Packit 1fb8d4
	if (!h264 || !h264->subsystem)
Packit 1fb8d4
		return;
Packit 1fb8d4
Packit 1fb8d4
	sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
Packit 1fb8d4
Packit 1fb8d4
	if (sys->codecEncoderContext)
Packit 1fb8d4
	{
Packit 1fb8d4
		avcodec_close(sys->codecEncoderContext);
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
Packit 1fb8d4
		avcodec_free_context(&sys->codecEncoderContext);
Packit 1fb8d4
#else
Packit 1fb8d4
		av_free(sys->codecEncoderContext);
Packit 1fb8d4
#endif
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	sys->codecEncoder = NULL;
Packit 1fb8d4
	sys->codecEncoderContext = NULL;
Packit 1fb8d4
}
Packit 1fb8d4
Packit 1fb8d4
static BOOL libavcodec_create_encoder(H264_CONTEXT* h264)
Packit 1fb8d4
{
Packit 1fb8d4
	BOOL recreate = FALSE;
Packit 1fb8d4
	H264_CONTEXT_LIBAVCODEC* sys;
Packit 1fb8d4
Packit 1fb8d4
	if (!h264 || !h264->subsystem)
Packit 1fb8d4
		return FALSE;
Packit 1fb8d4
Packit 1fb8d4
	sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
Packit 1fb8d4
	recreate = !sys->codecEncoder || !sys->codecEncoderContext;
Packit 1fb8d4
Packit 1fb8d4
	if (sys->codecEncoderContext)
Packit 1fb8d4
	{
Packit 1fb8d4
		if ((sys->codecEncoderContext->width != h264->width) ||
Packit 1fb8d4
		    (sys->codecEncoderContext->height != h264->height))
Packit 1fb8d4
			recreate = TRUE;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	if (!recreate)
Packit 1fb8d4
		return TRUE;
Packit 1fb8d4
Packit 1fb8d4
	libavcodec_destroy_encoder(h264);
Packit 1fb8d4
	sys->codecEncoder = avcodec_find_encoder(AV_CODEC_ID_H264);
Packit 1fb8d4
Packit 1fb8d4
	if (!sys->codecEncoder)
Packit 1fb8d4
		goto EXCEPTION;
Packit 1fb8d4
Packit 1fb8d4
	sys->codecEncoderContext = avcodec_alloc_context3(sys->codecEncoder);
Packit 1fb8d4
Packit 1fb8d4
	if (!sys->codecEncoderContext)
Packit 1fb8d4
		goto EXCEPTION;
Packit 1fb8d4
Packit 1fb8d4
	switch (h264->RateControlMode)
Packit 1fb8d4
	{
Packit 1fb8d4
		case H264_RATECONTROL_VBR:
Packit 1fb8d4
			sys->codecEncoderContext->bit_rate = h264->BitRate;
Packit 1fb8d4
			break;
Packit 1fb8d4
Packit 1fb8d4
		case H264_RATECONTROL_CQP:
Packit 1fb8d4
			/* TODO: sys->codecEncoderContext-> = h264->QP; */
Packit 1fb8d4
			break;
Packit 1fb8d4
Packit 1fb8d4
		default:
Packit 1fb8d4
			break;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	sys->codecEncoderContext->width = h264->width;
Packit 1fb8d4
	sys->codecEncoderContext->height = h264->height;
Packit 1fb8d4
	sys->codecEncoderContext->delay = 0;
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
Packit 1fb8d4
	sys->codecEncoderContext->framerate = (AVRational)
Packit 1fb8d4
	{
Packit 1fb8d4
		h264->FrameRate, 1
Packit 1fb8d4
	};
Packit 1fb8d4
#endif
Packit 1fb8d4
	sys->codecEncoderContext->time_base = (AVRational)
Packit 1fb8d4
	{
Packit 1fb8d4
		1, h264->FrameRate
Packit 1fb8d4
	};
Packit 1fb8d4
	av_opt_set(sys->codecEncoderContext, "preset", "medium", AV_OPT_SEARCH_CHILDREN);
Packit 1fb8d4
	av_opt_set(sys->codecEncoderContext, "tune", "zerolatency", AV_OPT_SEARCH_CHILDREN);
Packit 1fb8d4
	sys->codecEncoderContext->flags |= AV_CODEC_FLAG_LOOP_FILTER;
Packit 1fb8d4
	sys->codecEncoderContext->pix_fmt = AV_PIX_FMT_YUV420P;
Packit 1fb8d4
Packit 1fb8d4
	if (avcodec_open2(sys->codecEncoderContext, sys->codecEncoder, NULL) < 0)
Packit 1fb8d4
		goto EXCEPTION;
Packit 1fb8d4
Packit 1fb8d4
	return TRUE;
Packit 1fb8d4
EXCEPTION:
Packit 1fb8d4
	libavcodec_destroy_encoder(h264);
Packit 1fb8d4
	return FALSE;
Packit 1fb8d4
}
Packit 1fb8d4
Packit 1fb8d4
static int libavcodec_decompress(H264_CONTEXT* h264, const BYTE* pSrcData,
Packit 1fb8d4
                                 UINT32 SrcSize)
Packit 1fb8d4
{
Packit 1fb8d4
	int status;
Packit 1fb8d4
	int gotFrame = 0;
Packit 1fb8d4
	H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*) h264->pSystemData;
Packit 1fb8d4
	BYTE** pYUVData = h264->pYUVData;
Packit 1fb8d4
	UINT32* iStride = h264->iStride;
Packit 1fb8d4
	av_init_packet(&sys->packet);
Packit 1fb8d4
	sys->packet.data = (BYTE*)pSrcData;
Packit 1fb8d4
	sys->packet.size = SrcSize;
Packit 1fb8d4
	/* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
Packit 1fb8d4
	status = avcodec_send_packet(sys->codecDecoderContext, &sys->packet);
Packit 1fb8d4
Packit 1fb8d4
	if (status < 0)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
Packit 1fb8d4
		return -1;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	sys->videoFrame->format = AV_PIX_FMT_YUV420P;
Packit 1fb8d4
Packit 1fb8d4
	do
Packit 1fb8d4
	{
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
		status = avcodec_receive_frame(sys->codecDecoderContext,
Packit 1fb8d4
		                               sys->hwctx ? sys->hwVideoFrame : sys->videoFrame);
Packit 1fb8d4
#else
Packit 1fb8d4
		status = avcodec_receive_frame(sys->codecDecoderContext, sys->videoFrame);
Packit 1fb8d4
#endif
Packit 1fb8d4
	}
Packit 1fb8d4
	while (status == AVERROR(EAGAIN));
Packit 1fb8d4
Packit 1fb8d4
	gotFrame = (status == 0);
Packit 1fb8d4
#else
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
	status = avcodec_decode_video2(sys->codecDecoderContext,
Packit 1fb8d4
	                               sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame,
Packit 1fb8d4
	                               &sys->packet);
Packit 1fb8d4
#else
Packit 1fb8d4
	status = avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame,
Packit 1fb8d4
	                               &sys->packet);
Packit 1fb8d4
#endif
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
	if (status < 0)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to decode video frame (status=%d)", status);
Packit 1fb8d4
		return -1;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
Packit 1fb8d4
	if (sys->hwctx)
Packit 1fb8d4
	{
Packit 1fb8d4
		if (sys->hwVideoFrame->format == sys->hw_pix_fmt)
Packit 1fb8d4
		{
Packit 1fb8d4
			sys->videoFrame->width = sys->hwVideoFrame->width;
Packit 1fb8d4
			sys->videoFrame->height = sys->hwVideoFrame->height;
Packit 1fb8d4
			status = av_hwframe_transfer_data(sys->videoFrame, sys->hwVideoFrame, 0);
Packit 1fb8d4
		}
Packit 1fb8d4
		else
Packit 1fb8d4
		{
Packit 1fb8d4
			status = av_frame_copy(sys->videoFrame, sys->hwVideoFrame);
Packit 1fb8d4
		}
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	gotFrame = (status == 0);
Packit 1fb8d4
Packit 1fb8d4
	if (status < 0)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to transfer video frame (status=%d) (%s)", status,
Packit 1fb8d4
		           av_err2str(status));
Packit 1fb8d4
		return -1;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#endif
Packit 1fb8d4
#if 0
Packit 1fb8d4
	WLog_Print(h264->log, WLOG_INFO,
Packit 1fb8d4
	           "libavcodec_decompress: frame decoded (status=%d, gotFrame=%d, width=%d, height=%d, Y=[%p,%d], U=[%p,%d], V=[%p,%d])",
Packit 1fb8d4
	           status, gotFrame, sys->videoFrame->width, sys->videoFrame->height,
Packit 1fb8d4
	           (void*) sys->videoFrame->data[0], sys->videoFrame->linesize[0],
Packit 1fb8d4
	           (void*) sys->videoFrame->data[1], sys->videoFrame->linesize[1],
Packit 1fb8d4
	           (void*) sys->videoFrame->data[2], sys->videoFrame->linesize[2]);
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
	if (gotFrame)
Packit 1fb8d4
	{
Packit 1fb8d4
		pYUVData[0] = sys->videoFrame->data[0];
Packit 1fb8d4
		pYUVData[1] = sys->videoFrame->data[1];
Packit 1fb8d4
		pYUVData[2] = sys->videoFrame->data[2];
Packit 1fb8d4
		iStride[0] = sys->videoFrame->linesize[0];
Packit 1fb8d4
		iStride[1] = sys->videoFrame->linesize[1];
Packit 1fb8d4
		iStride[2] = sys->videoFrame->linesize[2];
Packit 1fb8d4
		h264->width = sys->videoFrame->width;
Packit 1fb8d4
		h264->height = sys->videoFrame->height;
Packit 1fb8d4
	}
Packit 1fb8d4
	else
Packit 1fb8d4
		return -2;
Packit 1fb8d4
Packit 1fb8d4
	return 1;
Packit 1fb8d4
}
Packit 1fb8d4
Packit 1fb8d4
static int libavcodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const UINT32* pStride,
Packit 1fb8d4
                               BYTE** ppDstData, UINT32* pDstSize)
Packit 1fb8d4
{
Packit 1fb8d4
	int status;
Packit 1fb8d4
	int gotFrame = 0;
Packit 1fb8d4
	H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*) h264->pSystemData;
Packit 1fb8d4
Packit 1fb8d4
	if (!libavcodec_create_encoder(h264))
Packit 1fb8d4
		return -1;
Packit 1fb8d4
Packit 1fb8d4
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 39, 100)
Packit 1fb8d4
	av_packet_unref(&sys->packet);
Packit 1fb8d4
#else
Packit 1fb8d4
	av_free(sys->packet.data);
Packit 1fb8d4
#endif
Packit 1fb8d4
	av_init_packet(&sys->packet);
Packit 1fb8d4
	sys->packet.data = NULL;
Packit 1fb8d4
	sys->packet.size = 0;
Packit 1fb8d4
	sys->videoFrame->format = sys->codecEncoderContext->pix_fmt;
Packit 1fb8d4
	sys->videoFrame->width  = sys->codecEncoderContext->width;
Packit 1fb8d4
	sys->videoFrame->height = sys->codecEncoderContext->height;
Packit 1fb8d4
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 48, 100)
Packit 1fb8d4
	sys->videoFrame->colorspace = AVCOL_SPC_BT709;
Packit 1fb8d4
#endif
Packit 1fb8d4
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(52, 92, 100)
Packit 1fb8d4
	sys->videoFrame->chroma_location = AVCHROMA_LOC_LEFT;
Packit 1fb8d4
#endif
Packit 1fb8d4
	sys->videoFrame->data[0] = (uint8_t*)pSrcYuv[0];
Packit 1fb8d4
	sys->videoFrame->data[1] = (uint8_t*)pSrcYuv[1];
Packit 1fb8d4
	sys->videoFrame->data[2] = (uint8_t*)pSrcYuv[2];
Packit 1fb8d4
	sys->videoFrame->linesize[0] = (int)pStride[0];
Packit 1fb8d4
	sys->videoFrame->linesize[1] = (int)pStride[1];
Packit 1fb8d4
	sys->videoFrame->linesize[2] = (int)pStride[2];
Packit 1fb8d4
	sys->videoFrame->pts++;
Packit 1fb8d4
	/* avcodec_encode_video2 is deprecated with libavcodec 57.48.101 */
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
Packit 1fb8d4
	status = avcodec_send_frame(sys->codecEncoderContext, sys->videoFrame);
Packit 1fb8d4
Packit 1fb8d4
	if (status < 0)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
Packit 1fb8d4
		           av_err2str(status), status);
Packit 1fb8d4
		return -1;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	status = avcodec_receive_packet(sys->codecEncoderContext,
Packit 1fb8d4
	                                &sys->packet);
Packit 1fb8d4
Packit 1fb8d4
	if (status < 0)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
Packit 1fb8d4
		           av_err2str(status), status);
Packit 1fb8d4
		return -1;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	gotFrame = (status == 0);
Packit 1fb8d4
#elif LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(54, 59, 100)
Packit 1fb8d4
Packit 1fb8d4
	do
Packit 1fb8d4
	{
Packit 1fb8d4
		status = avcodec_encode_video2(sys->codecEncoderContext,
Packit 1fb8d4
		                               &sys->packet,
Packit 1fb8d4
		                               sys->videoFrame, &gotFrame);
Packit 1fb8d4
	}
Packit 1fb8d4
	while ((status >= 0) && (gotFrame == 0));
Packit 1fb8d4
Packit 1fb8d4
#else
Packit 1fb8d4
	sys->packet.size = avpicture_get_size(sys->codecDecoderContext->pix_fmt,
Packit 1fb8d4
	                                      sys->codecDecoderContext->width,
Packit 1fb8d4
	                                      sys->codecDecoderContext->height);
Packit 1fb8d4
	sys->packet.data = av_malloc(sys->packet.size);
Packit 1fb8d4
Packit 1fb8d4
	if (!sys->packet.data)
Packit 1fb8d4
		status = -1;
Packit 1fb8d4
	else
Packit 1fb8d4
	{
Packit 1fb8d4
		status = avcodec_encode_video(sys->codecDecoderContext, sys->packet.data,
Packit 1fb8d4
		                              sys->packet.size, sys->videoFrame);
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
	if (status < 0)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to encode video frame (%s [%d])",
Packit 1fb8d4
		           av_err2str(status), status);
Packit 1fb8d4
		return -1;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	*ppDstData = sys->packet.data;
Packit 1fb8d4
	*pDstSize = sys->packet.size;
Packit 1fb8d4
Packit 1fb8d4
	if (!gotFrame)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Did not get frame! (%s [%d])", av_err2str(status), status);
Packit 1fb8d4
		return -2;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	return 1;
Packit 1fb8d4
}
Packit 1fb8d4
Packit 1fb8d4
static void libavcodec_uninit(H264_CONTEXT* h264)
Packit 1fb8d4
{
Packit 1fb8d4
	H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*) h264->pSystemData;
Packit 1fb8d4
Packit 1fb8d4
	if (!sys)
Packit 1fb8d4
		return;
Packit 1fb8d4
Packit 1fb8d4
	if (sys->videoFrame)
Packit 1fb8d4
	{
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
Packit 1fb8d4
		av_frame_free(&sys->videoFrame);
Packit 1fb8d4
#else
Packit 1fb8d4
		av_free(sys->videoFrame);
Packit 1fb8d4
#endif
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
Packit 1fb8d4
	if (sys->hwVideoFrame)
Packit 1fb8d4
	{
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
Packit 1fb8d4
		av_frame_free(&sys->hwVideoFrame);
Packit 1fb8d4
#else
Packit 1fb8d4
		av_free(sys->hwVideoFrame);
Packit 1fb8d4
#endif
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	if (sys->hwctx)
Packit 1fb8d4
	{
Packit 1fb8d4
		av_buffer_unref(&sys->hwctx);
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
Packit 1fb8d4
Packit 1fb8d4
	if (sys->hw_frames_ctx)
Packit 1fb8d4
	{
Packit 1fb8d4
		av_buffer_unref(&sys->hw_frames_ctx);
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#endif
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
	if (sys->codecParser)
Packit 1fb8d4
		av_parser_close(sys->codecParser);
Packit 1fb8d4
Packit 1fb8d4
	if (sys->codecDecoderContext)
Packit 1fb8d4
	{
Packit 1fb8d4
		avcodec_close(sys->codecDecoderContext);
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 69, 100)
Packit 1fb8d4
		avcodec_free_context(&sys->codecDecoderContext);
Packit 1fb8d4
#else
Packit 1fb8d4
		av_free(sys->codecDecoderContext);
Packit 1fb8d4
#endif
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	libavcodec_destroy_encoder(h264);
Packit 1fb8d4
	free(sys);
Packit 1fb8d4
	h264->pSystemData = NULL;
Packit 1fb8d4
}
Packit 1fb8d4
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
static enum AVPixelFormat libavcodec_get_format(struct AVCodecContext* ctx,
Packit 1fb8d4
        const enum AVPixelFormat* fmts)
Packit 1fb8d4
{
Packit 1fb8d4
	H264_CONTEXT* h264 = (H264_CONTEXT*) ctx->opaque;
Packit 1fb8d4
	H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*) h264->subsystem;
Packit 1fb8d4
	const enum AVPixelFormat* p;
Packit 1fb8d4
Packit 1fb8d4
	for (p = fmts; *p != AV_PIX_FMT_NONE; p++)
Packit 1fb8d4
	{
Packit 1fb8d4
		if (*p == sys->hw_pix_fmt)
Packit 1fb8d4
		{
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(57, 80, 100)
Packit 1fb8d4
			sys->hw_frames_ctx = av_hwframe_ctx_alloc(sys->hwctx);
Packit 1fb8d4
Packit 1fb8d4
			if (!sys->hw_frames_ctx)
Packit 1fb8d4
			{
Packit 1fb8d4
				return AV_PIX_FMT_NONE;
Packit 1fb8d4
			}
Packit 1fb8d4
Packit 1fb8d4
			sys->codecDecoderContext->pix_fmt = *p;
Packit 1fb8d4
			AVHWFramesContext* frames = (AVHWFramesContext*) sys->hw_frames_ctx->data;
Packit 1fb8d4
			frames->format = *p;
Packit 1fb8d4
			frames->height = sys->codecDecoderContext->coded_height;
Packit 1fb8d4
			frames->width = sys->codecDecoderContext->coded_width;
Packit 1fb8d4
			frames->sw_format = (sys->codecDecoderContext->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ?
Packit 1fb8d4
			                     AV_PIX_FMT_P010 : AV_PIX_FMT_NV12);
Packit 1fb8d4
			frames->initial_pool_size = 20;
Packit 1fb8d4
Packit 1fb8d4
			if (sys->codecDecoderContext->active_thread_type & FF_THREAD_FRAME)
Packit 1fb8d4
				frames->initial_pool_size += sys->codecDecoderContext->thread_count;
Packit 1fb8d4
Packit 1fb8d4
			int err = av_hwframe_ctx_init(sys->hw_frames_ctx);
Packit 1fb8d4
Packit 1fb8d4
			if (err < 0)
Packit 1fb8d4
			{
Packit 1fb8d4
				WLog_Print(h264->log, WLOG_ERROR, "Could not init hwframes context: %s", av_err2str(err));
Packit 1fb8d4
				return AV_PIX_FMT_NONE;
Packit 1fb8d4
			}
Packit 1fb8d4
Packit 1fb8d4
			sys->codecDecoderContext->hw_frames_ctx = av_buffer_ref(sys->hw_frames_ctx);
Packit 1fb8d4
#endif
Packit 1fb8d4
			return *p;
Packit 1fb8d4
		}
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	return AV_PIX_FMT_NONE;
Packit 1fb8d4
}
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
static BOOL libavcodec_init(H264_CONTEXT* h264)
Packit 1fb8d4
{
Packit 1fb8d4
	H264_CONTEXT_LIBAVCODEC* sys;
Packit 1fb8d4
	sys = (H264_CONTEXT_LIBAVCODEC*) calloc(1, sizeof(H264_CONTEXT_LIBAVCODEC));
Packit 1fb8d4
Packit 1fb8d4
	if (!sys)
Packit 1fb8d4
	{
Packit 1fb8d4
		goto EXCEPTION;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
	h264->pSystemData = (void*) sys;
Packit 1fb8d4
	avcodec_register_all();
Packit 1fb8d4
Packit 1fb8d4
	if (!h264->Compressor)
Packit 1fb8d4
	{
Packit 1fb8d4
		sys->codecDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
Packit 1fb8d4
Packit 1fb8d4
		if (!sys->codecDecoder)
Packit 1fb8d4
		{
Packit 1fb8d4
			WLog_Print(h264->log, WLOG_ERROR, "Failed to find libav H.264 codec");
Packit 1fb8d4
			goto EXCEPTION;
Packit 1fb8d4
		}
Packit 1fb8d4
Packit 1fb8d4
		sys->codecDecoderContext = avcodec_alloc_context3(sys->codecDecoder);
Packit 1fb8d4
Packit 1fb8d4
		if (!sys->codecDecoderContext)
Packit 1fb8d4
		{
Packit 1fb8d4
			WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav codec context");
Packit 1fb8d4
			goto EXCEPTION;
Packit 1fb8d4
		}
Packit 1fb8d4
Packit 1fb8d4
		if (sys->codecDecoder->capabilities & AV_CODEC_CAP_TRUNCATED)
Packit 1fb8d4
		{
Packit 1fb8d4
			sys->codecDecoderContext->flags |= AV_CODEC_FLAG_TRUNCATED;
Packit 1fb8d4
		}
Packit 1fb8d4
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
Packit 1fb8d4
		if (!sys->hwctx)
Packit 1fb8d4
		{
Packit 1fb8d4
			int ret = av_hwdevice_ctx_create(&sys->hwctx, AV_HWDEVICE_TYPE_VAAPI, VAAPI_DEVICE, NULL, 0);
Packit 1fb8d4
Packit 1fb8d4
			if (ret < 0)
Packit 1fb8d4
			{
Packit 1fb8d4
				WLog_Print(h264->log, WLOG_ERROR,
Packit 1fb8d4
				           "Could not initialize hardware decoder, falling back to software: %s",
Packit 1fb8d4
				           av_err2str(ret));
Packit 1fb8d4
				sys->hwctx = NULL;
Packit 1fb8d4
				goto fail_hwdevice_create;
Packit 1fb8d4
			}
Packit 1fb8d4
		}
Packit 1fb8d4
Packit 1fb8d4
		sys->codecDecoderContext->get_format = libavcodec_get_format;
Packit 1fb8d4
		sys->hw_pix_fmt = AV_PIX_FMT_VAAPI;
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 80, 100)
Packit 1fb8d4
		sys->codecDecoderContext->hw_device_ctx = av_buffer_ref(sys->hwctx);
Packit 1fb8d4
#endif
Packit 1fb8d4
		sys->codecDecoderContext->opaque = (void*) h264;
Packit 1fb8d4
	fail_hwdevice_create:
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
		if (avcodec_open2(sys->codecDecoderContext, sys->codecDecoder, NULL) < 0)
Packit 1fb8d4
		{
Packit 1fb8d4
			WLog_Print(h264->log, WLOG_ERROR, "Failed to open libav codec");
Packit 1fb8d4
			goto EXCEPTION;
Packit 1fb8d4
		}
Packit 1fb8d4
Packit 1fb8d4
		sys->codecParser = av_parser_init(AV_CODEC_ID_H264);
Packit 1fb8d4
Packit 1fb8d4
		if (!sys->codecParser)
Packit 1fb8d4
		{
Packit 1fb8d4
			WLog_Print(h264->log, WLOG_ERROR, "Failed to initialize libav parser");
Packit 1fb8d4
			goto EXCEPTION;
Packit 1fb8d4
		}
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 18, 102)
Packit 1fb8d4
	sys->videoFrame = av_frame_alloc();
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
	sys->hwVideoFrame = av_frame_alloc();
Packit 1fb8d4
#endif
Packit 1fb8d4
#else
Packit 1fb8d4
	sys->videoFrame = avcodec_alloc_frame();
Packit 1fb8d4
#endif
Packit 1fb8d4
Packit 1fb8d4
	if (!sys->videoFrame)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav frame");
Packit 1fb8d4
		goto EXCEPTION;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#ifdef WITH_VAAPI
Packit 1fb8d4
Packit 1fb8d4
	if (!sys->hwVideoFrame)
Packit 1fb8d4
	{
Packit 1fb8d4
		WLog_Print(h264->log, WLOG_ERROR, "Failed to allocate libav hw frame");
Packit 1fb8d4
		goto EXCEPTION;
Packit 1fb8d4
	}
Packit 1fb8d4
Packit 1fb8d4
#endif
Packit 1fb8d4
	sys->videoFrame->pts = 0;
Packit 1fb8d4
	return TRUE;
Packit 1fb8d4
EXCEPTION:
Packit 1fb8d4
	libavcodec_uninit(h264);
Packit 1fb8d4
	return FALSE;
Packit 1fb8d4
}
Packit 1fb8d4
Packit 1fb8d4
H264_CONTEXT_SUBSYSTEM g_Subsystem_libavcodec =
Packit 1fb8d4
{
Packit 1fb8d4
	"libavcodec",
Packit 1fb8d4
	libavcodec_init,
Packit 1fb8d4
	libavcodec_uninit,
Packit 1fb8d4
	libavcodec_decompress,
Packit 1fb8d4
	libavcodec_compress
Packit 1fb8d4
};