/* GStreamer Video Overlay Composition

 * Copyright (C) 2011 Intel Corporation

 * Copyright (C) 2011 Collabora Ltd.

 * Copyright (C) 2011 Tim-Philipp Müller <tim centricular net>

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Library General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Library General Public License for more details.

 *

 * You should have received a copy of the GNU Library General Public

 * License along with this library; if not, write to the

 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,

 * Boston, MA 02110-1301, USA.

 */

/**

 * SECTION:gstvideooverlaycomposition

 * @title: GstVideoOverlayRectangle

 * @short_description: Video Buffer Overlay Compositions (Subtitles, Logos)

 *

 * Functions to create and handle overlay compositions on video buffers.

 *

 * An overlay composition describes one or more overlay rectangles to be

 * blended on top of a video buffer.

 *

 * This API serves two main purposes:

 *

 * * it can be used to attach overlay information (subtitles or logos)

 *   to non-raw video buffers such as GL/VAAPI/VDPAU surfaces. The actual

 *   blending of the overlay can then be done by e.g. the video sink that

 *   processes these non-raw buffers.

 *

 * * it can also be used to blend overlay rectangles on top of raw video

 *   buffers, thus consolidating blending functionality for raw video in

 *   one place.

 *

 * Together, this allows existing overlay elements to easily handle raw

 * and non-raw video as input in without major changes (once the overlays

 * have been put into a #GstVideoOverlayComposition object anyway) - for raw

 * video the overlay can just use the blending function to blend the data

 * on top of the video, and for surface buffers it can just attach them to

 * the buffer and let the sink render the overlays.

 *

 */

/* TODO:

 *  - provide accessors for seq_num and other fields (as needed)

 *  - allow overlay to set/get original pango markup string on/from rectangle

 */

#ifdef HAVE_CONFIG_H

#include "config.h"

#endif

#include "video-overlay-composition.h"

#include "video-blend.h"

#include "gstvideometa.h"

#include <string.h>

struct _GstVideoOverlayComposition

{

  GstMiniObject parent;

  guint num_rectangles;

  GstVideoOverlayRectangle **rectangles;

  /* lowest rectangle sequence number still used by the upstream

   * overlay element. This way a renderer maintaining some kind of

   * rectangles <-> surface cache can know when to free cached

   * surfaces/rectangles. */

  guint min_seq_num_used;

  /* sequence number for the composition (same series as rectangles) */

  guint seq_num;

};

struct _GstVideoOverlayRectangle

{

  GstMiniObject parent;

  /* Position on video frame and dimension of output rectangle in

   * output frame terms (already adjusted for the PAR of the output

   * frame). x/y can be negative (overlay will be clipped then) */

  gint x, y;

  guint render_width, render_height;

  /* Info on overlay pixels (format, width, height) */

  GstVideoInfo info;

  /* The flags associated to this rectangle */

  GstVideoOverlayFormatFlags flags;

  /* Refcounted blob of memory, no caps or timestamps */

  GstBuffer *pixels;

  /* FIXME: how to express source like text or pango markup?

   *        (just add source type enum + source buffer with data)

   *

   * FOR 0.10: always send pixel blobs, but attach source data in

   * addition (reason: if downstream changes, we can't renegotiate

   * that properly, if we just do a query of supported formats from

   * the start). Sink will just ignore pixels and use pango markup

   * from source data if it supports that.

   *

   * FOR 0.11: overlay should query formats (pango markup, pixels)

   * supported by downstream and then only send that. We can

   * renegotiate via the reconfigure event.

   */

  /* sequence number: useful for backends/renderers/sinks that want

   * to maintain a cache of rectangles <-> surfaces. The value of

   * the min_seq_num_used in the composition tells the renderer which

   * rectangles have expired. */

  guint seq_num;

  /* global alpha: global alpha value of the rectangle. Each each per-pixel

   * alpha value of image-data will be multiplied with the global alpha value

   * during blending.

   * Can be used for efficient fading in/out of overlay rectangles.

   * GstElements that render OverlayCompositions and don't support global alpha

   * should simply ignore it.*/

  gfloat global_alpha;

  /* track alpha-values already applied: */

  gfloat applied_global_alpha;

  /* store initial per-pixel alpha values: */

  guint8 *initial_alpha;

  /* FIXME: we may also need a (private) way to cache converted/scaled

   * pixel blobs */

  GMutex lock;

  GList *scaled_rectangles;

};

#define GST_RECTANGLE_LOCK(rect)   g_mutex_lock(&rect->lock)

#define GST_RECTANGLE_UNLOCK(rect) g_mutex_unlock(&rect->lock)

/* --------------------------- utility functions --------------------------- */

#ifndef GST_DISABLE_GST_DEBUG

#define GST_CAT_DEFAULT ensure_debug_category()

static GstDebugCategory *

ensure_debug_category (void)

{

  static gsize cat_gonce = 0;

  if (g_once_init_enter (&cat_gonce)) {

    gsize cat_done;

    cat_done = (gsize) _gst_debug_category_new ("video-composition", 0,

        "video overlay composition");

    g_once_init_leave (&cat_gonce, cat_done);

  }

  return (GstDebugCategory *) cat_gonce;

}

#else

#define ensure_debug_category() /* NOOP */

#endif /* GST_DISABLE_GST_DEBUG */

static guint

gst_video_overlay_get_seqnum (void)

{

  static gint seqnum;           /* 0 */

  return (guint) g_atomic_int_add (&seqnum, 1);

}

static gboolean

gst_video_overlay_composition_meta_init (GstMeta * meta, gpointer params,

    GstBuffer * buf)

{

  GstVideoOverlayCompositionMeta *ometa;

  ometa = (GstVideoOverlayCompositionMeta *) meta;

  ometa->overlay = NULL;

  return TRUE;

}

static void

gst_video_overlay_composition_meta_free (GstMeta * meta, GstBuffer * buf)

{

  GstVideoOverlayCompositionMeta *ometa;

  ometa = (GstVideoOverlayCompositionMeta *) meta;

  if (ometa->overlay)

    gst_video_overlay_composition_unref (ometa->overlay);

}

static gboolean

gst_video_overlay_composition_meta_transform (GstBuffer * dest, GstMeta * meta,

    GstBuffer * buffer, GQuark type, gpointer data)

{

  GstVideoOverlayCompositionMeta *dmeta, *smeta;

  smeta = (GstVideoOverlayCompositionMeta *) meta;

  if (GST_META_TRANSFORM_IS_COPY (type)) {

    GstMetaTransformCopy *copy = data;

    if (!copy->region) {

      GST_DEBUG ("copy video overlay composition metadata");

      /* only copy if the complete data is copied as well */

      dmeta =

          (GstVideoOverlayCompositionMeta *) gst_buffer_add_meta (dest,

          GST_VIDEO_OVERLAY_COMPOSITION_META_INFO, NULL);

      if (!dmeta)

        return FALSE;

      dmeta->overlay = gst_video_overlay_composition_ref (smeta->overlay);

    }

  } else {

    /* return FALSE, if transform type is not supported */

    return FALSE;

  }

  return TRUE;

}

GType

gst_video_overlay_composition_meta_api_get_type (void)

{

  static volatile GType type = 0;

  static const gchar *tags[] = { NULL };

  if (g_once_init_enter (&type)) {

    GType _type =

        gst_meta_api_type_register ("GstVideoOverlayCompositionMetaAPI", tags);

    g_once_init_leave (&type, _type);

  }

  return type;

}

/* video overlay composition metadata */

const GstMetaInfo *

gst_video_overlay_composition_meta_get_info (void)

{

  static const GstMetaInfo *video_overlay_composition_meta_info = NULL;

  if (g_once_init_enter ((GstMetaInfo **) &

          video_overlay_composition_meta_info)) {

    const GstMetaInfo *meta =

        gst_meta_register (GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE,

        "GstVideoOverlayCompositionMeta",

        sizeof (GstVideoOverlayCompositionMeta),

        (GstMetaInitFunction) gst_video_overlay_composition_meta_init,

        (GstMetaFreeFunction) gst_video_overlay_composition_meta_free,

        (GstMetaTransformFunction)

        gst_video_overlay_composition_meta_transform);

    g_once_init_leave ((GstMetaInfo **) & video_overlay_composition_meta_info,

        (GstMetaInfo *) meta);

  }

  return video_overlay_composition_meta_info;

}

/**

 * gst_buffer_add_video_overlay_composition_meta:

 * @buf: a #GstBuffer

 * @comp: (allow-none): a #GstVideoOverlayComposition

 *

 * Sets an overlay composition on a buffer. The buffer will obtain its own

 * reference to the composition, meaning this function does not take ownership

 * of @comp.

 *

 * Returns: (transfer none): a #GstVideoOverlayCompositionMeta

 */

GstVideoOverlayCompositionMeta *

gst_buffer_add_video_overlay_composition_meta (GstBuffer * buf,

    GstVideoOverlayComposition * comp)

{

  GstVideoOverlayCompositionMeta *ometa;

  g_return_val_if_fail (gst_buffer_is_writable (buf), NULL);

  ometa = (GstVideoOverlayCompositionMeta *)

      gst_buffer_add_meta (buf, GST_VIDEO_OVERLAY_COMPOSITION_META_INFO, NULL);

  ometa->overlay = gst_video_overlay_composition_ref (comp);

  return ometa;

}

/* ------------------------------ composition ------------------------------ */

#define RECTANGLE_ARRAY_STEP 4  /* premature optimization */

GST_DEFINE_MINI_OBJECT_TYPE (GstVideoOverlayComposition,

    gst_video_overlay_composition);

static void

gst_video_overlay_composition_free (GstMiniObject * mini_obj)

{

  GstVideoOverlayComposition *comp = (GstVideoOverlayComposition *) mini_obj;

  guint num;

  num = comp->num_rectangles;

  while (num > 0) {

    gst_mini_object_remove_parent (GST_MINI_OBJECT_CAST (comp->rectangles[num -

                1]), GST_MINI_OBJECT_CAST (comp));

    gst_video_overlay_rectangle_unref (comp->rectangles[num - 1]);

    --num;

  }

  g_free (comp->rectangles);

  comp->rectangles = NULL;

  comp->num_rectangles = 0;

  g_slice_free (GstVideoOverlayComposition, comp);

}

/**

 * gst_video_overlay_composition_new:

 * @rectangle: (transfer none): a #GstVideoOverlayRectangle to add to the

 *     composition

 *

 * Creates a new video overlay composition object to hold one or more

 * overlay rectangles.

 *

 * Returns: (transfer full): a new #GstVideoOverlayComposition. Unref with

 *     gst_video_overlay_composition_unref() when no longer needed.

 */

GstVideoOverlayComposition *

gst_video_overlay_composition_new (GstVideoOverlayRectangle * rectangle)

{

  GstVideoOverlayComposition *comp;

  /* FIXME: should we allow empty compositions? Could also be expressed as

   * buffer without a composition on it. Maybe there are cases where doing

   * an empty new + _add() in a loop is easier? */

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_RECTANGLE (rectangle), NULL);

  comp = g_slice_new0 (GstVideoOverlayComposition);

  gst_mini_object_init (GST_MINI_OBJECT_CAST (comp), 0,

      GST_TYPE_VIDEO_OVERLAY_COMPOSITION,

      (GstMiniObjectCopyFunction) gst_video_overlay_composition_copy,

      NULL, (GstMiniObjectFreeFunction) gst_video_overlay_composition_free);

  comp->rectangles = g_new0 (GstVideoOverlayRectangle *, RECTANGLE_ARRAY_STEP);

  comp->rectangles[0] = gst_video_overlay_rectangle_ref (rectangle);

  gst_mini_object_add_parent (GST_MINI_OBJECT_CAST (rectangle),

      GST_MINI_OBJECT_CAST (comp));

  comp->num_rectangles = 1;

  comp->seq_num = gst_video_overlay_get_seqnum ();

  /* since the rectangle was created earlier, its seqnum is smaller than ours */

  comp->min_seq_num_used = rectangle->seq_num;

  GST_LOG ("new composition %p: seq_num %u with rectangle %p", comp,

      comp->seq_num, rectangle);

  return comp;

}

/**

 * gst_video_overlay_composition_add_rectangle:

 * @comp: a #GstVideoOverlayComposition

 * @rectangle: (transfer none): a #GstVideoOverlayRectangle to add to the

 *     composition

 *

 * Adds an overlay rectangle to an existing overlay composition object. This

 * must be done right after creating the overlay composition.

 */

void

gst_video_overlay_composition_add_rectangle (GstVideoOverlayComposition * comp,

    GstVideoOverlayRectangle * rectangle)

{

  g_return_if_fail (GST_IS_VIDEO_OVERLAY_COMPOSITION (comp));

  g_return_if_fail (GST_IS_VIDEO_OVERLAY_RECTANGLE (rectangle));

  g_return_if_fail (gst_mini_object_is_writable (GST_MINI_OBJECT_CAST (comp)));

  if (comp->num_rectangles % RECTANGLE_ARRAY_STEP == 0) {

    comp->rectangles =

        g_renew (GstVideoOverlayRectangle *, comp->rectangles,

        comp->num_rectangles + RECTANGLE_ARRAY_STEP);

  }

  comp->rectangles[comp->num_rectangles] =

      gst_video_overlay_rectangle_ref (rectangle);

  gst_mini_object_add_parent (GST_MINI_OBJECT_CAST (rectangle),

      GST_MINI_OBJECT_CAST (comp));

  comp->num_rectangles += 1;

  comp->min_seq_num_used = MIN (comp->min_seq_num_used, rectangle->seq_num);

  GST_LOG ("composition %p: added rectangle %p", comp, rectangle);

}

/**

 * gst_video_overlay_composition_n_rectangles:

 * @comp: a #GstVideoOverlayComposition

 *

 * Returns the number of #GstVideoOverlayRectangles contained in @comp.

 *

 * Returns: the number of rectangles

 */

guint

gst_video_overlay_composition_n_rectangles (GstVideoOverlayComposition * comp)

{

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_COMPOSITION (comp), 0);

  return comp->num_rectangles;

}

/**

 * gst_video_overlay_composition_get_rectangle:

 * @comp: a #GstVideoOverlayComposition

 * @n: number of the rectangle to get

 *

 * Returns the @n-th #GstVideoOverlayRectangle contained in @comp.

 *

 * Returns: (transfer none): the @n-th rectangle, or NULL if @n is out of

 *     bounds. Will not return a new reference, the caller will need to

 *     obtain her own reference using gst_video_overlay_rectangle_ref()

 *     if needed.

 */

GstVideoOverlayRectangle *

gst_video_overlay_composition_get_rectangle (GstVideoOverlayComposition * comp,

    guint n)

{

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_COMPOSITION (comp), NULL);

  if (n >= comp->num_rectangles)

    return NULL;

  return comp->rectangles[n];

}

static gboolean

gst_video_overlay_rectangle_needs_scaling (GstVideoOverlayRectangle * r)

{

  return (GST_VIDEO_INFO_WIDTH (&r->info) != r->render_width ||

      GST_VIDEO_INFO_HEIGHT (&r->info) != r->render_height);

}

/**

 * gst_video_overlay_composition_blend:

 * @comp: a #GstVideoOverlayComposition

 * @video_buf: a #GstVideoFrame containing raw video data in a

 *             supported format. It should be mapped using GST_MAP_READWRITE

 *

 * Blends the overlay rectangles in @comp on top of the raw video data

 * contained in @video_buf. The data in @video_buf must be writable and

 * mapped appropriately.

 *

 * Since @video_buf data is read and will be modified, it ought be

 * mapped with flag GST_MAP_READWRITE.

 */

/* FIXME: formats with more than 8 bit per component which get unpacked into

 * ARGB64 or AYUV64 (such as v210, v216, UYVP, GRAY16_LE and GRAY16_BE)

 * are not supported yet by the code in video-blend.c.

 */

gboolean

gst_video_overlay_composition_blend (GstVideoOverlayComposition * comp,

    GstVideoFrame * video_buf)

{

  GstVideoInfo scaled_info;

  GstVideoInfo *vinfo;

  GstVideoFrame rectangle_frame;

  GstVideoFormat fmt;

  GstBuffer *pixels = NULL;

  gboolean ret = TRUE;

  guint n, num;

  int w, h;

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_COMPOSITION (comp), FALSE);

  g_return_val_if_fail (video_buf != NULL, FALSE);

  w = GST_VIDEO_FRAME_WIDTH (video_buf);

  h = GST_VIDEO_FRAME_HEIGHT (video_buf);

  fmt = GST_VIDEO_FRAME_FORMAT (video_buf);

  num = comp->num_rectangles;

  GST_LOG ("Blending composition %p with %u rectangles onto video buffer %p "

      "(%ux%u, format %u)", comp, num, video_buf, w, h, fmt);

  for (n = 0; n < num; ++n) {

    GstVideoOverlayRectangle *rect;

    gboolean needs_scaling;

    rect = comp->rectangles[n];

    GST_LOG (" rectangle %u %p: %ux%u, format %u", n, rect,

        GST_VIDEO_INFO_WIDTH (&rect->info), GST_VIDEO_INFO_HEIGHT (&rect->info),

        GST_VIDEO_INFO_FORMAT (&rect->info));

    needs_scaling = gst_video_overlay_rectangle_needs_scaling (rect);

    if (needs_scaling) {

      gst_video_blend_scale_linear_RGBA (&rect->info, rect->pixels,

          rect->render_height, rect->render_width, &scaled_info, &pixels);

      vinfo = &scaled_info;

    } else {

      pixels = gst_buffer_ref (rect->pixels);

      vinfo = &rect->info;

    }

    gst_video_frame_map (&rectangle_frame, vinfo, pixels, GST_MAP_READ);

    ret = gst_video_blend (video_buf, &rectangle_frame, rect->x, rect->y,

        rect->global_alpha);

    gst_video_frame_unmap (&rectangle_frame);

    if (!ret) {

      GST_WARNING ("Could not blend overlay rectangle onto video buffer");

    }

    /* FIXME: should cache scaled pixels in the rectangle struct */

    gst_buffer_unref (pixels);

  }

  return ret;

}

/**

 * gst_video_overlay_composition_copy:

 * @comp: (transfer none): a #GstVideoOverlayComposition to copy

 *

 * Makes a copy of @comp and all contained rectangles, so that it is possible

 * to modify the composition and contained rectangles (e.g. add additional

 * rectangles or change the render co-ordinates or render dimension). The

 * actual overlay pixel data buffers contained in the rectangles are not

 * copied.

 *

 * Returns: (transfer full): a new #GstVideoOverlayComposition equivalent

 *     to @comp.

 */

GstVideoOverlayComposition *

gst_video_overlay_composition_copy (GstVideoOverlayComposition * comp)

{

  GstVideoOverlayComposition *copy;

  GstVideoOverlayRectangle *rect;

  guint n;

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_COMPOSITION (comp), NULL);

  if (G_LIKELY (comp->num_rectangles == 0))

    return gst_video_overlay_composition_new (NULL);

  rect = gst_video_overlay_rectangle_copy (comp->rectangles[0]);

  copy = gst_video_overlay_composition_new (rect);

  gst_video_overlay_rectangle_unref (rect);

  for (n = 1; n < comp->num_rectangles; ++n) {

    rect = gst_video_overlay_rectangle_copy (comp->rectangles[n]);

    gst_video_overlay_composition_add_rectangle (copy, rect);

    gst_video_overlay_rectangle_unref (rect);

  }

  return copy;

}

/**

 * gst_video_overlay_composition_make_writable:

 * @comp: (transfer full): a #GstVideoOverlayComposition to copy

 *

 * Takes ownership of @comp and returns a version of @comp that is writable

 * (i.e. can be modified). Will either return @comp right away, or create a

 * new writable copy of @comp and unref @comp itself. All the contained

 * rectangles will also be copied, but the actual overlay pixel data buffers

 * contained in the rectangles are not copied.

 *

 * Returns: (transfer full): a writable #GstVideoOverlayComposition

 *     equivalent to @comp.

 */

GstVideoOverlayComposition *

gst_video_overlay_composition_make_writable (GstVideoOverlayComposition * comp)

{

  GstVideoOverlayComposition *writable_comp;

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_COMPOSITION (comp), NULL);

  if (gst_mini_object_is_writable (GST_MINI_OBJECT_CAST (comp))) {

    guint n;

    for (n = 0; n < comp->num_rectangles; ++n) {

      if (!gst_mini_object_is_writable (GST_MINI_OBJECT_CAST (comp->rectangles

                  [n])))

        goto copy;

    }

    return comp;

  }

copy:

  writable_comp = gst_video_overlay_composition_copy (comp);

  gst_video_overlay_composition_unref (comp);

  return writable_comp;

}

/**

 * gst_video_overlay_composition_get_seqnum:

 * @comp: a #GstVideoOverlayComposition

 *

 * Returns the sequence number of this composition. Sequence numbers are

 * monotonically increasing and unique for overlay compositions and rectangles

 * (meaning there will never be a rectangle with the same sequence number as

 * a composition).

 *

 * Returns: the sequence number of @comp

 */

guint

gst_video_overlay_composition_get_seqnum (GstVideoOverlayComposition * comp)

{

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_COMPOSITION (comp), 0);

  return comp->seq_num;

}

/* ------------------------------ rectangles ------------------------------ -*/

GST_DEFINE_MINI_OBJECT_TYPE (GstVideoOverlayRectangle,

    gst_video_overlay_rectangle);

static void

gst_video_overlay_rectangle_free (GstMiniObject * mini_obj)

{

  GstVideoOverlayRectangle *rect = (GstVideoOverlayRectangle *) mini_obj;

  gst_mini_object_remove_parent (GST_MINI_OBJECT_CAST (rect->pixels),

      GST_MINI_OBJECT_CAST (rect));

  gst_buffer_replace (&rect->pixels, NULL);

  while (rect->scaled_rectangles != NULL) {

    GstVideoOverlayRectangle *scaled_rect = rect->scaled_rectangles->data;

    gst_video_overlay_rectangle_unref (scaled_rect);

    rect->scaled_rectangles =

        g_list_delete_link (rect->scaled_rectangles, rect->scaled_rectangles);

  }

  g_free (rect->initial_alpha);

  g_mutex_clear (&rect->lock);

  g_slice_free (GstVideoOverlayRectangle, rect);

}

static inline gboolean

gst_video_overlay_rectangle_check_flags (GstVideoOverlayFormatFlags flags)

{

  /* Check flags only contains flags we know about */

  return (flags & ~(GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA |

          GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA)) == 0;

}

static gboolean

gst_video_overlay_rectangle_is_same_alpha_type (GstVideoOverlayFormatFlags

    flags1, GstVideoOverlayFormatFlags flags2)

{

  return ((flags1 ^ flags2) & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA)

      == 0;

}

/**

 * gst_video_overlay_rectangle_new_raw:

 * @pixels: (transfer none): a #GstBuffer pointing to the pixel memory

 * @render_x: the X co-ordinate on the video where the top-left corner of this

 *     overlay rectangle should be rendered to

 * @render_y: the Y co-ordinate on the video where the top-left corner of this

 *     overlay rectangle should be rendered to

 * @render_width: the render width of this rectangle on the video

 * @render_height: the render height of this rectangle on the video

 * @flags: flags

 *

 * Creates a new video overlay rectangle with ARGB or AYUV pixel data.

 * The layout in case of ARGB of the components in memory is B-G-R-A

 * on little-endian platforms

 * (corresponding to #GST_VIDEO_FORMAT_BGRA) and A-R-G-B on big-endian

 * platforms (corresponding to #GST_VIDEO_FORMAT_ARGB). In other words,

 * pixels are treated as 32-bit words and the lowest 8 bits then contain

 * the blue component value and the highest 8 bits contain the alpha

 * component value. Unless specified in the flags, the RGB values are

 * non-premultiplied. This is the format that is used by most hardware,

 * and also many rendering libraries such as Cairo, for example.

 * The pixel data buffer must have #GstVideoMeta set.

 *

 * Returns: (transfer full): a new #GstVideoOverlayRectangle. Unref with

 *     gst_video_overlay_rectangle_unref() when no longer needed.

 */

GstVideoOverlayRectangle *

gst_video_overlay_rectangle_new_raw (GstBuffer * pixels,

    gint render_x, gint render_y, guint render_width, guint render_height,

    GstVideoOverlayFormatFlags flags)

{

  GstVideoOverlayRectangle *rect;

  GstVideoMeta *vmeta;

  GstVideoFormat format;

  guint width, height;

  g_return_val_if_fail (GST_IS_BUFFER (pixels), NULL);

  g_return_val_if_fail (render_height > 0 && render_width > 0, NULL);

  g_return_val_if_fail (gst_video_overlay_rectangle_check_flags (flags), NULL);

  /* buffer must have video meta with some expected settings */

  vmeta = gst_buffer_get_video_meta (pixels);

  g_return_val_if_fail (vmeta, NULL);

  g_return_val_if_fail (vmeta->format ==

      GST_VIDEO_OVERLAY_COMPOSITION_FORMAT_RGB ||

      vmeta->format == GST_VIDEO_OVERLAY_COMPOSITION_FORMAT_YUV, NULL);

  g_return_val_if_fail (vmeta->flags == GST_VIDEO_FRAME_FLAG_NONE, NULL);

  format = vmeta->format;

  width = vmeta->width;

  height = vmeta->height;

  /* technically ((height-1)*stride)+width might be okay too */

  g_return_val_if_fail (gst_buffer_get_size (pixels) >= height * width * 4,

      NULL);

  g_return_val_if_fail (height > 0 && width > 0, NULL);

  rect = g_slice_new0 (GstVideoOverlayRectangle);

  gst_mini_object_init (GST_MINI_OBJECT_CAST (rect), 0,

      GST_TYPE_VIDEO_OVERLAY_RECTANGLE,

      (GstMiniObjectCopyFunction) gst_video_overlay_rectangle_copy,

      NULL, (GstMiniObjectFreeFunction) gst_video_overlay_rectangle_free);

  g_mutex_init (&rect->lock);

  rect->pixels = gst_buffer_ref (pixels);

  gst_mini_object_add_parent (GST_MINI_OBJECT_CAST (pixels),

      GST_MINI_OBJECT_CAST (rect));

  rect->scaled_rectangles = NULL;

  gst_video_info_init (&rect->info);

  if (!gst_video_info_set_format (&rect->info, format, width, height)) {

    gst_mini_object_unref (GST_MINI_OBJECT_CAST (rect));

    return NULL;

  }

  if (flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA)

    rect->info.flags |= GST_VIDEO_FLAG_PREMULTIPLIED_ALPHA;

  rect->x = render_x;

  rect->y = render_y;

  rect->render_width = render_width;

  rect->render_height = render_height;

  rect->global_alpha = 1.0;

  rect->applied_global_alpha = 1.0;

  rect->initial_alpha = NULL;

  rect->flags = flags;

  rect->seq_num = gst_video_overlay_get_seqnum ();

  GST_LOG ("new rectangle %p: %ux%u => %ux%u @ %u,%u, seq_num %u, format %u, "

      "flags %x, pixels %p, global_alpha=%f", rect, width, height, render_width,

      render_height, render_x, render_y, rect->seq_num, format,

      rect->flags, pixels, rect->global_alpha);

  return rect;

}

/**

 * gst_video_overlay_rectangle_get_render_rectangle:

 * @rectangle: a #GstVideoOverlayRectangle

 * @render_x: (out) (allow-none): address where to store the X render offset

 * @render_y: (out) (allow-none): address where to store the Y render offset

 * @render_width: (out) (allow-none): address where to store the render width

 * @render_height: (out) (allow-none): address where to store the render height

 *

 * Retrieves the render position and render dimension of the overlay

 * rectangle on the video.

 *

 * Returns: TRUE if valid render dimensions were retrieved.

 */

gboolean

gst_video_overlay_rectangle_get_render_rectangle (GstVideoOverlayRectangle *

    rectangle, gint * render_x, gint * render_y, guint * render_width,

    guint * render_height)

{

  g_return_val_if_fail (GST_IS_VIDEO_OVERLAY_RECTANGLE (rectangle), FALSE);

  if (render_x)

    *render_x = rectangle->x;

  if (render_y)

    *render_y = rectangle->y;

  if (render_width)

    *render_width = rectangle->render_width;

  if (render_height)

    *render_height = rectangle->render_height;

  return TRUE;

}

/**

 * gst_video_overlay_rectangle_set_render_rectangle:

 * @rectangle: a #GstVideoOverlayRectangle

 * @render_x: render X position of rectangle on video

 * @render_y: render Y position of rectangle on video

 * @render_width: render width of rectangle

 * @render_height: render height of rectangle

 *

 * Sets the render position and dimensions of the rectangle on the video.

 * This function is mainly for elements that modify the size of the video

 * in some way (e.g. through scaling or cropping) and need to adjust the

 * details of any overlays to match the operation that changed the size.

 *

 * @rectangle must be writable, meaning its refcount must be 1. You can

 * make the rectangles inside a #GstVideoOverlayComposition writable using

 * gst_video_overlay_composition_make_writable() or

 * gst_video_overlay_composition_copy().

 */

void

gst_video_overlay_rectangle_set_render_rectangle (GstVideoOverlayRectangle *

    rectangle, gint render_x, gint render_y, guint render_width,

    guint render_height)

{

  g_return_if_fail (GST_IS_VIDEO_OVERLAY_RECTANGLE (rectangle));

  g_return_if_fail (gst_mini_object_is_writable (GST_MINI_OBJECT_CAST

          (rectangle)));

  rectangle->x = render_x;

  rectangle->y = render_y;

  rectangle->render_width = render_width;

  rectangle->render_height = render_height;

}

/* FIXME: orc-ify */

static void

gst_video_overlay_rectangle_premultiply_0 (GstVideoFrame * frame)

{

  int i, j;

  int width = GST_VIDEO_FRAME_WIDTH (frame);

  int height = GST_VIDEO_FRAME_HEIGHT (frame);

  int stride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);

  guint8 *data = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);

  for (j = 0; j < height; ++j) {

    guint8 *line;

    line = data;

    line += stride * j;

    for (i = 0; i < width; ++i) {

      int a = line[0];

      line[1] = line[1] * a / 255;

      line[2] = line[2] * a / 255;

      line[3] = line[3] * a / 255;

      line += 4;

    }

  }

}

static void

gst_video_overlay_rectangle_premultiply_3 (GstVideoFrame * frame)

{

  int i, j;

  int width = GST_VIDEO_FRAME_WIDTH (frame);

  int height = GST_VIDEO_FRAME_HEIGHT (frame);

  int stride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0);

  guint8 *data = GST_VIDEO_FRAME_PLANE_DATA (frame, 0);

  for (j = 0; j < height; ++j) {

    guint8 *line;

    line = data;

    line += stride * j;

    for (i = 0; i < width; ++i) {

      int a = line[3];