/**************************************************************************

 *

 * Copyright © 2007 Red Hat Inc.

 * Copyright © 2007-2012 Intel Corporation

 * Copyright 2006 Tungsten Graphics, Inc., Bismarck, ND., USA

 * All Rights Reserved.

 *

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the

 * "Software"), to deal in the Software without restriction, including

 * without limitation the rights to use, copy, modify, merge, publish,

 * distribute, sub license, and/or sell copies of the Software, and to

 * permit persons to whom the Software is furnished to do so, subject to

 * the following conditions:

 *

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,

 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL

 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,

 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR

 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE

 * USE OR OTHER DEALINGS IN THE SOFTWARE.

 *

 * The above copyright notice and this permission notice (including the

 * next paragraph) shall be included in all copies or substantial portions

 * of the Software.

 *

 *

 **************************************************************************/

/*

 * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>

 *          Keith Whitwell <keithw-at-tungstengraphics-dot-com>

 *	    Eric Anholt <eric@anholt.net>

 *	    Dave Airlie <airlied@linux.ie>

 */

#include <xf86drm.h>

#include <xf86atomic.h>

#include <fcntl.h>

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#include <assert.h>

#include <pthread.h>

#include <sys/ioctl.h>

#include <sys/stat.h>

#include <sys/types.h>

#include <stdbool.h>

#include "errno.h"

#ifndef ETIME

#define ETIME ETIMEDOUT

#endif

#include "libdrm_macros.h"

#include "libdrm_lists.h"

#include "intel_bufmgr.h"

#include "intel_bufmgr_priv.h"

#include "intel_chipset.h"

#include "string.h"

#include "i915_drm.h"

#include "uthash.h"

#if HAVE_VALGRIND

#include <valgrind.h>

#include <memcheck.h>

#define VG(x) x

#else

#define VG(x)

#endif

#define memclear(s) memset(&s, 0, sizeof(s))

#define DBG(...) do {					\

	if (bufmgr_gem->bufmgr.debug)			\

		fprintf(stderr, __VA_ARGS__);		\

} while (0)

#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))

#define MAX2(A, B) ((A) > (B) ? (A) : (B))

/**

 * upper_32_bits - return bits 32-63 of a number

 * @n: the number we're accessing

 *

 * A basic shift-right of a 64- or 32-bit quantity.  Use this to suppress

 * the "right shift count >= width of type" warning when that quantity is

 * 32-bits.

 */

#define upper_32_bits(n) ((__u32)(((n) >> 16) >> 16))

/**

 * lower_32_bits - return bits 0-31 of a number

 * @n: the number we're accessing

 */

#define lower_32_bits(n) ((__u32)(n))

typedef struct _drm_intel_bo_gem drm_intel_bo_gem;

struct drm_intel_gem_bo_bucket {

	drmMMListHead head;

	unsigned long size;

};

typedef struct _drm_intel_bufmgr_gem {

	drm_intel_bufmgr bufmgr;

	atomic_t refcount;

	int fd;

	int max_relocs;

	pthread_mutex_t lock;

	struct drm_i915_gem_exec_object *exec_objects;

	struct drm_i915_gem_exec_object2 *exec2_objects;

	drm_intel_bo **exec_bos;

	int exec_size;

	int exec_count;

	/** Array of lists of cached gem objects of power-of-two sizes */

	struct drm_intel_gem_bo_bucket cache_bucket[14 * 4];

	int num_buckets;

	time_t time;

	drmMMListHead managers;

	drm_intel_bo_gem *name_table;

	drm_intel_bo_gem *handle_table;

	drmMMListHead vma_cache;

	int vma_count, vma_open, vma_max;

	uint64_t gtt_size;

	int available_fences;

	int pci_device;

	int gen;

	unsigned int has_bsd : 1;

	unsigned int has_blt : 1;

	unsigned int has_relaxed_fencing : 1;

	unsigned int has_llc : 1;

	unsigned int has_wait_timeout : 1;

	unsigned int bo_reuse : 1;

	unsigned int no_exec : 1;

	unsigned int has_vebox : 1;

	unsigned int has_exec_async : 1;

	bool fenced_relocs;

	struct {

		void *ptr;

		uint32_t handle;

	} userptr_active;

} drm_intel_bufmgr_gem;

#define DRM_INTEL_RELOC_FENCE (1<<0)

typedef struct _drm_intel_reloc_target_info {

	drm_intel_bo *bo;

	int flags;

} drm_intel_reloc_target;

struct _drm_intel_bo_gem {

	drm_intel_bo bo;

	atomic_t refcount;

	uint32_t gem_handle;

	const char *name;

	/**

	 * Kenel-assigned global name for this object

         *

         * List contains both flink named and prime fd'd objects

	 */

	unsigned int global_name;

	UT_hash_handle handle_hh;

	UT_hash_handle name_hh;

	/**

	 * Index of the buffer within the validation list while preparing a

	 * batchbuffer execution.

	 */

	int validate_index;

	/**

	 * Current tiling mode

	 */

	uint32_t tiling_mode;

	uint32_t swizzle_mode;

	unsigned long stride;

	unsigned long kflags;

	time_t free_time;

	/** Array passed to the DRM containing relocation information. */

	struct drm_i915_gem_relocation_entry *relocs;

	/**

	 * Array of info structs corresponding to relocs[i].target_handle etc

	 */

	drm_intel_reloc_target *reloc_target_info;

	/** Number of entries in relocs */

	int reloc_count;

	/** Array of BOs that are referenced by this buffer and will be softpinned */

	drm_intel_bo **softpin_target;

	/** Number softpinned BOs that are referenced by this buffer */

	int softpin_target_count;

	/** Maximum amount of softpinned BOs that are referenced by this buffer */

	int softpin_target_size;

	/** Mapped address for the buffer, saved across map/unmap cycles */

	void *mem_virtual;

	/** GTT virtual address for the buffer, saved across map/unmap cycles */

	void *gtt_virtual;

	/** WC CPU address for the buffer, saved across map/unmap cycles */

	void *wc_virtual;

	/**

	 * Virtual address of the buffer allocated by user, used for userptr

	 * objects only.

	 */

	void *user_virtual;

	int map_count;

	drmMMListHead vma_list;

	/** BO cache list */

	drmMMListHead head;

	/**

	 * Boolean of whether this BO and its children have been included in

	 * the current drm_intel_bufmgr_check_aperture_space() total.

	 */

	bool included_in_check_aperture;

	/**

	 * Boolean of whether this buffer has been used as a relocation

	 * target and had its size accounted for, and thus can't have any

	 * further relocations added to it.

	 */

	bool used_as_reloc_target;

	/**

	 * Boolean of whether we have encountered an error whilst building the relocation tree.

	 */

	bool has_error;

	/**

	 * Boolean of whether this buffer can be re-used

	 */

	bool reusable;

	/**

	 * Boolean of whether the GPU is definitely not accessing the buffer.

	 *

	 * This is only valid when reusable, since non-reusable

	 * buffers are those that have been shared with other

	 * processes, so we don't know their state.

	 */

	bool idle;

	/**

	 * Boolean of whether this buffer was allocated with userptr

	 */

	bool is_userptr;

	/**

	 * Size in bytes of this buffer and its relocation descendents.

	 *

	 * Used to avoid costly tree walking in

	 * drm_intel_bufmgr_check_aperture in the common case.

	 */

	int reloc_tree_size;

	/**

	 * Number of potential fence registers required by this buffer and its

	 * relocations.

	 */

	int reloc_tree_fences;

	/** Flags that we may need to do the SW_FINISH ioctl on unmap. */

	bool mapped_cpu_write;

};

static unsigned int

drm_intel_gem_estimate_batch_space(drm_intel_bo ** bo_array, int count);

static unsigned int

drm_intel_gem_compute_batch_space(drm_intel_bo ** bo_array, int count);

static int

drm_intel_gem_bo_get_tiling(drm_intel_bo *bo, uint32_t * tiling_mode,

			    uint32_t * swizzle_mode);

static int

drm_intel_gem_bo_set_tiling_internal(drm_intel_bo *bo,

				     uint32_t tiling_mode,

				     uint32_t stride);

static void drm_intel_gem_bo_unreference_locked_timed(drm_intel_bo *bo,

						      time_t time);

static void drm_intel_gem_bo_unreference(drm_intel_bo *bo);

static void drm_intel_gem_bo_free(drm_intel_bo *bo);

static inline drm_intel_bo_gem *to_bo_gem(drm_intel_bo *bo)

{

        return (drm_intel_bo_gem *)bo;

}

static unsigned long

drm_intel_gem_bo_tile_size(drm_intel_bufmgr_gem *bufmgr_gem, unsigned long size,

			   uint32_t *tiling_mode)

{

	unsigned long min_size, max_size;

	unsigned long i;

	if (*tiling_mode == I915_TILING_NONE)

		return size;

	/* 965+ just need multiples of page size for tiling */

	if (bufmgr_gem->gen >= 4)

		return ROUND_UP_TO(size, 4096);

	/* Older chips need powers of two, of at least 512k or 1M */

	if (bufmgr_gem->gen == 3) {

		min_size = 1024*1024;

		max_size = 128*1024*1024;

	} else {

		min_size = 512*1024;

		max_size = 64*1024*1024;

	}

	if (size > max_size) {

		*tiling_mode = I915_TILING_NONE;

		return size;

	}

	/* Do we need to allocate every page for the fence? */

	if (bufmgr_gem->has_relaxed_fencing)

		return ROUND_UP_TO(size, 4096);

	for (i = min_size; i < size; i <<= 1)

		;

	return i;

}

/*

 * Round a given pitch up to the minimum required for X tiling on a

 * given chip.  We use 512 as the minimum to allow for a later tiling

 * change.

 */

static unsigned long

drm_intel_gem_bo_tile_pitch(drm_intel_bufmgr_gem *bufmgr_gem,

			    unsigned long pitch, uint32_t *tiling_mode)

{

	unsigned long tile_width;

	unsigned long i;

	/* If untiled, then just align it so that we can do rendering

	 * to it with the 3D engine.

	 */

	if (*tiling_mode == I915_TILING_NONE)

		return ALIGN(pitch, 64);

	if (*tiling_mode == I915_TILING_X

			|| (IS_915(bufmgr_gem->pci_device)

			    && *tiling_mode == I915_TILING_Y))

		tile_width = 512;

	else

		tile_width = 128;

	/* 965 is flexible */

	if (bufmgr_gem->gen >= 4)

		return ROUND_UP_TO(pitch, tile_width);

	/* The older hardware has a maximum pitch of 8192 with tiled

	 * surfaces, so fallback to untiled if it's too large.

	 */

	if (pitch > 8192) {

		*tiling_mode = I915_TILING_NONE;

		return ALIGN(pitch, 64);

	}

	/* Pre-965 needs power of two tile width */

	for (i = tile_width; i < pitch; i <<= 1)

		;

	return i;

}

static struct drm_intel_gem_bo_bucket *

drm_intel_gem_bo_bucket_for_size(drm_intel_bufmgr_gem *bufmgr_gem,

				 unsigned long size)

{

	int i;

	for (i = 0; i < bufmgr_gem->num_buckets; i++) {

		struct drm_intel_gem_bo_bucket *bucket =

		    &bufmgr_gem->cache_bucket[i];

		if (bucket->size >= size) {

			return bucket;

		}

	}

	return NULL;

}

static void

drm_intel_gem_dump_validation_list(drm_intel_bufmgr_gem *bufmgr_gem)

{

	int i, j;

	for (i = 0; i < bufmgr_gem->exec_count; i++) {

		drm_intel_bo *bo = bufmgr_gem->exec_bos[i];

		drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

		if (bo_gem->relocs == NULL && bo_gem->softpin_target == NULL) {

			DBG("%2d: %d %s(%s)\n", i, bo_gem->gem_handle,

			    bo_gem->kflags & EXEC_OBJECT_PINNED ? "*" : "",

			    bo_gem->name);

			continue;

		}

		for (j = 0; j < bo_gem->reloc_count; j++) {

			drm_intel_bo *target_bo = bo_gem->reloc_target_info[j].bo;

			drm_intel_bo_gem *target_gem =

			    (drm_intel_bo_gem *) target_bo;

			DBG("%2d: %d %s(%s)@0x%08x %08x -> "

			    "%d (%s)@0x%08x %08x + 0x%08x\n",

			    i,

			    bo_gem->gem_handle,

			    bo_gem->kflags & EXEC_OBJECT_PINNED ? "*" : "",

			    bo_gem->name,

			    upper_32_bits(bo_gem->relocs[j].offset),

			    lower_32_bits(bo_gem->relocs[j].offset),

			    target_gem->gem_handle,

			    target_gem->name,

			    upper_32_bits(target_bo->offset64),

			    lower_32_bits(target_bo->offset64),

			    bo_gem->relocs[j].delta);

		}

		for (j = 0; j < bo_gem->softpin_target_count; j++) {

			drm_intel_bo *target_bo = bo_gem->softpin_target[j];

			drm_intel_bo_gem *target_gem =

			    (drm_intel_bo_gem *) target_bo;

			DBG("%2d: %d %s(%s) -> "

			    "%d *(%s)@0x%08x %08x\n",

			    i,

			    bo_gem->gem_handle,

			    bo_gem->kflags & EXEC_OBJECT_PINNED ? "*" : "",

			    bo_gem->name,

			    target_gem->gem_handle,

			    target_gem->name,

			    upper_32_bits(target_bo->offset64),

			    lower_32_bits(target_bo->offset64));

		}

	}

}

static inline void

drm_intel_gem_bo_reference(drm_intel_bo *bo)

{

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	atomic_inc(&bo_gem->refcount);

}

/**

 * Adds the given buffer to the list of buffers to be validated (moved into the

 * appropriate memory type) with the next batch submission.

 *

 * If a buffer is validated multiple times in a batch submission, it ends up

 * with the intersection of the memory type flags and the union of the

 * access flags.

 */

static void

drm_intel_add_validate_buffer(drm_intel_bo *bo)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	int index;

	if (bo_gem->validate_index != -1)

		return;

	/* Extend the array of validation entries as necessary. */

	if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {

		int new_size = bufmgr_gem->exec_size * 2;

		if (new_size == 0)

			new_size = 5;

		bufmgr_gem->exec_objects =

		    realloc(bufmgr_gem->exec_objects,

			    sizeof(*bufmgr_gem->exec_objects) * new_size);

		bufmgr_gem->exec_bos =

		    realloc(bufmgr_gem->exec_bos,

			    sizeof(*bufmgr_gem->exec_bos) * new_size);

		bufmgr_gem->exec_size = new_size;

	}

	index = bufmgr_gem->exec_count;

	bo_gem->validate_index = index;

	/* Fill in array entry */

	bufmgr_gem->exec_objects[index].handle = bo_gem->gem_handle;

	bufmgr_gem->exec_objects[index].relocation_count = bo_gem->reloc_count;

	bufmgr_gem->exec_objects[index].relocs_ptr = (uintptr_t) bo_gem->relocs;

	bufmgr_gem->exec_objects[index].alignment = bo->align;

	bufmgr_gem->exec_objects[index].offset = 0;

	bufmgr_gem->exec_bos[index] = bo;

	bufmgr_gem->exec_count++;

}

static void

drm_intel_add_validate_buffer2(drm_intel_bo *bo, int need_fence)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *)bo->bufmgr;

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *)bo;

	int index;

	unsigned long flags;

	flags = 0;

	if (need_fence)

		flags |= EXEC_OBJECT_NEEDS_FENCE;

	if (bo_gem->validate_index != -1) {

		bufmgr_gem->exec2_objects[bo_gem->validate_index].flags |= flags;

		return;

	}

	/* Extend the array of validation entries as necessary. */

	if (bufmgr_gem->exec_count == bufmgr_gem->exec_size) {

		int new_size = bufmgr_gem->exec_size * 2;

		if (new_size == 0)

			new_size = 5;

		bufmgr_gem->exec2_objects =

			realloc(bufmgr_gem->exec2_objects,

				sizeof(*bufmgr_gem->exec2_objects) * new_size);

		bufmgr_gem->exec_bos =

			realloc(bufmgr_gem->exec_bos,

				sizeof(*bufmgr_gem->exec_bos) * new_size);

		bufmgr_gem->exec_size = new_size;

	}

	index = bufmgr_gem->exec_count;

	bo_gem->validate_index = index;

	/* Fill in array entry */

	bufmgr_gem->exec2_objects[index].handle = bo_gem->gem_handle;

	bufmgr_gem->exec2_objects[index].relocation_count = bo_gem->reloc_count;

	bufmgr_gem->exec2_objects[index].relocs_ptr = (uintptr_t)bo_gem->relocs;

	bufmgr_gem->exec2_objects[index].alignment = bo->align;

	bufmgr_gem->exec2_objects[index].offset = bo->offset64;

	bufmgr_gem->exec2_objects[index].flags = bo_gem->kflags | flags;

	bufmgr_gem->exec2_objects[index].rsvd1 = 0;

	bufmgr_gem->exec2_objects[index].rsvd2 = 0;

	bufmgr_gem->exec_bos[index] = bo;

	bufmgr_gem->exec_count++;

}

#define RELOC_BUF_SIZE(x) ((I915_RELOC_HEADER + x * I915_RELOC0_STRIDE) * \

	sizeof(uint32_t))

static void

drm_intel_bo_gem_set_in_aperture_size(drm_intel_bufmgr_gem *bufmgr_gem,

				      drm_intel_bo_gem *bo_gem,

				      unsigned int alignment)

{

	unsigned int size;

	assert(!bo_gem->used_as_reloc_target);

	/* The older chipsets are far-less flexible in terms of tiling,

	 * and require tiled buffer to be size aligned in the aperture.

	 * This means that in the worst possible case we will need a hole

	 * twice as large as the object in order for it to fit into the

	 * aperture. Optimal packing is for wimps.

	 */

	size = bo_gem->bo.size;

	if (bufmgr_gem->gen < 4 && bo_gem->tiling_mode != I915_TILING_NONE) {

		unsigned int min_size;

		if (bufmgr_gem->has_relaxed_fencing) {

			if (bufmgr_gem->gen == 3)

				min_size = 1024*1024;

			else

				min_size = 512*1024;

			while (min_size < size)

				min_size *= 2;

		} else

			min_size = size;

		/* Account for worst-case alignment. */

		alignment = MAX2(alignment, min_size);

	}

	bo_gem->reloc_tree_size = size + alignment;

}

static int

drm_intel_setup_reloc_list(drm_intel_bo *bo)

{

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;

	unsigned int max_relocs = bufmgr_gem->max_relocs;

	if (bo->size / 4 < max_relocs)

		max_relocs = bo->size / 4;

	bo_gem->relocs = malloc(max_relocs *

				sizeof(struct drm_i915_gem_relocation_entry));

	bo_gem->reloc_target_info = malloc(max_relocs *

					   sizeof(drm_intel_reloc_target));

	if (bo_gem->relocs == NULL || bo_gem->reloc_target_info == NULL) {

		bo_gem->has_error = true;

		free (bo_gem->relocs);

		bo_gem->relocs = NULL;

		free (bo_gem->reloc_target_info);

		bo_gem->reloc_target_info = NULL;

		return 1;

	}

	return 0;

}

static int

drm_intel_gem_bo_busy(drm_intel_bo *bo)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bo->bufmgr;

	drm_intel_bo_gem *bo_gem = (drm_intel_bo_gem *) bo;

	struct drm_i915_gem_busy busy;

	int ret;

	if (bo_gem->reusable && bo_gem->idle)

		return false;

	memclear(busy);

	busy.handle = bo_gem->gem_handle;

	ret = drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);

	if (ret == 0) {

		bo_gem->idle = !busy.busy;

		return busy.busy;

	} else {

		return false;

	}

}

static int

drm_intel_gem_bo_madvise_internal(drm_intel_bufmgr_gem *bufmgr_gem,

				  drm_intel_bo_gem *bo_gem, int state)

{

	struct drm_i915_gem_madvise madv;

	memclear(madv);

	madv.handle = bo_gem->gem_handle;

	madv.madv = state;

	madv.retained = 1;

	drmIoctl(bufmgr_gem->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);

	return madv.retained;

}

static int

drm_intel_gem_bo_madvise(drm_intel_bo *bo, int madv)

{

	return drm_intel_gem_bo_madvise_internal

		((drm_intel_bufmgr_gem *) bo->bufmgr,

		 (drm_intel_bo_gem *) bo,

		 madv);

}

/* drop the oldest entries that have been purged by the kernel */

static void

drm_intel_gem_bo_cache_purge_bucket(drm_intel_bufmgr_gem *bufmgr_gem,

				    struct drm_intel_gem_bo_bucket *bucket)

{

	while (!DRMLISTEMPTY(&bucket->head)) {

		drm_intel_bo_gem *bo_gem;

		bo_gem = DRMLISTENTRY(drm_intel_bo_gem,

				      bucket->head.next, head);

		if (drm_intel_gem_bo_madvise_internal

		    (bufmgr_gem, bo_gem, I915_MADV_DONTNEED))

			break;

		DRMLISTDEL(&bo_gem->head);

		drm_intel_gem_bo_free(&bo_gem->bo);

	}

}

static drm_intel_bo *

drm_intel_gem_bo_alloc_internal(drm_intel_bufmgr *bufmgr,

				const char *name,

				unsigned long size,

				unsigned long flags,

				uint32_t tiling_mode,

				unsigned long stride,

				unsigned int alignment)

{

	drm_intel_bufmgr_gem *bufmgr_gem = (drm_intel_bufmgr_gem *) bufmgr;

	drm_intel_bo_gem *bo_gem;

	unsigned int page_size = getpagesize();

	int ret;

	struct drm_intel_gem_bo_bucket *bucket;

	bool alloc_from_cache;

	unsigned long bo_size;

	bool for_render = false;

	if (flags & BO_ALLOC_FOR_RENDER)

		for_render = true;

	/* Round the allocated size up to a power of two number of pages. */

	bucket = drm_intel_gem_bo_bucket_for_size(bufmgr_gem, size);

	/* If we don't have caching at this size, don't actually round the

	 * allocation up.

	 */

	if (bucket == NULL) {

		bo_size = size;

		if (bo_size < page_size)

			bo_size = page_size;

	} else {

		bo_size = bucket->size;

	}

	pthread_mutex_lock(&bufmgr_gem->lock);

	/* Get a buffer out of the cache if available */

retry:

	alloc_from_cache = false;

	if (bucket != NULL && !DRMLISTEMPTY(&bucket->head)) {

		if (for_render) {

			/* Allocate new render-target BOs from the tail (MRU)

			 * of the list, as it will likely be hot in the GPU

			 * cache and in the aperture for us.

			 */

			bo_gem = DRMLISTENTRY(drm_intel_bo_gem,

					      bucket->head.prev, head);

			DRMLISTDEL(&bo_gem->head);

			alloc_from_cache = true;

			bo_gem->bo.align = alignment;

		} else {

			assert(alignment == 0);

			/* For non-render-target BOs (where we're probably

			 * going to map it first thing in order to fill it

			 * with data), check if the last BO in the cache is

			 * unbusy, and only reuse in that case. Otherwise,

			 * allocating a new buffer is probably faster than

			 * waiting for the GPU to finish.

			 */

			bo_gem = DRMLISTENTRY(drm_intel_bo_gem,

					      bucket->head.next, head);

			if (!drm_intel_gem_bo_busy(&bo_gem->bo)) {

				alloc_from_cache = true;

				DRMLISTDEL(&bo_gem->head);

			}

		}

		if (alloc_from_cache) {

			if (!drm_intel_gem_bo_madvise_internal

			    (bufmgr_gem, bo_gem, I915_MADV_WILLNEED)) {

				drm_intel_gem_bo_free(&bo_gem->bo);

				drm_intel_gem_bo_cache_purge_bucket(bufmgr_gem,

								    bucket);

				goto retry;

			}

			if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,

								 tiling_mode,

								 stride)) {

				drm_intel_gem_bo_free(&bo_gem->bo);

				goto retry;

			}

		}

	}

	if (!alloc_from_cache) {

		struct drm_i915_gem_create create;

		bo_gem = calloc(1, sizeof(*bo_gem));

		if (!bo_gem)

			goto err;

		/* drm_intel_gem_bo_free calls DRMLISTDEL() for an uninitialized

		   list (vma_list), so better set the list head here */

		DRMINITLISTHEAD(&bo_gem->vma_list);

		bo_gem->bo.size = bo_size;

		memclear(create);

		create.size = bo_size;

		ret = drmIoctl(bufmgr_gem->fd,

			       DRM_IOCTL_I915_GEM_CREATE,

			       &create);

		if (ret != 0) {

			free(bo_gem);

			goto err;

		}

		bo_gem->gem_handle = create.handle;

		HASH_ADD(handle_hh, bufmgr_gem->handle_table,

			 gem_handle, sizeof(bo_gem->gem_handle),

			 bo_gem);

		bo_gem->bo.handle = bo_gem->gem_handle;

		bo_gem->bo.bufmgr = bufmgr;

		bo_gem->bo.align = alignment;

		bo_gem->tiling_mode = I915_TILING_NONE;

		bo_gem->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;

		bo_gem->stride = 0;

		if (drm_intel_gem_bo_set_tiling_internal(&bo_gem->bo,

							 tiling_mode,

							 stride))

			goto err_free;

	}

	bo_gem->name = name;

	atomic_set(&bo_gem->refcount, 1);

	bo_gem->validate_index = -1;

	bo_gem->reloc_tree_fences = 0;

	bo_gem->used_as_reloc_target = false;

	bo_gem->has_error = false;

	bo_gem->reusable = true;

	drm_intel_bo_gem_set_in_aperture_size(bufmgr_gem, bo_gem, alignment);

	pthread_mutex_unlock(&bufmgr_gem->lock);

	DBG("bo_create: buf %d (%s) %ldb\n",

	    bo_gem->gem_handle, bo_gem->name, size);

	return &bo_gem->bo;

err_free:

	drm_intel_gem_bo_free(&bo_gem->bo);

err:

	pthread_mutex_unlock(&bufmgr_gem->lock);

	return NULL;

}

static drm_intel_bo *

drm_intel_gem_bo_alloc_for_render(drm_intel_bufmgr *bufmgr,

				  const char *name,

				  unsigned long size,

				  unsigned int alignment)

{

	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size,

					       BO_ALLOC_FOR_RENDER,

					       I915_TILING_NONE, 0,

					       alignment);

}

static drm_intel_bo *

drm_intel_gem_bo_alloc(drm_intel_bufmgr *bufmgr,

		       const char *name,

		       unsigned long size,

		       unsigned int alignment)

{

	return drm_intel_gem_bo_alloc_internal(bufmgr, name, size, 0,

					       I915_TILING_NONE, 0, 0);

}

static drm_intel_bo *

drm_intel_gem_bo_alloc_tiled(drm_intel_bufmgr *bufmgr, const char *name,

			     int x, int y, int cpp, uint32_t *tiling_mode,

			     unsigned long *pitch, unsigned long flags)

{