/**************************************************************************

 * 

 * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.

 * All Rights Reserved.

 * 

 * Permission is hereby granted, free of charge, to any person obtaining a

 * copy of this software and associated documentation files (the

 * "Software"), to deal in the Software without restriction, including

 * without limitation the rights to use, copy, modify, merge, publish,

 * distribute, sub license, and/or sell copies of the Software, and to

 * permit persons to whom the Software is furnished to do so, subject to

 * the following conditions:

 * 

 * The above copyright notice and this permission notice (including the

 * next paragraph) shall be included in all copies or substantial portions

 * of the Software.

 * 

 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS

 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF

 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.

 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR

 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,

 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE

 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

 * 

 **************************************************************************/

/* Originally a fake version of the buffer manager so that we can

 * prototype the changes in a driver fairly quickly, has been fleshed

 * out to a fully functional interim solution.

 *

 * Basically wraps the old style memory management in the new

 * programming interface, but is more expressive and avoids many of

 * the bugs in the old texture manager.

 */

#include <stdlib.h>

#include <string.h>

#include <assert.h>

#include <errno.h>

#include <strings.h>

#include <xf86drm.h>

#include <pthread.h>

#include "intel_bufmgr.h"

#include "intel_bufmgr_priv.h"

#include "drm.h"

#include "i915_drm.h"

#include "mm.h"

#include "libdrm_macros.h"

#include "libdrm_lists.h"

#define DBG(...) do {					\

	if (bufmgr_fake->bufmgr.debug)			\

		drmMsg(__VA_ARGS__);			\

} while (0)

/* Internal flags:

 */

#define BM_NO_BACKING_STORE			0x00000001

#define BM_NO_FENCE_SUBDATA			0x00000002

#define BM_PINNED				0x00000004

/* Wrapper around mm.c's mem_block, which understands that you must

 * wait for fences to expire before memory can be freed.  This is

 * specific to our use of memcpy for uploads - an upload that was

 * processed through the command queue wouldn't need to care about

 * fences.

 */

#define MAX_RELOCS 4096

struct fake_buffer_reloc {

	/** Buffer object that the relocation points at. */

	drm_intel_bo *target_buf;

	/** Offset of the relocation entry within reloc_buf. */

	uint32_t offset;

	/**

	 * Cached value of the offset when we last performed this relocation.

	 */

	uint32_t last_target_offset;

	/** Value added to target_buf's offset to get the relocation entry. */

	uint32_t delta;

	/** Cache domains the target buffer is read into. */

	uint32_t read_domains;

	/** Cache domain the target buffer will have dirty cachelines in. */

	uint32_t write_domain;

};

struct block {

	struct block *next, *prev;

	struct mem_block *mem;	/* BM_MEM_AGP */

	/**

	 * Marks that the block is currently in the aperture and has yet to be

	 * fenced.

	 */

	unsigned on_hardware:1;

	/**

	 * Marks that the block is currently fenced (being used by rendering)

	 * and can't be freed until @fence is passed.

	 */

	unsigned fenced:1;

	/** Fence cookie for the block. */

	unsigned fence;		/* Split to read_fence, write_fence */

	drm_intel_bo *bo;

	void *virtual;

};

typedef struct _bufmgr_fake {

	drm_intel_bufmgr bufmgr;

	pthread_mutex_t lock;

	unsigned long low_offset;

	unsigned long size;

	void *virtual;

	struct mem_block *heap;

	unsigned buf_nr;	/* for generating ids */

	/**

	 * List of blocks which are currently in the GART but haven't been

	 * fenced yet.

	 */

	struct block on_hardware;

	/**

	 * List of blocks which are in the GART and have an active fence on

	 * them.

	 */

	struct block fenced;

	/**

	 * List of blocks which have an expired fence and are ready to be

	 * evicted.

	 */

	struct block lru;

	unsigned int last_fence;

	unsigned fail:1;

	unsigned need_fence:1;

	int thrashing;

	/**

	 * Driver callback to emit a fence, returning the cookie.

	 *

	 * This allows the driver to hook in a replacement for the DRM usage in

	 * bufmgr_fake.

	 *

	 * Currently, this also requires that a write flush be emitted before

	 * emitting the fence, but this should change.

	 */

	unsigned int (*fence_emit) (void *private);

	/** Driver callback to wait for a fence cookie to have passed. */

	void (*fence_wait) (unsigned int fence, void *private);

	void *fence_priv;

	/**

	 * Driver callback to execute a buffer.

	 *

	 * This allows the driver to hook in a replacement for the DRM usage in

	 * bufmgr_fake.

	 */

	int (*exec) (drm_intel_bo *bo, unsigned int used, void *priv);

	void *exec_priv;

	/** Driver-supplied argument to driver callbacks */

	void *driver_priv;

	/**

	 * Pointer to kernel-updated sarea data for the last completed user irq

	 */

	volatile int *last_dispatch;

	int fd;

	int debug;

	int performed_rendering;

} drm_intel_bufmgr_fake;

typedef struct _drm_intel_bo_fake {

	drm_intel_bo bo;

	unsigned id;		/* debug only */

	const char *name;

	unsigned dirty:1;

	/**

	 * has the card written to this buffer - we make need to copy it back

	 */

	unsigned card_dirty:1;

	unsigned int refcount;

	/* Flags may consist of any of the DRM_BO flags, plus

	 * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the

	 * first two driver private flags.

	 */

	uint64_t flags;

	/** Cache domains the target buffer is read into. */

	uint32_t read_domains;

	/** Cache domain the target buffer will have dirty cachelines in. */

	uint32_t write_domain;

	unsigned int alignment;

	int is_static, validated;

	unsigned int map_count;

	/** relocation list */

	struct fake_buffer_reloc *relocs;

	int nr_relocs;

	/**

	 * Total size of the target_bos of this buffer.

	 *

	 * Used for estimation in check_aperture.

	 */

	unsigned int child_size;

	struct block *block;

	void *backing_store;

	void (*invalidate_cb) (drm_intel_bo *bo, void *ptr);

	void *invalidate_ptr;

} drm_intel_bo_fake;

static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake,

			unsigned int fence_cookie);

#define MAXFENCE 0x7fffffff

static int

FENCE_LTE(unsigned a, unsigned b)

{

	if (a == b)

		return 1;

	if (a < b && b - a < (1 << 24))

		return 1;

	if (a > b && MAXFENCE - a + b < (1 << 24))

		return 1;

	return 0;

}

drm_public void

drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr,

					 unsigned int (*emit) (void *priv),

					 void (*wait) (unsigned int fence,

						       void *priv),

					 void *priv)

{

	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;

	bufmgr_fake->fence_emit = emit;

	bufmgr_fake->fence_wait = wait;

	bufmgr_fake->fence_priv = priv;

}

static unsigned int

_fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake)

{

	struct drm_i915_irq_emit ie;

	int ret, seq = 1;

	if (bufmgr_fake->fence_emit != NULL) {

		seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv);

		return seq;

	}

	ie.irq_seq = &seq;

	ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT,

				  &ie, sizeof(ie));

	if (ret) {

		drmMsg("%s: drm_i915_irq_emit: %d\n", __func__, ret);

		abort();

	}

	DBG("emit 0x%08x\n", seq);

	return seq;

}

static void

_fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq)

{

	struct drm_i915_irq_wait iw;

	int hw_seq, busy_count = 0;

	int ret;

	int kernel_lied;

	if (bufmgr_fake->fence_wait != NULL) {

		bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv);

		clear_fenced(bufmgr_fake, seq);

		return;

	}

	iw.irq_seq = seq;

	DBG("wait 0x%08x\n", iw.irq_seq);

	/* The kernel IRQ_WAIT implementation is all sorts of broken.

	 * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit

	 *    unsigned range.

	 * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit

	 *    signed range.

	 * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit

	 *    signed range.

	 * 4) It returns -EBUSY in 3 seconds even if the hardware is still

	 *    successfully chewing through buffers.

	 *

	 * Assume that in userland we treat sequence numbers as ints, which

	 * makes some of the comparisons convenient, since the sequence

	 * numbers are all positive signed integers.

	 *

	 * From this we get several cases we need to handle.  Here's a timeline.

	 * 0x2   0x7                                    0x7ffffff8   0x7ffffffd

	 *   |    |                                             |    |

	 * ------------------------------------------------------------

	 *

	 * A) Normal wait for hw to catch up

	 * hw_seq seq

	 *   |    |

	 * ------------------------------------------------------------

	 * seq - hw_seq = 5.  If we call IRQ_WAIT, it will wait for hw to

	 * catch up.

	 *

	 * B) Normal wait for a sequence number that's already passed.

	 * seq    hw_seq

	 *   |    |

	 * ------------------------------------------------------------

	 * seq - hw_seq = -5.  If we call IRQ_WAIT, it returns 0 quickly.

	 *

	 * C) Hardware has already wrapped around ahead of us

	 * hw_seq                                                    seq

	 *   |                                                       |

	 * ------------------------------------------------------------

	 * seq - hw_seq = 0x80000000 - 5.  If we called IRQ_WAIT, it would wait

	 * for hw_seq >= seq, which may never occur.  Thus, we want to catch

	 * this in userland and return 0.

	 *

	 * D) We've wrapped around ahead of the hardware.

	 * seq                                                      hw_seq

	 *   |                                                       |

	 * ------------------------------------------------------------

	 * seq - hw_seq = -(0x80000000 - 5).  If we called IRQ_WAIT, it would

	 * return 0 quickly because hw_seq >= seq, even though the hardware

	 * isn't caught up. Thus, we need to catch this early return in

	 * userland and bother the kernel until the hardware really does

	 * catch up.

	 *

	 * E) Hardware might wrap after we test in userland.

	 *                                                  hw_seq  seq

	 *                                                      |    |

	 * ------------------------------------------------------------

	 * seq - hw_seq = 5.  If we call IRQ_WAIT, it will likely see seq >=

	 * hw_seq and wait.  However, suppose hw_seq wraps before we make it

	 * into the kernel.  The kernel sees hw_seq >= seq and waits for 3

	 * seconds then returns -EBUSY.  This is case C).  We should catch

	 * this and then return successfully.

	 *

	 * F) Hardware might take a long time on a buffer.

	 * hw_seq seq

	 *   |    |

	 * -------------------------------------------------------------------

	 * seq - hw_seq = 5.  If we call IRQ_WAIT, if sequence 2 through 5

	 * take too long, it will return -EBUSY.  Batchbuffers in the

	 * gltestperf demo were seen to take up to 7 seconds.  We should

	 * catch early -EBUSY return and keep trying.

	 */

	do {

		/* Keep a copy of last_dispatch so that if the wait -EBUSYs

		 * because the hardware didn't catch up in 3 seconds, we can

		 * see if it at least made progress and retry.

		 */

		hw_seq = *bufmgr_fake->last_dispatch;

		/* Catch case C */

		if (seq - hw_seq > 0x40000000)

			return;

		ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT,

				      &iw, sizeof(iw));

		/* Catch case D */

		kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch <

					     -0x40000000);

		/* Catch case E */

		if (ret == -EBUSY

		    && (seq - *bufmgr_fake->last_dispatch > 0x40000000))

			ret = 0;

		/* Catch case F: Allow up to 15 seconds chewing on one buffer. */

		if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch))

			busy_count = 0;

		else

			busy_count++;

	} while (kernel_lied || ret == -EAGAIN || ret == -EINTR ||

		 (ret == -EBUSY && busy_count < 5));

	if (ret != 0) {

		drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__,

		       __LINE__, strerror(-ret));

		abort();

	}

	clear_fenced(bufmgr_fake, seq);

}

static int

_fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)

{

	/* Slight problem with wrap-around:

	 */

	return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence);

}

/**

 * Allocate a memory manager block for the buffer.

 */

static int

alloc_block(drm_intel_bo *bo)

{

	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;

	drm_intel_bufmgr_fake *bufmgr_fake =

	    (drm_intel_bufmgr_fake *) bo->bufmgr;

	struct block *block = (struct block *)calloc(sizeof *block, 1);

	unsigned int align_log2 = ffs(bo_fake->alignment) - 1;

	unsigned int sz;

	if (!block)

		return 1;

	sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1);

	block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0);

	if (!block->mem) {

		free(block);

		return 0;

	}

	DRMINITLISTHEAD(block);

	/* Insert at head or at tail??? */

	DRMLISTADDTAIL(block, &bufmgr_fake->lru);

	block->virtual = (uint8_t *) bufmgr_fake->virtual +

	    block->mem->ofs - bufmgr_fake->low_offset;

	block->bo = bo;

	bo_fake->block = block;

	return 1;

}

/* Release the card storage associated with buf:

 */

static void

free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block,

	   int skip_dirty_copy)

{

	drm_intel_bo_fake *bo_fake;

	DBG("free block %p %08x %d %d\n", block, block->mem->ofs,

	    block->on_hardware, block->fenced);

	if (!block)

		return;

	bo_fake = (drm_intel_bo_fake *) block->bo;

	if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))

		skip_dirty_copy = 1;

	if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) {

		memcpy(bo_fake->backing_store, block->virtual, block->bo->size);

		bo_fake->card_dirty = 0;

		bo_fake->dirty = 1;

	}

	if (block->on_hardware) {

		block->bo = NULL;

	} else if (block->fenced) {

		block->bo = NULL;

	} else {

		DBG("    - free immediately\n");

		DRMLISTDEL(block);

		mmFreeMem(block->mem);

		free(block);

	}

}

static void

alloc_backing_store(drm_intel_bo *bo)

{

	drm_intel_bufmgr_fake *bufmgr_fake =

	    (drm_intel_bufmgr_fake *) bo->bufmgr;

	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;

	assert(!bo_fake->backing_store);

	assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));

	bo_fake->backing_store = malloc(bo->size);

	DBG("alloc_backing - buf %d %p %lu\n", bo_fake->id,

	    bo_fake->backing_store, bo->size);

	assert(bo_fake->backing_store);

}

static void

free_backing_store(drm_intel_bo *bo)

{

	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;

	if (bo_fake->backing_store) {

		assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));

		free(bo_fake->backing_store);

		bo_fake->backing_store = NULL;

	}

}

static void

set_dirty(drm_intel_bo *bo)

{

	drm_intel_bufmgr_fake *bufmgr_fake =

	    (drm_intel_bufmgr_fake *) bo->bufmgr;

	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;

	if (bo_fake->flags & BM_NO_BACKING_STORE

	    && bo_fake->invalidate_cb != NULL)

		bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr);

	assert(!(bo_fake->flags & BM_PINNED));

	DBG("set_dirty - buf %d\n", bo_fake->id);

	bo_fake->dirty = 1;

}

static int

evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence)

{

	struct block *block, *tmp;

	DBG("%s\n", __func__);

	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {

		drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;

		if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA))

			continue;

		if (block->fence && max_fence && !FENCE_LTE(block->fence,

							    max_fence))

			return 0;

		set_dirty(&bo_fake->bo);

		bo_fake->block = NULL;

		free_block(bufmgr_fake, block, 0);

		return 1;

	}

	return 0;

}

static int

evict_mru(drm_intel_bufmgr_fake *bufmgr_fake)

{

	struct block *block, *tmp;

	DBG("%s\n", __func__);

	DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) {

		drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;

		if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA))

			continue;

		set_dirty(&bo_fake->bo);

		bo_fake->block = NULL;

		free_block(bufmgr_fake, block, 0);

		return 1;

	}

	return 0;

}

/**

 * Removes all objects from the fenced list older than the given fence.

 */

static int

clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie)

{

	struct block *block, *tmp;

	int ret = 0;

	bufmgr_fake->last_fence = fence_cookie;

	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) {

		assert(block->fenced);

		if (_fence_test(bufmgr_fake, block->fence)) {

			block->fenced = 0;

			if (!block->bo) {

				DBG("delayed free: offset %x sz %x\n",

				    block->mem->ofs, block->mem->size);

				DRMLISTDEL(block);

				mmFreeMem(block->mem);

				free(block);

			} else {

				DBG("return to lru: offset %x sz %x\n",

				    block->mem->ofs, block->mem->size);

				DRMLISTDEL(block);

				DRMLISTADDTAIL(block, &bufmgr_fake->lru);

			}

			ret = 1;

		} else {

			/* Blocks are ordered by fence, so if one fails, all

			 * from here will fail also:

			 */

			DBG("fence not passed: offset %x sz %x %d %d \n",

			    block->mem->ofs, block->mem->size, block->fence,

			    bufmgr_fake->last_fence);

			break;

		}

	}

	DBG("%s: %d\n", __func__, ret);

	return ret;

}

static void

fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)

{

	struct block *block, *tmp;

	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {

		DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n",

		    block, block->mem->size, block->mem->ofs, block->bo, fence);

		block->fence = fence;

		block->on_hardware = 0;

		block->fenced = 1;

		/* Move to tail of pending list here

		 */

		DRMLISTDEL(block);

		DRMLISTADDTAIL(block, &bufmgr_fake->fenced);

	}

	assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));

}

static int

evict_and_alloc_block(drm_intel_bo *bo)

{

	drm_intel_bufmgr_fake *bufmgr_fake =

	    (drm_intel_bufmgr_fake *) bo->bufmgr;

	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;

	assert(bo_fake->block == NULL);

	/* Search for already free memory:

	 */

	if (alloc_block(bo))

		return 1;

	/* If we're not thrashing, allow lru eviction to dig deeper into

	 * recently used textures.  We'll probably be thrashing soon:

	 */

	if (!bufmgr_fake->thrashing) {

		while (evict_lru(bufmgr_fake, 0))

			if (alloc_block(bo))

				return 1;

	}

	/* Keep thrashing counter alive?

	 */

	if (bufmgr_fake->thrashing)

		bufmgr_fake->thrashing = 20;

	/* Wait on any already pending fences - here we are waiting for any

	 * freed memory that has been submitted to hardware and fenced to

	 * become available:

	 */

	while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {

		uint32_t fence = bufmgr_fake->fenced.next->fence;

		_fence_wait_internal(bufmgr_fake, fence);

		if (alloc_block(bo))

			return 1;

	}

	if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) {

		while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {

			uint32_t fence = bufmgr_fake->fenced.next->fence;

			_fence_wait_internal(bufmgr_fake, fence);

		}

		if (!bufmgr_fake->thrashing) {

			DBG("thrashing\n");

		}

		bufmgr_fake->thrashing = 20;

		if (alloc_block(bo))

			return 1;

	}

	while (evict_mru(bufmgr_fake))

		if (alloc_block(bo))

			return 1;

	DBG("%s 0x%lx bytes failed\n", __func__, bo->size);

	return 0;

}

/***********************************************************************

 * Public functions

 */

/**

 * Wait for hardware idle by emitting a fence and waiting for it.

 */

static void

drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake)

{

	unsigned int cookie;

	cookie = _fence_emit_internal(bufmgr_fake);

	_fence_wait_internal(bufmgr_fake, cookie);

}

/**

 * Wait for rendering to a buffer to complete.

 *

 * It is assumed that the batchbuffer which performed the rendering included

 * the necessary flushing.

 */

static void

drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo)

{

	drm_intel_bufmgr_fake *bufmgr_fake =

	    (drm_intel_bufmgr_fake *) bo->bufmgr;

	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;

	if (bo_fake->block == NULL || !bo_fake->block->fenced)

		return;

	_fence_wait_internal(bufmgr_fake, bo_fake->block->fence);

}

static void

drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo)

{

	drm_intel_bufmgr_fake *bufmgr_fake =

	    (drm_intel_bufmgr_fake *) bo->bufmgr;

	pthread_mutex_lock(&bufmgr_fake->lock);

	drm_intel_fake_bo_wait_rendering_locked(bo);

	pthread_mutex_unlock(&bufmgr_fake->lock);

}

/* Specifically ignore texture memory sharing.

 *  -- just evict everything

 *  -- and wait for idle

 */

drm_public void

drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr *bufmgr)

{

	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;

	struct block *block, *tmp;

	pthread_mutex_lock(&bufmgr_fake->lock);

	bufmgr_fake->need_fence = 1;

	bufmgr_fake->fail = 0;

	/* Wait for hardware idle.  We don't know where acceleration has been

	 * happening, so we'll need to wait anyway before letting anything get

	 * put on the card again.

	 */

	drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);

	/* Check that we hadn't released the lock without having fenced the last

	 * set of buffers.

	 */

	assert(DRMLISTEMPTY(&bufmgr_fake->fenced));

	assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));

	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {

		assert(_fence_test(bufmgr_fake, block->fence));

		set_dirty(block->bo);

	}

	pthread_mutex_unlock(&bufmgr_fake->lock);

}

static drm_intel_bo *

drm_intel_fake_bo_alloc(drm_intel_bufmgr *bufmgr,

			const char *name,

			unsigned long size,

			unsigned int alignment)

{

	drm_intel_bufmgr_fake *bufmgr_fake;

	drm_intel_bo_fake *bo_fake;

	bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;

	assert(size != 0);

	bo_fake = calloc(1, sizeof(*bo_fake));

	if (!bo_fake)

		return NULL;

	bo_fake->bo.size = size;

	bo_fake->bo.offset = -1;

	bo_fake->bo.virtual = NULL;

	bo_fake->bo.bufmgr = bufmgr;

	bo_fake->refcount = 1;

	/* Alignment must be a power of two */

	assert((alignment & (alignment - 1)) == 0);

	if (alignment == 0)

		alignment = 1;

	bo_fake->alignment = alignment;

	bo_fake->id = ++bufmgr_fake->buf_nr;

	bo_fake->name = name;

	bo_fake->flags = 0;

	bo_fake->is_static = 0;

	DBG("drm_bo_alloc: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name,

	    bo_fake->bo.size / 1024);

	return &bo_fake->bo;

}

static drm_intel_bo *

drm_intel_fake_bo_alloc_tiled(drm_intel_bufmgr * bufmgr,

			      const char *name,

			      int x, int y, int cpp,

			      uint32_t *tiling_mode,

			      unsigned long *pitch,

			      unsigned long flags)

{

	unsigned long stride, aligned_y;

	/* No runtime tiling support for fake. */

	*tiling_mode = I915_TILING_NONE;

	/* Align it for being a render target.  Shouldn't need anything else. */

	stride = x * cpp;

	stride = ROUND_UP_TO(stride, 64);

	/* 965 subspan loading alignment */

	aligned_y = ALIGN(y, 2);

	*pitch = stride;

	return drm_intel_fake_bo_alloc(bufmgr, name, stride * aligned_y,

				       4096);

}

drm_public drm_intel_bo *

drm_intel_bo_fake_alloc_static(drm_intel_bufmgr *bufmgr,

			       const char *name,

			       unsigned long offset,

			       unsigned long size, void *virtual)

{

	drm_intel_bufmgr_fake *bufmgr_fake;

	drm_intel_bo_fake *bo_fake;

	bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;

	assert(size != 0);

	bo_fake = calloc(1, sizeof(*bo_fake));

	if (!bo_fake)

		return NULL;

	bo_fake->bo.size = size;