It uses architecture proposed by the content streaming sample in DirectX 9&10. More...

#include <AsyncLoader.h>

Inheritance diagram for ParaEngine::CAsyncLoader:

Public Types
enum	AssetLogLevelEnum { Log_All = 0, Log_Debug, Log_Remote, Log_Warn, Log_Error }

typedef boost::shared_ptr< boost::thread >	Boost_Thread_ptr_type

Public Types inherited from ParaEngine::IAttributeFields
typedef ParaEngine::weak_ptr< IObject, IAttributeFields >	WeakPtr_type

Public Types inherited from ParaEngine::IObject
typedef ParaEngine::weak_ptr< IObject >	WeakPtr_type

Public Member Functions
	ATTRIBUTE_DEFINE_CLASS (CAsyncLoader)

virtual int	InstallFields (CAttributeClass *pClass, bool bOverride)
	this class should be implemented if one wants to add new attribute. More...

	ATTRIBUTE_METHOD1 (CAsyncLoader, GetEstimatedSizeInBytes_s, int *)

	ATTRIBUTE_METHOD1 (CAsyncLoader, GetItemsLeft_s, int *)

	ATTRIBUTE_METHOD1 (CAsyncLoader, GetBytesProcessed_s, int *)

	ATTRIBUTE_METHOD1 (CAsyncLoader, SetWorkerThreads_s, Vector2)

	ATTRIBUTE_METHOD1 (CAsyncLoader, SetProcessorQueueSize_s, Vector2)

	ATTRIBUTE_METHOD1 (CAsyncLoader, GetLogLevel_s, int *)

	ATTRIBUTE_METHOD1 (CAsyncLoader, SetLogLevel_s, int)

	ATTRIBUTE_METHOD1 (CAsyncLoader, log_s, const char *)

	ATTRIBUTE_METHOD (CAsyncLoader, WaitForAllItems_s)

void	CleanUp ()
	clean up everything, exit all threads created. More...

int	Start (int nWorkerCount=5)
	call this only once to start async loader More...

int	Stop ()
	stop everything. More...

int	AddWorkItem (IDataLoader pDataLoader, IDataProcessor pDataProcessor, HRESULT pHResult, void *ppDeviceObject, int nProcessorThreadID=0)
	Add a work item to the queue of work items Only call this from graphics thread @ param nProcessorThreadID: default to 0. More...

int	AddWorkItem (ResourceRequest_ptr &request)
	Add a work item to the queue of work items Only call this from graphics thread.

HRESULT	RunWorkItem (IDataLoader pDataLoader, IDataProcessor pDataProcessor, HRESULT pHResult, void *ppDeviceObject)
	this is same as AddWorkItem, except that it is a synchronous function. More...

HRESULT	RunWorkItem (ResourceRequest_ptr &request)
	this is same as AddWorkItem, except that it is a synchronous function. More...

void	ProcessDeviceWorkItems (int CurrentNumResourcesToService=100, bool bRetryLoads=false)
	ProcessDeviceWorkItems is called by the graphics thread. More...

void	ProcessDeviceWorkItemImp (ResourceRequest_ptr &pResourceRequest, bool bRetryLoads=false)

bool	CreateWorkerThreads (int nProcessorQueueID, int nMaxCount)
	make sure that there are nMaxCount workers threads processing the queue at nProcessorQueueID. More...

int	GetWorkerThreadsCount (int nProcessorQueueID)

void	SetProcessorQueueSize (int nProcessorQueueID, int nSize)
	message queue size of a given processor id

int	GetProcessorQueueSize (int nProcessorQueueID)

void	WaitForAllItems ()
	Wait for all work in the queues to finish. More...

void	AddPendingRequest (const char *sURL)
	add a string url to a set. More...

bool	HasPendingRequest (const char *sURL)
	whether the given request is already in the pending set. More...

void	RemovePendingRequest (const char *sURL)
	remove a string url from a set. More...

void	ClearAllPendingRequests ()
	clear all pending request. More...

int	GetLogLevel () const

void	SetLogLevel (int val)

void	log (const string &msg)
	write formated text to "asset.log". More...

void	log (int nLogLevel, const string &msg)

bool	interruption_requested ()
	this is a global interrupt signal. More...

void	Interrupt ()
	call this function to interrupt all threads. More...

int	GetItemsLeft (int nItemType=-1)
	Get the number of items left in the asynchronous content loader queue. More...

int	GetEstimatedSizeInBytes ()
	get total estimated size in bytes.

int	GetBytesProcessed (int nItemType=-1)
	get the total number of bytes processed. More...

Public Member Functions inherited from ParaEngine::IAttributeFields
virtual int	GetAttributeClassID ()
	attribute class ID should be identical, unless one knows how overriding rules work. More...

virtual const char *	GetAttributeClassName ()
	a static string, describing the attribute class object's name

virtual const char *	GetAttributeClassDescription ()
	a static string, describing the attribute class object

virtual int	GetChildAttributeColumnCount ()
	we support multi-dimensional child object. More...

virtual int	GetChildAttributeObjectCount (int nColumnIndex=0)
	get the number of child objects (row count) in the given column. More...

virtual IAttributeFields *	GetChildAttributeObject (int nRowIndex, int nColumnIndex=0)

virtual IAttributeFields *	GetChildAttributeObject (const std::string &sName)
	get attribute by child object. More...

	ATTRIBUTE_METHOD1 (IAttributeFields, GetName_s, const char **)

	ATTRIBUTE_METHOD1 (IAttributeFields, SetName_s, const char *)

	ATTRIBUTE_METHOD (IAttributeFields, PrintObject_s)

	ATTRIBUTE_METHOD (IAttributeFields, AddRef_s)

	ATTRIBUTE_METHOD1 (IAttributeFields, GetRefCount_s, int *)

	ATTRIBUTE_METHOD1 (IAttributeFields, SetTime_s, int)

	ATTRIBUTE_METHOD1 (IAttributeFields, GetTime_s, int *)

	ATTRIBUTE_METHOD (IAttributeFields, Release_s)

virtual const std::string &	GetIdentifier ()

virtual void	SetIdentifier (const std::string &sID)

virtual bool	IsModified ()
	whether some of the fields are modified.It is up to the implementation class to provide this functionality if necessary. More...

virtual void	SetModified (bool bModified)
	set whether any field has been modified. More...

virtual bool	ValidateFields ()
	validate all fields and return true if validation passed. More...

virtual string	GetValidationMessage ()
	get the recent validation message due to the most recent call to ValidateFields()

virtual bool	ResetField (int nFieldID)
	Reset the field to its initial or default value. More...

virtual bool	InvokeEditor (int nFieldID, const std::string &sParameters)
	Invoke an (external) editor for a given field. More...

virtual bool	AddChildAttributeObject (IAttributeFields *pChild, int nRowIndex=-1, int nColumnIndex=0)
	add child object. More...

virtual void *	QueryObjectByName (const std::string &sObjectType)
	convert to object of a given type. More...

virtual void *	QueryObject (int nObjectType)
	convert to object of a given type. More...

virtual int	GetTime ()
	get the current local time in case it is animated in milli seconds frames. More...

virtual void	SetTime (int nTime)

virtual CDynamicAttributeField *	GetDynamicField (const std::string &sName)
	Get a dynamic field with a given name. More...

virtual CDynamicAttributeField *	GetDynamicField (int nIndex)
	Get a dynamic field with a given index. More...

virtual const char *	GetDynamicFieldNameByIndex (int nIndex)
	get field name by index

virtual int	GetDynamicFieldCount ()
	how many dynamic field this object currently have. More...

virtual int	SetDynamicField (const std::string &sName, const CVariable &value)
	set a dynamic field with a given name. More...

virtual int	AddDynamicField (const std::string &sName, ATTRIBUTE_FIELDTYPE dwType)
	add dynamic field and return field index

virtual void	RemoveAllDynamicFields ()
	remove all dynamic fields

virtual int	SaveDynamicFieldsToString (std::string &output)
	save only text dynamic fields to fieldname = value text strings. More...

virtual int	LoadDynamicFieldsFromString (const std::string &input)
	load only text dynamic fields from string More...

CAttributeClass *	GetAttributeClass ()
	get the main attribute class object. More...

void	PrintObject ()
	print the content of this object to a text file at temp/doc/[ClassName].txt. More...

Public Member Functions inherited from ParaEngine::IObject
virtual void	Clone (IObject *obj) const
	Clone the object's contains to a pointer. More...

virtual IObject *	Clone () const
	Clone the object's contains and return a pointer to the newly created object. More...

virtual bool	Equals (const IObject *obj) const
	Compare the object with another object. More...

virtual const IType *	GetType () const

virtual std::string	ToString () const

WeakPtr_type &	GetWeakReference ()
	get weak reference object. More...

virtual int	ProcessObjectEvent (const ObjectEvent &event)
	this function is only used to backward compatibility of ParaObject:AddEvent() function. More...

Public Member Functions inherited from ParaEngine::CRefCounted
void	addref () const
	add reference count of the object. More...

bool	delref () const
	decrease reference count of the object. More...

int	GetRefCount () const
	get the reference count

virtual int	Release ()

CRefCounted *	AddToAutoReleasePool ()
	addref and releases the ownership sometime soon automatically (usually at the end of the current frame). More...

Static Public Member Functions
static CAsyncLoader &	GetSingleton ()
	get singleton instance. More...

Static Public Member Functions inherited from ParaEngine::IAttributeFields
static HRESULT	GetAttributeClassID_s (IAttributeFields cls, int p1)

static HRESULT	GetAttributeClassName_s (IAttributeFields cls, const char *p1)

static bool	OpenWithDefaultEditor (const char *sFilename, bool bWaitOnReturn=false)
	Open a given file with the default registered editor in the game engine. More...

Protected Member Functions
int	FileIOThreadProc ()
	This is the one IO threadproc. More...

int	FileIOThreadProc_HandleRequest (ResourceRequest_ptr &ResourceRequest)
	this is usually called by FileIOThreadProc(), but may be called by other thread as well if IsDeviceObject() is false. More...

int	ProcessingThreadProc (ProcessorWorkerThread *pThreadData)
	This is the threadproc for the processing thread. More...

Protected Member Functions inherited from ParaEngine::IAttributeFields
virtual CAttributeClass *	CreateAttributeClass ()
	initialize fields

virtual CDynamicAttributesSet *	GetDynamicAttributes (bool bCreateIfNotExist=false)

Additional Inherited Members
Protected Attributes inherited from ParaEngine::IObject
WeakPtr_type	m_weak_reference

Protected Attributes inherited from ParaEngine::CRefCounted
int	m_refcount

Detailed Description

It uses architecture proposed by the content streaming sample in DirectX 9&10.

Three types of threads come into play.

The first is the render thread. The render thread's main job is to draw the scene without stalling. Lesser jobs of the rendering thread include sending I/O requests to the I/O thread by way of the request queue, and handling lock and unlock requests from the I/O thread. To keep the render thread from stalling, both of these lesser jobs are handled when it is convenient for the render thread.
The second type of thread is the I/O thread. The I/O thread handles requests from the render thread. It also either loads files from the packed file or gets memory-mapped pointers to them. The I/O thread handles communications with the various process threads and also places lock/unlock requests into the lock/unlock queue for consumption by the render thread. In addition, on Direct3D 9, the I/O thread does the actual copying of bits into the device object. There is generally one I/O thread for each storage device that will be used on the system.
The last type of thread is the process thread. Process threads handle the dirty work of processing the data after it has been loaded from disk by the I/O thread. Process threads can decompress data, prefetch data when using memory-mapped I/O, or handle other operations, such as swizzling, that may need to be performed on the data before it is copied into the device object. There should generally be a single process thread for each available hardware thread.

The typical chain of events is as follows: 1.The I/O thread and the process threads are sleeping, waiting for something to do. The graphics thread is rendering away. 2.The graphics thread determines that a resource is going to be visible sometime in the near future. It places a resource request on the request queue. 3.The I/O thread picks this request up and loads the data from the packed file or download from network. 4.The I/O thread then sends a process request to one of the waiting process threads. The processing thread performs some computation on the data. Such as checking for integrity and decompression. 5.When the process thread is done processing, it puts a lock request into the lock/unlock queue of the render thread. 6.When the render thread feels like it, it picks up the lock request, and locks the resource. 7.The render thread then sends the request back to the I/O thread. On Direct3D 9, the I/O thread copies the data into the device object. 8.Then the I/O thread puts an unlock request in the lock/unlock queue. 9.When the render thread feels like it, it picks up the unlock request and unlocks the resource. The resource is now ready to use.

Member Function Documentation

§ AddPendingRequest()

void ParaEngine::CAsyncLoader::AddPendingRequest ( const char * sURL )

add a string url to a set.

this is used for tracking in processing requests. this function is called automatically when AppendURLRequest is called. [thread safe]

§ AddWorkItem()

int ParaEngine::CAsyncLoader::AddWorkItem	(	IDataLoader *	pDataLoader,
		IDataProcessor *	pDataProcessor,
		HRESULT *	pHResult,
		void **	ppDeviceObject,
		int	nProcessorThreadID = `0`
	)

Add a work item to the queue of work items Only call this from graphics thread @ param nProcessorThreadID: default to 0.

it can also be ResourceRequestID

§ CleanUp()

void ParaEngine::CAsyncLoader::CleanUp ( )

clean up everything, exit all threads created.

§ ClearAllPendingRequests()

void ParaEngine::CAsyncLoader::ClearAllPendingRequests ( )

clear all pending request.

[thread safe]

§ CreateWorkerThreads()

bool ParaEngine::CAsyncLoader::CreateWorkerThreads	(	int	nProcessorQueueID,
		int	nMaxCount
	)

make sure that there are nMaxCount workers threads processing the queue at nProcessorQueueID.

Note: : worker threads of internal queues are created internally. [NOTE thread safe] function must be called by the main thread.

Parameters

nProcessorQueueID	[0, 16). And each processor thread can be associated with just one processor queue. Please note that, following are internal queues: queue[0] is for local CPU intensive tasks like unzip. (only one thread process it) queue[1] is for remote background asset loading. (4 threads process it) queue[2] is for remote REST URL request. (2 threads process it)
nMaxCount	the max number of threads that can coexist for the nProcessorQueueID queue.

Returns: true if success.

§ FileIOThreadProc()

int CAsyncLoader::FileIOThreadProc ( )

protected

This is the one IO threadproc.

This function is responsible for processing read requests made by the application. There should only be one IO thread per device. This ensures that the disk is only trying to read one part of the disk at a time.

This thread performs double-duty as the copy thread as well. It manages the copying of resource data from temporary system memory buffer (or memory mapped pointer) into the locked data of the resource.

§ FileIOThreadProc_HandleRequest()

int CAsyncLoader::FileIOThreadProc_HandleRequest ( ResourceRequest_ptr & ResourceRequest )

protected

this is usually called by FileIOThreadProc(), but may be called by other thread as well if IsDeviceObject() is false.

§ GetBytesProcessed()

int ParaEngine::CAsyncLoader::GetBytesProcessed ( int nItemType = -1 )

get the total number of bytes processed.

Parameters

nItemType -2 : all remote assets(excluding HTTP web requests) in the queue. -1 : all url requests(include web request) in the queue. 0 : all local requests in the queue. 1 : Smaller remote requests in the queue. 2 : HTTP web requests in the queue. 3 : big remote requests in the queue.

§ GetItemsLeft()

int ParaEngine::CAsyncLoader::GetItemsLeft ( int nItemType = -1 )

Get the number of items left in the asynchronous content loader queue.

Parameters

nItemType -2 : all remote assets(excluding HTTP web requests) in the queue. -1 : all outstanding resources in the queue. 0 : all local requests in the queue. 1 : Smaller remote requests in the queue. 2 : HTTP web requests in the queue. 3 : big remote requests in the queue.

§ GetSingleton()

CAsyncLoader & ParaEngine::CAsyncLoader::GetSingleton ( )

static

get singleton instance.

one needs to call start() before adding work items.

§ HasPendingRequest()

bool ParaEngine::CAsyncLoader::HasPendingRequest ( const char * sURL )

whether the given request is already in the pending set.

one can use this function to check whether there is a pending request due to previous call of AppendURLRequest [thread safe]

§ InstallFields()

int ParaEngine::CAsyncLoader::InstallFields	(	CAttributeClass *	pClass,
		bool	bOverride
	)

virtual

this class should be implemented if one wants to add new attribute.

This function is always called internally.

Reimplemented from ParaEngine::IAttributeFields.

§ Interrupt()

void ParaEngine::CAsyncLoader::Interrupt ( )

call this function to interrupt all threads.

This function returns immediately. Usually we call this function first, and then call thread.timed_join() or join().

§ interruption_requested()

bool ParaEngine::CAsyncLoader::interruption_requested ( )

inline

this is a global interrupt signal.

Once set, all thread should try to exit ASAP

§ log()

void ParaEngine::CAsyncLoader::log ( const string & msg )

write formated text to "asset.log".

the input is same to printf

§ ProcessDeviceWorkItems()

void ParaEngine::CAsyncLoader::ProcessDeviceWorkItems	(	int	CurrentNumResourcesToService = `100`,
		bool	bRetryLoads = `false`
	)

ProcessDeviceWorkItems is called by the graphics thread.

Depending on the request it either Locks or Unlocks a resource (or calls UpdateSubresource for D3D10). Only call this from graphics thread at regular interval such as in the render frame move.

Parameters

CurrentNumResourcesToService	the number of resources to service. This ensure that no matter how many items are in the queue, the graphics thread doesn't stall trying to process all of them.
bRetryLoads	if true, we will retry if failed and retry is requested by the loader.

§ ProcessingThreadProc()

int CAsyncLoader::ProcessingThreadProc ( ProcessorWorkerThread * pThreadData )

protected

This is the threadproc for the processing thread.

There are multiple processing threads. The job of the processing thread is to uncompress, unpack, or otherwise manipulate the data loaded by the loading thread in order to get it ready for the ProcessDeviceWorkItems function in the graphics thread to lock or unlock the resource.

Parameters

pThreadData some thread local data, such as which process message queue this thread should read from.

§ RemovePendingRequest()

void ParaEngine::CAsyncLoader::RemovePendingRequest ( const char * sURL )

remove a string url from a set.

This function is automatically called when a url request is completed. [thread safe]

§ RunWorkItem() [1/2]

HRESULT ParaEngine::CAsyncLoader::RunWorkItem	(	IDataLoader *	pDataLoader,
		IDataProcessor *	pDataProcessor,
		HRESULT *	pHResult,
		void **	ppDeviceObject
	)

this is same as AddWorkItem, except that it is a synchronous function.

and will only return after everything is processed.

Note: : use AddWorkItem instead if possible.

§ RunWorkItem() [2/2]

HRESULT ParaEngine::CAsyncLoader::RunWorkItem ( ResourceRequest_ptr & request )

this is same as AddWorkItem, except that it is a synchronous function.

and will only return after everything is processed.

Note: : use AddWorkItem instead if possible.

§ Start()

int CAsyncLoader::Start ( int nWorkerCount = 5 )

call this only once to start async loader

Parameters

nWorkerCount how many worker threads to spawn.

§ Stop()

int CAsyncLoader::Stop ( void )

stop everything.

this is like cleaning up.

§ WaitForAllItems()

void ParaEngine::CAsyncLoader::WaitForAllItems ( )

Wait for all work in the queues to finish.

Only call this from graphics thread

The documentation for this class was generated from the following files:

Client/trunk/ParaEngineClient/IO/AsyncLoader.h
Client/trunk/ParaEngineClient/IO/AsyncLoader.cpp

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Member Function Documentation

§ AddPendingRequest()

§ AddWorkItem()

§ CleanUp()

§ ClearAllPendingRequests()

§ CreateWorkerThreads()

§ FileIOThreadProc()

§ FileIOThreadProc_HandleRequest()

§ GetBytesProcessed()

§ GetItemsLeft()

§ GetSingleton()

§ HasPendingRequest()

§ InstallFields()

§ Interrupt()

§ interruption_requested()

§ log()

§ ProcessDeviceWorkItems()

§ ProcessingThreadProc()

§ RemovePendingRequest()

§ RunWorkItem() [1/2]

§ RunWorkItem() [2/2]

§ Start()

§ Stop()

§ WaitForAllItems()