/*************************************************************************/ /*! @Title Kernel side command queue functions @Copyright Copyright (c) Imagination Technologies Ltd. All Rights Reserved @License Dual MIT/GPLv2 The contents of this file are subject to the MIT license as set out below. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. Alternatively, the contents of this file may be used under the terms of the GNU General Public License Version 2 ("GPL") in which case the provisions of GPL are applicable instead of those above. If you wish to allow use of your version of this file only under the terms of GPL, and not to allow others to use your version of this file under the terms of the MIT license, indicate your decision by deleting the provisions above and replace them with the notice and other provisions required by GPL as set out in the file called "GPL-COPYING" included in this distribution. If you do not delete the provisions above, a recipient may use your version of this file under the terms of either the MIT license or GPL. This License is also included in this distribution in the file called "MIT-COPYING". EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ /**************************************************************************/ #include "services_headers.h" #include "pvr_bridge_km.h" #include "lists.h" #include "ttrace.h" #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE) #include #if (LINUX_VERSION_CODE < KERNEL_VERSION(3,10,0)) #include #elif (LINUX_VERSION_CODE < KERNEL_VERSION(4,9,0)) #include <../drivers/staging/android/sw_sync.h> #else #include <../drivers/dma-buf/sync_debug.h> #endif #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE) #include "pvrsrv_sync_server.h" #endif #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) #include #include typedef struct _PVR_QUEUE_SYNC_KERNEL_SYNC_INFO_ { /* Base services sync info structure */ PVRSRV_KERNEL_SYNC_INFO *psBase; struct list_head sHead; } PVR_QUEUE_SYNC_KERNEL_SYNC_INFO; static IMG_BOOL PVRSyncIsSyncInfoInUse(PVRSRV_KERNEL_SYNC_INFO *psSyncInfo) { return !(psSyncInfo->psSyncData->ui32WriteOpsPending == psSyncInfo->psSyncData->ui32WriteOpsComplete && psSyncInfo->psSyncData->ui32ReadOpsPending == psSyncInfo->psSyncData->ui32ReadOpsComplete && psSyncInfo->psSyncData->ui32ReadOps2Pending == psSyncInfo->psSyncData->ui32ReadOps2Complete); } /* Defer Workqueue for releasing command kernel sync info */ static struct workqueue_struct *gpsWorkQueue; /* Linux work struct for workqueue. */ static struct work_struct gsWork; /* The "defer-free" sync object list. */ static LIST_HEAD(gSyncInfoFreeList); static DEFINE_SPINLOCK(gSyncInfoFreeListLock); static void PVRSyncWorkQueueFunction(struct work_struct *data) { struct list_head sFreeList, *psEntry, *n; PVR_QUEUE_SYNC_KERNEL_SYNC_INFO *psSyncInfo; INIT_LIST_HEAD(&sFreeList); spin_lock(&gSyncInfoFreeListLock); list_for_each_safe(psEntry, n, &gSyncInfoFreeList) { psSyncInfo = container_of(psEntry, PVR_QUEUE_SYNC_KERNEL_SYNC_INFO, sHead); if(!PVRSyncIsSyncInfoInUse(psSyncInfo->psBase)) list_move_tail(psEntry, &sFreeList); } spin_unlock(&gSyncInfoFreeListLock); list_for_each_safe(psEntry, n, &sFreeList) { psSyncInfo = container_of(psEntry, PVR_QUEUE_SYNC_KERNEL_SYNC_INFO, sHead); list_del(psEntry); PVRSRVKernelSyncInfoDecRef(psSyncInfo->psBase, IMG_NULL); } } #endif #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) static struct sync_fence *AllocQueueFence(struct sw_sync_timeline *psTimeline, IMG_UINT32 ui32FenceValue, const char *szName) { struct sync_fence *psFence = IMG_NULL; struct sync_pt *psPt; psPt = sw_sync_pt_create(psTimeline, ui32FenceValue); if(psPt) { psFence = sync_fence_create(szName, psPt); if(!psFence) { sync_pt_free(psPt); } } return psFence; } #endif /* defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) */ #endif /* defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE) */ /* * The number of commands of each type which can be in flight at once. */ #define DC_MAX_SUPPORTED_QUEUES 1 #if defined(SUPPORT_DC_CMDCOMPLETE_WHEN_NO_LONGER_DISPLAYED) #define DC_NUM_COMMANDS_PER_QUEUE 2 #else #define DC_NUM_COMMANDS_PER_QUEUE 1 #endif #define DC_NUM_COMMANDS_PER_TYPE (DC_NUM_COMMANDS_PER_QUEUE * DC_MAX_SUPPORTED_QUEUES) static IMG_UINT32 ui32NoOfSwapchainCreated = 0; /* * List of private command processing function pointer tables and command * complete tables for a device in the system. * Each table is allocated when the device registers its private command * processing functions. */ typedef struct _DEVICE_COMMAND_DATA_ { PFN_CMD_PROC pfnCmdProc; PCOMMAND_COMPLETE_DATA apsCmdCompleteData[DC_NUM_COMMANDS_PER_TYPE]; IMG_UINT32 ui32CCBOffset; IMG_UINT32 ui32MaxDstSyncCount; /*!< Maximum number of dest syncs */ IMG_UINT32 ui32MaxSrcSyncCount; /*!< Maximum number of source syncs */ } DEVICE_COMMAND_DATA; #if defined(__linux__) && defined(__KERNEL__) #include "proc.h" /***************************************************************************** FUNCTION : ProcSeqShowQueue PURPOSE : Print the content of queue element to /proc file (See env/linux/proc.c:CreateProcReadEntrySeq) PARAMETERS : sfile - /proc seq_file el - Element to print *****************************************************************************/ void ProcSeqShowQueue(struct seq_file *sfile,void* el) { PVRSRV_QUEUE_INFO *psQueue = (PVRSRV_QUEUE_INFO*)el; IMG_INT cmds = 0; IMG_SIZE_T uReadOffset; IMG_SIZE_T uWriteOffset; PVRSRV_COMMAND *psCmd; if(el == PVR_PROC_SEQ_START_TOKEN) { seq_printf( sfile, "Command Queues\n" "Queue CmdPtr Pid Command Size DevInd DSC SSC #Data ...\n"); return; } uReadOffset = psQueue->uReadOffset; uWriteOffset = psQueue->uWriteOffset; while (uReadOffset != uWriteOffset) { psCmd= (PVRSRV_COMMAND *)((IMG_UINTPTR_T)psQueue->pvLinQueueKM + uReadOffset); seq_printf(sfile, "%p %p %5u %6u %3" SIZE_T_FMT_LEN "u %5u %2u %2u %3" SIZE_T_FMT_LEN "u \n", psQueue, psCmd, psCmd->ui32ProcessID, psCmd->CommandType, psCmd->uCmdSize, psCmd->ui32DevIndex, psCmd->ui32DstSyncCount, psCmd->ui32SrcSyncCount, psCmd->uDataSize); { IMG_UINT32 i; for (i = 0; i < psCmd->ui32SrcSyncCount; i++) { PVRSRV_SYNC_DATA *psSyncData = psCmd->psSrcSync[i].psKernelSyncInfoKM->psSyncData; seq_printf(sfile, " Sync %u: ROP/ROC: 0x%x/0x%x WOP/WOC: 0x%x/0x%x ROC-VA: 0x%x WOC-VA: 0x%x\n", i, psCmd->psSrcSync[i].ui32ReadOps2Pending, psSyncData->ui32ReadOps2Complete, psCmd->psSrcSync[i].ui32WriteOpsPending, psSyncData->ui32WriteOpsComplete, psCmd->psSrcSync[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psCmd->psSrcSync[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr); } } /* taken from UPDATE_QUEUE_ROFF in queue.h */ uReadOffset += psCmd->uCmdSize; uReadOffset &= psQueue->uQueueSize - 1; cmds++; } if (cmds == 0) { seq_printf(sfile, "%p \n", psQueue); } } /***************************************************************************** FUNCTION : ProcSeqOff2ElementQueue PURPOSE : Transale offset to element (/proc stuff) PARAMETERS : sfile - /proc seq_file off - the offset into the buffer RETURNS : element to print *****************************************************************************/ void* ProcSeqOff2ElementQueue(struct seq_file * sfile, loff_t off) { PVRSRV_QUEUE_INFO *psQueue = IMG_NULL; SYS_DATA *psSysData; PVR_UNREFERENCED_PARAMETER(sfile); if(!off) { return PVR_PROC_SEQ_START_TOKEN; } psSysData = SysAcquireDataNoCheck(); if (psSysData != IMG_NULL) { for (psQueue = psSysData->psQueueList; (((--off) > 0) && (psQueue != IMG_NULL)); psQueue = psQueue->psNextKM); } return psQueue; } #endif /* __linux__ && __KERNEL__ */ /*! * Macro to return space in given command queue */ #define GET_SPACE_IN_CMDQ(psQueue) \ ((((psQueue)->uReadOffset - (psQueue)->uWriteOffset) \ + ((psQueue)->uQueueSize - 1)) & ((psQueue)->uQueueSize - 1)) /*! * Macro to Write Offset in given command queue */ #define UPDATE_QUEUE_WOFF(psQueue, uSize) \ (psQueue)->uWriteOffset = ((psQueue)->uWriteOffset + (uSize)) \ & ((psQueue)->uQueueSize - 1); /*! * Check if an ops complete value has gone past the pending value. * This can happen when dummy processing multiple operations, e.g. hardware recovery. */ #define SYNCOPS_STALE(ui32OpsComplete, ui32OpsPending) \ ((ui32OpsComplete) >= (ui32OpsPending)) /*! **************************************************************************** @Function : PVRSRVGetWriteOpsPending @Description : Gets the next operation to wait for in a sync object @Input : psSyncInfo - pointer to sync information struct @Input : bIsReadOp - Is this a read or write op @Return : Next op value *****************************************************************************/ #ifdef INLINE_IS_PRAGMA #pragma inline(PVRSRVGetWriteOpsPending) #endif static INLINE IMG_UINT32 PVRSRVGetWriteOpsPending(PVRSRV_KERNEL_SYNC_INFO *psSyncInfo, IMG_BOOL bIsReadOp) { IMG_UINT32 ui32WriteOpsPending; if(bIsReadOp) { ui32WriteOpsPending = psSyncInfo->psSyncData->ui32WriteOpsPending; } else { /* Note: This needs to be atomic and is provided the kernel driver is single threaded (non-rentrant) */ ui32WriteOpsPending = SyncTakeWriteOp(psSyncInfo, SYNC_OP_CLASS_QUEUE); } return ui32WriteOpsPending; } /*! ***************************************************************************** @Function : PVRSRVGetReadOpsPending @Description : Gets the number of pending read ops @Input : psSyncInfo - pointer to sync information struct @Input : bIsReadOp - Is this a read or write op @Return : Next op value *****************************************************************************/ #ifdef INLINE_IS_PRAGMA #pragma inline(PVRSRVGetReadOpsPending) #endif static INLINE IMG_UINT32 PVRSRVGetReadOpsPending(PVRSRV_KERNEL_SYNC_INFO *psSyncInfo, IMG_BOOL bIsReadOp) { IMG_UINT32 ui32ReadOpsPending; if(bIsReadOp) { ui32ReadOpsPending = SyncTakeReadOp2(psSyncInfo, SYNC_OP_CLASS_QUEUE); } else { ui32ReadOpsPending = psSyncInfo->psSyncData->ui32ReadOps2Pending; } return ui32ReadOpsPending; } static IMG_VOID QueueDumpCmdComplete(COMMAND_COMPLETE_DATA *psCmdCompleteData, IMG_UINT32 i, IMG_BOOL bIsSrc) { PVRSRV_SYNC_OBJECT *psSyncObject; psSyncObject = bIsSrc ? psCmdCompleteData->psSrcSync : psCmdCompleteData->psDstSync; if (psCmdCompleteData->bInUse) { PVR_LOG(("\t%s %u: ROC DevVAddr:0x%X ROP:0x%x ROC:0x%x, WOC DevVAddr:0x%X WOP:0x%x WOC:0x%x", bIsSrc ? "SRC" : "DEST", i, psSyncObject[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psSyncObject[i].psKernelSyncInfoKM->psSyncData->ui32ReadOps2Pending, psSyncObject[i].psKernelSyncInfoKM->psSyncData->ui32ReadOps2Complete, psSyncObject[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr, psSyncObject[i].psKernelSyncInfoKM->psSyncData->ui32WriteOpsPending, psSyncObject[i].psKernelSyncInfoKM->psSyncData->ui32WriteOpsComplete)) } else { PVR_LOG(("\t%s %u: (Not in use)", bIsSrc ? "SRC" : "DEST", i)) } } static IMG_VOID QueueDumpDebugInfo_ForEachCb(PVRSRV_DEVICE_NODE *psDeviceNode) { if (psDeviceNode->sDevId.eDeviceClass == PVRSRV_DEVICE_CLASS_DISPLAY) { IMG_UINT32 ui32CmdCounter, ui32SyncCounter; SYS_DATA *psSysData; DEVICE_COMMAND_DATA *psDeviceCommandData; PCOMMAND_COMPLETE_DATA psCmdCompleteData; SysAcquireData(&psSysData); psDeviceCommandData = psSysData->apsDeviceCommandData[psDeviceNode->sDevId.ui32DeviceIndex]; if (psDeviceCommandData != IMG_NULL) { for (ui32CmdCounter = 0; ui32CmdCounter < DC_NUM_COMMANDS_PER_TYPE; ui32CmdCounter++) { psCmdCompleteData = psDeviceCommandData[DC_FLIP_COMMAND].apsCmdCompleteData[ui32CmdCounter]; PVR_LOG(("Flip Command Complete Data %u for display device %u:", ui32CmdCounter, psDeviceNode->sDevId.ui32DeviceIndex)) for (ui32SyncCounter = 0; ui32SyncCounter < psCmdCompleteData->ui32SrcSyncCount; ui32SyncCounter++) { QueueDumpCmdComplete(psCmdCompleteData, ui32SyncCounter, IMG_TRUE); } for (ui32SyncCounter = 0; ui32SyncCounter < psCmdCompleteData->ui32DstSyncCount; ui32SyncCounter++) { QueueDumpCmdComplete(psCmdCompleteData, ui32SyncCounter, IMG_FALSE); } } } else { PVR_LOG(("There is no Command Complete Data for display device %u", psDeviceNode->sDevId.ui32DeviceIndex)) } } } IMG_VOID QueueDumpDebugInfo(IMG_VOID) { SYS_DATA *psSysData; SysAcquireData(&psSysData); List_PVRSRV_DEVICE_NODE_ForEach(psSysData->psDeviceNodeList, &QueueDumpDebugInfo_ForEachCb); } /***************************************************************************** Kernel-side functions of User->Kernel transitions ******************************************************************************/ static IMG_SIZE_T NearestPower2(IMG_SIZE_T uValue) { IMG_SIZE_T uTemp, uResult = 1; if(!uValue) return 0; uTemp = uValue - 1; while(uTemp) { uResult <<= 1; uTemp >>= 1; } return uResult; } /*! ****************************************************************************** @Function PVRSRVCreateCommandQueueKM @Description Creates a new command queue into which render/blt commands etc can be inserted. @Input uQueueSize : @Output ppsQueueInfo : @Return PVRSRV_ERROR : ******************************************************************************/ IMG_EXPORT PVRSRV_ERROR IMG_CALLCONV PVRSRVCreateCommandQueueKM(IMG_SIZE_T uQueueSize, PVRSRV_QUEUE_INFO **ppsQueueInfo) { PVRSRV_QUEUE_INFO *psQueueInfo; IMG_SIZE_T uPower2QueueSize = NearestPower2(uQueueSize); SYS_DATA *psSysData; PVRSRV_ERROR eError; IMG_HANDLE hMemBlock; if (ui32NoOfSwapchainCreated >= DC_NUM_COMMANDS_PER_TYPE) { PVR_DPF((PVR_DBG_ERROR,"PVRSRVCreateCommandQueueKM: Swapchain already exists, increment DC_MAX_SUPPORTED_QUEUES to support more than one swapchain")); return PVRSRV_ERROR_FLIP_CHAIN_EXISTS; } SysAcquireData(&psSysData); /* allocate an internal queue info structure */ eError = OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP, sizeof(PVRSRV_QUEUE_INFO), (IMG_VOID **)&psQueueInfo, &hMemBlock, "Queue Info"); if (eError != PVRSRV_OK) { PVR_DPF((PVR_DBG_ERROR,"PVRSRVCreateCommandQueueKM: Failed to alloc queue struct")); goto ErrorExit; } OSMemSet(psQueueInfo, 0, sizeof(PVRSRV_QUEUE_INFO)); psQueueInfo->hMemBlock[0] = hMemBlock; psQueueInfo->ui32ProcessID = OSGetCurrentProcessIDKM(); /* allocate the command queue buffer - allow for overrun */ eError = OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP, uPower2QueueSize + PVRSRV_MAX_CMD_SIZE, &psQueueInfo->pvLinQueueKM, &hMemBlock, "Command Queue"); if (eError != PVRSRV_OK) { PVR_DPF((PVR_DBG_ERROR,"PVRSRVCreateCommandQueueKM: Failed to alloc queue buffer")); goto ErrorExit; } psQueueInfo->hMemBlock[1] = hMemBlock; psQueueInfo->pvLinQueueUM = psQueueInfo->pvLinQueueKM; /* Sanity check: Should be zeroed by OSMemSet */ PVR_ASSERT(psQueueInfo->uReadOffset == 0); PVR_ASSERT(psQueueInfo->uWriteOffset == 0); psQueueInfo->uQueueSize = uPower2QueueSize; #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) psQueueInfo->pvTimeline = sw_sync_timeline_create("pvr_queue_proc"); if(psQueueInfo->pvTimeline == IMG_NULL) { PVR_DPF((PVR_DBG_ERROR,"PVRSRVCreateCommandQueueKM: sw_sync_timeline_create() failed")); goto ErrorExit; } #endif /* if this is the first q, create a lock resource for the q list */ if (psSysData->psQueueList == IMG_NULL) { eError = OSCreateResource(&psSysData->sQProcessResource); if (eError != PVRSRV_OK) { goto ErrorExit; } } /* Ensure we don't corrupt queue list, by blocking access */ #if !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) eError = OSLockResourceAndBlockMISR(&psSysData->sQProcessResource, KERNEL_ID); #else /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ eError = OSLockResource(&psSysData->sQProcessResource, KERNEL_ID); #endif /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ if (eError != PVRSRV_OK) { goto ErrorExit; } psQueueInfo->psNextKM = psSysData->psQueueList; psSysData->psQueueList = psQueueInfo; #if !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) eError = OSUnlockResourceAndUnblockMISR(&psSysData->sQProcessResource, KERNEL_ID); #else /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ eError = OSUnlockResource(&psSysData->sQProcessResource, KERNEL_ID); #endif /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ if (eError != PVRSRV_OK) { goto ErrorExit; } *ppsQueueInfo = psQueueInfo; #if (defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE)) && defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) if(!ui32NoOfSwapchainCreated) { gpsWorkQueue = create_freezable_workqueue("flip_pvr_sync_workqueue"); if(!gpsWorkQueue) { PVR_DPF((PVR_DBG_ERROR, "%s: Failed to create flip_pvr_sync workqueue", __func__)); goto ErrorExit; } INIT_WORK(&gsWork, PVRSyncWorkQueueFunction); } #endif ui32NoOfSwapchainCreated++; return PVRSRV_OK; ErrorExit: if(psQueueInfo) { if(psQueueInfo->pvLinQueueKM) { OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, psQueueInfo->uQueueSize, psQueueInfo->pvLinQueueKM, psQueueInfo->hMemBlock[1]); psQueueInfo->pvLinQueueKM = IMG_NULL; } OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, sizeof(PVRSRV_QUEUE_INFO), psQueueInfo, psQueueInfo->hMemBlock[0]); /*not nulling pointer, out of scope*/ } return eError; } /*! ****************************************************************************** @Function PVRSRVDestroyCommandQueueKM @Description Destroys a command queue @Input psQueueInfo : @Return PVRSRV_ERROR ******************************************************************************/ IMG_EXPORT PVRSRV_ERROR IMG_CALLCONV PVRSRVDestroyCommandQueueKM(PVRSRV_QUEUE_INFO *psQueueInfo) { PVRSRV_QUEUE_INFO *psQueue; SYS_DATA *psSysData; PVRSRV_ERROR eError; IMG_BOOL bTimeout = IMG_TRUE; SysAcquireData(&psSysData); psQueue = psSysData->psQueueList; /* PRQA S 3415,4109 1 */ /* macro format critical - leave alone */ LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US) { if(psQueueInfo->uReadOffset == psQueueInfo->uWriteOffset) { bTimeout = IMG_FALSE; break; } OSSleepms(1); } END_LOOP_UNTIL_TIMEOUT(); if (bTimeout) { /* The command queue could not be flushed within the timeout period. Allow the queue to be destroyed before returning the error code. */ PVR_DPF((PVR_DBG_ERROR,"PVRSRVDestroyCommandQueueKM : Failed to empty queue")); eError = PVRSRV_ERROR_CANNOT_FLUSH_QUEUE; goto ErrorExit; } /* Ensure we don't corrupt queue list, by blocking access */ #if !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) eError = OSLockResourceAndBlockMISR(&psSysData->sQProcessResource, KERNEL_ID); #else /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ eError = OSLockResource(&psSysData->sQProcessResource, KERNEL_ID); #endif /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ if (eError != PVRSRV_OK) { goto ErrorExit; } ui32NoOfSwapchainCreated--; #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) sync_timeline_destroy(psQueueInfo->pvTimeline); #endif if(psQueue == psQueueInfo) { psSysData->psQueueList = psQueueInfo->psNextKM; OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, NearestPower2(psQueueInfo->uQueueSize) + PVRSRV_MAX_CMD_SIZE, psQueueInfo->pvLinQueueKM, psQueueInfo->hMemBlock[1]); psQueueInfo->pvLinQueueKM = IMG_NULL; OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, sizeof(PVRSRV_QUEUE_INFO), psQueueInfo, psQueueInfo->hMemBlock[0]); /* PRQA S 3199 1 */ /* see note */ psQueueInfo = IMG_NULL; /*it's a copy on stack, but null it because the function doesn't end right here*/ } else { while(psQueue) { if(psQueue->psNextKM == psQueueInfo) { psQueue->psNextKM = psQueueInfo->psNextKM; OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, psQueueInfo->uQueueSize, psQueueInfo->pvLinQueueKM, psQueueInfo->hMemBlock[1]); psQueueInfo->pvLinQueueKM = IMG_NULL; OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, sizeof(PVRSRV_QUEUE_INFO), psQueueInfo, psQueueInfo->hMemBlock[0]); /* PRQA S 3199 1 */ /* see note */ psQueueInfo = IMG_NULL; /*it's a copy on stack, but null it because the function doesn't end right here*/ break; } psQueue = psQueue->psNextKM; } if(!psQueue) { #if !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) eError = OSUnlockResourceAndUnblockMISR(&psSysData->sQProcessResource, KERNEL_ID); #else /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ eError = OSUnlockResource(&psSysData->sQProcessResource, KERNEL_ID); #endif /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ if (eError != PVRSRV_OK) { goto ErrorExit; } eError = PVRSRV_ERROR_INVALID_PARAMS; goto ErrorExit; } } /* unlock the Q list lock resource */ #if !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) eError = OSUnlockResourceAndUnblockMISR(&psSysData->sQProcessResource, KERNEL_ID); #else /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ eError = OSUnlockResource(&psSysData->sQProcessResource, KERNEL_ID); #endif /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ if (eError != PVRSRV_OK) { goto ErrorExit; } /* if the Q list is now empty, destroy the Q list lock resource */ if (psSysData->psQueueList == IMG_NULL) { eError = OSDestroyResource(&psSysData->sQProcessResource); if (eError != PVRSRV_OK) { goto ErrorExit; } } #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) && defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) if(!ui32NoOfSwapchainCreated && gpsWorkQueue) { destroy_workqueue(gpsWorkQueue); } #endif ErrorExit: return eError; } /*! ***************************************************************************** @Function : PVRSRVGetQueueSpaceKM @Description : Waits for queue access rights and checks for available space in queue for task param structure @Input : psQueue - pointer to queue information struct @Input : ui32ParamSize - size of task data structure @Output : ppvSpace @Return : PVRSRV_ERROR *****************************************************************************/ IMG_EXPORT PVRSRV_ERROR IMG_CALLCONV PVRSRVGetQueueSpaceKM(PVRSRV_QUEUE_INFO *psQueue, IMG_SIZE_T uParamSize, IMG_VOID **ppvSpace) { /* round to 4byte units */ uParamSize = (uParamSize + 3) & 0xFFFFFFFC; if (uParamSize > PVRSRV_MAX_CMD_SIZE) { PVR_DPF((PVR_DBG_WARNING,"PVRSRVGetQueueSpace: max command size is %d bytes", PVRSRV_MAX_CMD_SIZE)); return PVRSRV_ERROR_CMD_TOO_BIG; } if (GET_SPACE_IN_CMDQ(psQueue) > uParamSize) { *ppvSpace = (IMG_VOID *)((IMG_UINTPTR_T)psQueue->pvLinQueueUM + psQueue->uWriteOffset); } else { *ppvSpace = IMG_NULL; return PVRSRV_ERROR_CANNOT_GET_QUEUE_SPACE; } return PVRSRV_OK; } /*! ***************************************************************************** @Function PVRSRVInsertCommandKM @Description : command insertion utility - waits for space in the queue for a new command - fills in generic command information - returns a pointer to the caller who's expected to then fill in the private data. The caller should follow PVRSRVInsertCommand with PVRSRVSubmitCommand which will update the queue's write offset so the command can be executed. @Input psQueue : pointer to queue information struct @Output ppvCmdData : holds pointer to space in queue for private cmd data @Return PVRSRV_ERROR *****************************************************************************/ IMG_EXPORT PVRSRV_ERROR IMG_CALLCONV PVRSRVInsertCommandKM(PVRSRV_QUEUE_INFO *psQueue, PVRSRV_COMMAND **ppsCommand, IMG_UINT32 ui32DevIndex, IMG_UINT16 CommandType, IMG_UINT32 ui32DstSyncCount, PVRSRV_KERNEL_SYNC_INFO *apsDstSync[], IMG_UINT32 ui32SrcSyncCount, PVRSRV_KERNEL_SYNC_INFO *apsSrcSync[], IMG_SIZE_T uDataByteSize, PFN_QUEUE_COMMAND_COMPLETE pfnCommandComplete, IMG_HANDLE hCallbackData, IMG_HANDLE *phFence) { PVRSRV_ERROR eError; PVRSRV_COMMAND *psCommand; IMG_SIZE_T uCommandSize; IMG_UINT32 i; SYS_DATA *psSysData; DEVICE_COMMAND_DATA *psDeviceCommandData; #if !(defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE)) PVR_UNREFERENCED_PARAMETER(phFence); #endif /* Check that we've got enough space in our command complete data for this command */ SysAcquireData(&psSysData); psDeviceCommandData = psSysData->apsDeviceCommandData[ui32DevIndex]; if ((psDeviceCommandData[CommandType].ui32MaxDstSyncCount < ui32DstSyncCount) || (psDeviceCommandData[CommandType].ui32MaxSrcSyncCount < ui32SrcSyncCount)) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVInsertCommandKM: Too many syncs")); return PVRSRV_ERROR_INVALID_PARAMS; } /* Round up to nearest 32 bit size so pointer arithmetic works */ uDataByteSize = (uDataByteSize + 3UL) & ~3UL; /* calc. command size */ uCommandSize = sizeof(PVRSRV_COMMAND) + ((ui32DstSyncCount + ui32SrcSyncCount) * sizeof(PVRSRV_SYNC_OBJECT)) + uDataByteSize; /* wait for space in queue */ eError = PVRSRVGetQueueSpaceKM (psQueue, uCommandSize, (IMG_VOID**)&psCommand); if(eError != PVRSRV_OK) { return eError; } #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) if(phFence != IMG_NULL) { struct sync_fence *psRetireFence, *psCleanupFence; /* New command? New timeline target */ psQueue->ui32FenceValue++; psRetireFence = AllocQueueFence(psQueue->pvTimeline, psQueue->ui32FenceValue, "pvr_queue_retire"); if(!psRetireFence) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVInsertCommandKM: sync_fence_create() failed")); psQueue->ui32FenceValue--; return PVRSRV_ERROR_INVALID_PARAMS; } /* This similar to the retire fence, except that it is destroyed * when a display command completes, rather than at the whim of * userspace. It is used to keep the timeline alive. */ psCleanupFence = AllocQueueFence(psQueue->pvTimeline, psQueue->ui32FenceValue, "pvr_queue_cleanup"); if(!psCleanupFence) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVInsertCommandKM: sync_fence_create() #2 failed")); sync_fence_put(psRetireFence); psQueue->ui32FenceValue--; return PVRSRV_ERROR_INVALID_PARAMS; } psCommand->pvCleanupFence = psCleanupFence; psCommand->pvTimeline = psQueue->pvTimeline; *phFence = psRetireFence; } else { psCommand->pvTimeline = IMG_NULL; psCommand->pvCleanupFence = IMG_NULL; } #elif defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE) if(phFence != IMG_NULL) { struct fence *psRetireFence, *psCleanupFence; SyncSWGetTimelineObj(psQueue->i32TimelineFd, &psCommand->pvTimeline); if(psCommand->pvTimeline == IMG_NULL) { PVR_DPF((PVR_DBG_ERROR,"PVRSRVInsertCommandKM: timeline get failed")); return PVRSRV_ERROR_STREAM_ERROR; } /* New command? New timeline target */ psQueue->ui32FenceValue++; psRetireFence = SyncSWTimelineFenceCreateKM(psQueue->i32TimelineFd, psQueue->ui32FenceValue, "pvr_queue_retire"); if(!psRetireFence) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVInsertCommandKM: sync_file_create() failed")); psQueue->ui32FenceValue--; return PVRSRV_ERROR_INVALID_PARAMS; } /* This similar to the retire fence, except that it is destroyed * when a display command completes, rather than at the whim of * userspace. It is used to keep the timeline alive. */ psCleanupFence = SyncSWTimelineFenceCreateKM(psQueue->i32TimelineFd, psQueue->ui32FenceValue, "pvr_queue_cleanup"); if(!psCleanupFence) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVInsertCommandKM: sync_file_create() #2 failed")); SyncSWTimelineFenceReleaseKM(psRetireFence); psQueue->ui32FenceValue--; return PVRSRV_ERROR_INVALID_PARAMS; } psCommand->pvCleanupFence = psCleanupFence; *phFence = psRetireFence; } else { psCommand->pvTimeline = IMG_NULL; psCommand->pvCleanupFence = IMG_NULL; } #endif /* defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) */ psCommand->ui32ProcessID = OSGetCurrentProcessIDKM(); /* setup the command */ psCommand->uCmdSize = uCommandSize; /* this may change if cmd shrinks */ psCommand->ui32DevIndex = ui32DevIndex; psCommand->CommandType = CommandType; psCommand->ui32DstSyncCount = ui32DstSyncCount; psCommand->ui32SrcSyncCount = ui32SrcSyncCount; /* override QAC warning about stricter pointers */ /* PRQA S 3305 END_PTR_ASSIGNMENTS */ psCommand->psDstSync = (PVRSRV_SYNC_OBJECT*)(((IMG_UINTPTR_T)psCommand) + sizeof(PVRSRV_COMMAND)); psCommand->psSrcSync = (PVRSRV_SYNC_OBJECT*)(((IMG_UINTPTR_T)psCommand->psDstSync) + (ui32DstSyncCount * sizeof(PVRSRV_SYNC_OBJECT))); psCommand->pvData = (PVRSRV_SYNC_OBJECT*)(((IMG_UINTPTR_T)psCommand->psSrcSync) + (ui32SrcSyncCount * sizeof(PVRSRV_SYNC_OBJECT))); /* PRQA L:END_PTR_ASSIGNMENTS */ psCommand->uDataSize = uDataByteSize;/* this may change if cmd shrinks */ psCommand->pfnCommandComplete = pfnCommandComplete; psCommand->hCallbackData = hCallbackData; PVR_TTRACE(PVRSRV_TRACE_GROUP_QUEUE, PVRSRV_TRACE_CLASS_CMD_START, QUEUE_TOKEN_INSERTKM); PVR_TTRACE_UI32(PVRSRV_TRACE_GROUP_QUEUE, PVRSRV_TRACE_CLASS_NONE, QUEUE_TOKEN_COMMAND_TYPE, CommandType); /* setup dst sync objects and their sync dependencies */ for (i=0; ipsBase = apsDstSync[i]; spin_lock(&gSyncInfoFreeListLock); list_add_tail(&psQueueSync->sHead, &gSyncInfoFreeList); spin_unlock(&gSyncInfoFreeListLock); #endif PVR_TTRACE_SYNC_OBJECT(PVRSRV_TRACE_GROUP_QUEUE, QUEUE_TOKEN_DST_SYNC, apsDstSync[i], PVRSRV_SYNCOP_SAMPLE); psCommand->psDstSync[i].psKernelSyncInfoKM = apsDstSync[i]; psCommand->psDstSync[i].ui32WriteOpsPending = PVRSRVGetWriteOpsPending(apsDstSync[i], IMG_FALSE); psCommand->psDstSync[i].ui32ReadOps2Pending = PVRSRVGetReadOpsPending(apsDstSync[i], IMG_FALSE); PVRSRVKernelSyncInfoIncRef(apsDstSync[i], IMG_NULL); PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVInsertCommandKM: Dst %u RO-VA:0x%x WO-VA:0x%x ROP:0x%x WOP:0x%x", i, psCommand->psDstSync[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psCommand->psDstSync[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr, psCommand->psDstSync[i].ui32ReadOps2Pending, psCommand->psDstSync[i].ui32WriteOpsPending)); } /* setup src sync objects and their sync dependencies */ for (i=0; ipsBase = apsSrcSync[i]; spin_lock(&gSyncInfoFreeListLock); list_add_tail(&psQueueSync->sHead, &gSyncInfoFreeList); spin_unlock(&gSyncInfoFreeListLock); #endif PVR_TTRACE_SYNC_OBJECT(PVRSRV_TRACE_GROUP_QUEUE, QUEUE_TOKEN_DST_SYNC, apsSrcSync[i], PVRSRV_SYNCOP_SAMPLE); psCommand->psSrcSync[i].psKernelSyncInfoKM = apsSrcSync[i]; psCommand->psSrcSync[i].ui32WriteOpsPending = PVRSRVGetWriteOpsPending(apsSrcSync[i], IMG_TRUE); psCommand->psSrcSync[i].ui32ReadOps2Pending = PVRSRVGetReadOpsPending(apsSrcSync[i], IMG_TRUE); PVRSRVKernelSyncInfoIncRef(apsSrcSync[i], IMG_NULL); PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVInsertCommandKM: Src %u RO-VA:0x%x WO-VA:0x%x ROP:0x%x WOP:0x%x", i, psCommand->psSrcSync[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psCommand->psSrcSync[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr, psCommand->psSrcSync[i].ui32ReadOps2Pending, psCommand->psSrcSync[i].ui32WriteOpsPending)); } PVR_TTRACE(PVRSRV_TRACE_GROUP_QUEUE, PVRSRV_TRACE_CLASS_CMD_END, QUEUE_TOKEN_INSERTKM); /* return pointer to caller to fill out private data */ *ppsCommand = psCommand; return PVRSRV_OK; } /*! ******************************************************************************* @Function : PVRSRVSubmitCommandKM @Description : updates the queue's write offset so the command can be executed. @Input : psQueue - queue command is in @Input : psCommand @Return : PVRSRV_ERROR ******************************************************************************/ IMG_EXPORT PVRSRV_ERROR IMG_CALLCONV PVRSRVSubmitCommandKM(PVRSRV_QUEUE_INFO *psQueue, PVRSRV_COMMAND *psCommand) { /* override QAC warnings about stricter pointers */ /* PRQA S 3305 END_PTR_ASSIGNMENTS2 */ /* patch pointers in the command to be kernel pointers */ if (psCommand->ui32DstSyncCount > 0) { psCommand->psDstSync = (PVRSRV_SYNC_OBJECT*)(((IMG_UINTPTR_T)psQueue->pvLinQueueKM) + psQueue->uWriteOffset + sizeof(PVRSRV_COMMAND)); } if (psCommand->ui32SrcSyncCount > 0) { psCommand->psSrcSync = (PVRSRV_SYNC_OBJECT*)(((IMG_UINTPTR_T)psQueue->pvLinQueueKM) + psQueue->uWriteOffset + sizeof(PVRSRV_COMMAND) + (psCommand->ui32DstSyncCount * sizeof(PVRSRV_SYNC_OBJECT))); } psCommand->pvData = (PVRSRV_SYNC_OBJECT*)(((IMG_UINTPTR_T)psQueue->pvLinQueueKM) + psQueue->uWriteOffset + sizeof(PVRSRV_COMMAND) + (psCommand->ui32DstSyncCount * sizeof(PVRSRV_SYNC_OBJECT)) + (psCommand->ui32SrcSyncCount * sizeof(PVRSRV_SYNC_OBJECT))); /* PRQA L:END_PTR_ASSIGNMENTS2 */ /* update write offset before releasing access lock */ UPDATE_QUEUE_WOFF(psQueue, psCommand->uCmdSize); return PVRSRV_OK; } /*! ****************************************************************************** @Function CheckIfSyncIsQueued @Description Check if the specificed sync object is already queued and can safely be given to the display controller. This check is required as a 3rd party displayclass device can have several flips "in flight" and we need to ensure that we keep their pipeline full and don't deadlock waiting for them to complete an operation on a surface. @Input psSysData : system data @Input psCmdData : COMMAND_COMPLETE_DATA structure @Return PVRSRV_ERROR ******************************************************************************/ static PVRSRV_ERROR CheckIfSyncIsQueued(PVRSRV_SYNC_OBJECT *psSync, COMMAND_COMPLETE_DATA *psCmdData) { IMG_UINT32 k; if (psCmdData->bInUse) { for (k=0;kui32SrcSyncCount;k++) { if (psSync->psKernelSyncInfoKM == psCmdData->psSrcSync[k].psKernelSyncInfoKM) { PVRSRV_SYNC_DATA *psSyncData = psSync->psKernelSyncInfoKM->psSyncData; IMG_UINT32 ui32WriteOpsComplete = psSyncData->ui32WriteOpsComplete; /* We still need to ensure that we don't we don't give a command to the display controller if writes are outstanding on it */ if (ui32WriteOpsComplete == psSync->ui32WriteOpsPending) { return PVRSRV_OK; } else { if (SYNCOPS_STALE(ui32WriteOpsComplete, psSync->ui32WriteOpsPending)) { PVR_DPF((PVR_DBG_WARNING, "CheckIfSyncIsQueued: Stale syncops psSyncData:0x%p ui32WriteOpsComplete:0x%x ui32WriteOpsPending:0x%x", psSyncData, ui32WriteOpsComplete, psSync->ui32WriteOpsPending)); return PVRSRV_OK; } } } } } return PVRSRV_ERROR_FAILED_DEPENDENCIES; } /*! ****************************************************************************** @Function PVRSRVProcessCommand @Description Tries to process a command @Input psSysData : system data @Input psCommand : PVRSRV_COMMAND structure @Input bFlush : Check for stale dependencies (only used for HW recovery) @Return PVRSRV_ERROR ******************************************************************************/ static PVRSRV_ERROR PVRSRVProcessCommand(SYS_DATA *psSysData, PVRSRV_COMMAND *psCommand, IMG_BOOL bFlush) { PVRSRV_SYNC_OBJECT *psWalkerObj; PVRSRV_SYNC_OBJECT *psEndObj; IMG_UINT32 i; COMMAND_COMPLETE_DATA *psCmdCompleteData; PVRSRV_ERROR eError = PVRSRV_OK; IMG_UINT32 ui32WriteOpsComplete; IMG_UINT32 ui32ReadOpsComplete; DEVICE_COMMAND_DATA *psDeviceCommandData; IMG_UINT32 ui32CCBOffset; /* satisfy sync dependencies on the DST(s) */ psWalkerObj = psCommand->psDstSync; psEndObj = psWalkerObj + psCommand->ui32DstSyncCount; while (psWalkerObj < psEndObj) { PVRSRV_SYNC_DATA *psSyncData = psWalkerObj->psKernelSyncInfoKM->psSyncData; ui32WriteOpsComplete = psSyncData->ui32WriteOpsComplete; ui32ReadOpsComplete = psSyncData->ui32ReadOps2Complete; /* fail if reads or writes are not up to date */ if ((ui32WriteOpsComplete != psWalkerObj->ui32WriteOpsPending) || (ui32ReadOpsComplete != psWalkerObj->ui32ReadOps2Pending)) { if (!bFlush || !SYNCOPS_STALE(ui32WriteOpsComplete, psWalkerObj->ui32WriteOpsPending) || !SYNCOPS_STALE(ui32ReadOpsComplete, psWalkerObj->ui32ReadOps2Pending)) { return PVRSRV_ERROR_FAILED_DEPENDENCIES; } } psWalkerObj++; } /* satisfy sync dependencies on the SRC(s) */ psWalkerObj = psCommand->psSrcSync; psEndObj = psWalkerObj + psCommand->ui32SrcSyncCount; while (psWalkerObj < psEndObj) { PVRSRV_SYNC_DATA *psSyncData = psWalkerObj->psKernelSyncInfoKM->psSyncData; ui32ReadOpsComplete = psSyncData->ui32ReadOps2Complete; ui32WriteOpsComplete = psSyncData->ui32WriteOpsComplete; /* fail if writes are not up to date */ if ((ui32WriteOpsComplete != psWalkerObj->ui32WriteOpsPending) || (ui32ReadOpsComplete != psWalkerObj->ui32ReadOps2Pending)) { if (!bFlush && SYNCOPS_STALE(ui32WriteOpsComplete, psWalkerObj->ui32WriteOpsPending) && SYNCOPS_STALE(ui32ReadOpsComplete, psWalkerObj->ui32ReadOps2Pending)) { PVR_DPF((PVR_DBG_WARNING, "PVRSRVProcessCommand: Stale syncops psSyncData:0x%p ui32WriteOpsComplete:0x%x ui32WriteOpsPending:0x%x", psSyncData, ui32WriteOpsComplete, psWalkerObj->ui32WriteOpsPending)); } if (!bFlush || !SYNCOPS_STALE(ui32WriteOpsComplete, psWalkerObj->ui32WriteOpsPending) || !SYNCOPS_STALE(ui32ReadOpsComplete, psWalkerObj->ui32ReadOps2Pending)) { IMG_UINT32 j; PVRSRV_ERROR eError; IMG_BOOL bFound = IMG_FALSE; psDeviceCommandData = psSysData->apsDeviceCommandData[psCommand->ui32DevIndex]; for (j=0;jCommandType].apsCmdCompleteData[j]); if (eError == PVRSRV_OK) { bFound = IMG_TRUE; } } if (!bFound) return PVRSRV_ERROR_FAILED_DEPENDENCIES; } } psWalkerObj++; } /* validate device type */ if (psCommand->ui32DevIndex >= SYS_DEVICE_COUNT) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVProcessCommand: invalid DeviceType 0x%x", psCommand->ui32DevIndex)); return PVRSRV_ERROR_INVALID_PARAMS; } /* fish out the appropriate storage structure for the duration of the command */ psDeviceCommandData = psSysData->apsDeviceCommandData[psCommand->ui32DevIndex]; ui32CCBOffset = psDeviceCommandData[psCommand->CommandType].ui32CCBOffset; psCmdCompleteData = psDeviceCommandData[psCommand->CommandType].apsCmdCompleteData[ui32CCBOffset]; if (psCmdCompleteData->bInUse) { /* can use this to protect against concurrent execution of same command */ return PVRSRV_ERROR_FAILED_DEPENDENCIES; } /* mark the structure as in use */ psCmdCompleteData->bInUse = IMG_TRUE; /* copy src updates over */ psCmdCompleteData->ui32DstSyncCount = psCommand->ui32DstSyncCount; for (i=0; iui32DstSyncCount; i++) { psCmdCompleteData->psDstSync[i] = psCommand->psDstSync[i]; PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVProcessCommand: Dst %u RO-VA:0x%x WO-VA:0x%x ROP:0x%x WOP:0x%x (CCB:%u)", i, psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr, psCmdCompleteData->psDstSync[i].ui32ReadOps2Pending, psCmdCompleteData->psDstSync[i].ui32WriteOpsPending, ui32CCBOffset)); } psCmdCompleteData->pfnCommandComplete = psCommand->pfnCommandComplete; psCmdCompleteData->hCallbackData = psCommand->hCallbackData; #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE) psCmdCompleteData->pvCleanupFence = psCommand->pvCleanupFence; psCmdCompleteData->pvTimeline = psCommand->pvTimeline; #endif /* copy dst updates over */ psCmdCompleteData->ui32SrcSyncCount = psCommand->ui32SrcSyncCount; for (i=0; iui32SrcSyncCount; i++) { psCmdCompleteData->psSrcSync[i] = psCommand->psSrcSync[i]; PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVProcessCommand: Src %u RO-VA:0x%x WO-VA:0x%x ROP:0x%x WOP:0x%x (CCB:%u)", i, psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr, psCmdCompleteData->psSrcSync[i].ui32ReadOps2Pending, psCmdCompleteData->psSrcSync[i].ui32WriteOpsPending, ui32CCBOffset)); } /* call the cmd specific handler: it should: - check the cmd specific dependencies - setup private cmd complete structure - execute cmd on HW - store psCmdCompleteData `cookie' and later pass as argument to Generic Command Complete Callback n.b. ui32DataSize (packet size) is useful for packet validation */ if (psDeviceCommandData[psCommand->CommandType].pfnCmdProc((IMG_HANDLE)psCmdCompleteData, (IMG_UINT32)psCommand->uDataSize, psCommand->pvData) == IMG_FALSE) { /* clean-up: free cmd complete structure */ psCmdCompleteData->bInUse = IMG_FALSE; eError = PVRSRV_ERROR_CMD_NOT_PROCESSED; PVR_LOG(("Failed to submit command from queue processor, this could cause sync wedge!")); } else { /* Increment the CCB offset */ psDeviceCommandData[psCommand->CommandType].ui32CCBOffset = (ui32CCBOffset + 1) % DC_NUM_COMMANDS_PER_TYPE; } return eError; } static IMG_VOID PVRSRVProcessQueues_ForEachCb(PVRSRV_DEVICE_NODE *psDeviceNode) { if (psDeviceNode->bReProcessDeviceCommandComplete && psDeviceNode->pfnDeviceCommandComplete != IMG_NULL) { (*psDeviceNode->pfnDeviceCommandComplete)(psDeviceNode); } } /*! ****************************************************************************** @Function PVRSRVProcessQueues @Description Tries to process a command from each Q @input ui32CallerID - used to distinguish between async ISR/DPC type calls the synchronous services driver @input bFlush - flush commands with stale dependencies (only used for HW recovery) @Return PVRSRV_ERROR ******************************************************************************/ IMG_EXPORT PVRSRV_ERROR PVRSRVProcessQueues(IMG_BOOL bFlush) { PVRSRV_QUEUE_INFO *psQueue; SYS_DATA *psSysData; PVRSRV_COMMAND *psCommand; /* PVRSRV_DEVICE_NODE *psDeviceNode;*/ SysAcquireData(&psSysData); /* Ensure we don't corrupt queue list, by blocking access. This is required for OSs where multiple ISR threads may exist simultaneously (eg WinXP DPC routines) */ if (psSysData->psQueueList == IMG_NULL) { PVR_DPF((PVR_DBG_MESSAGE,"No Queues installed - cannot process commands")); return PVRSRV_OK; } #if !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) while (OSLockResourceAndBlockMISR(&psSysData->sQProcessResource, ISR_ID) != PVRSRV_OK) #else /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ while (OSLockResource(&psSysData->sQProcessResource, ISR_ID) != PVRSRV_OK) #endif /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ { OSWaitus(1); }; psQueue = psSysData->psQueueList; if(!psQueue) { PVR_DPF((PVR_DBG_MESSAGE,"No Queues installed - cannot process commands")); } if (bFlush) { PVRSRVSetDCState(DC_STATE_FLUSH_COMMANDS); } while (psQueue) { while (psQueue->uReadOffset != psQueue->uWriteOffset) { psCommand = (PVRSRV_COMMAND*)((IMG_UINTPTR_T)psQueue->pvLinQueueKM + psQueue->uReadOffset); if (PVRSRVProcessCommand(psSysData, psCommand, bFlush) == PVRSRV_OK) { /* processed cmd so update queue */ UPDATE_QUEUE_ROFF(psQueue, psCommand->uCmdSize) continue; } break; } psQueue = psQueue->psNextKM; } if (bFlush) { PVRSRVSetDCState(DC_STATE_NO_FLUSH_COMMANDS); } /* Re-process command complete handlers if necessary. */ List_PVRSRV_DEVICE_NODE_ForEach(psSysData->psDeviceNodeList, &PVRSRVProcessQueues_ForEachCb); #if !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) OSUnlockResourceAndUnblockMISR(&psSysData->sQProcessResource, ISR_ID); #else /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ OSUnlockResource(&psSysData->sQProcessResource, ISR_ID); #endif /* !defined(PVR_LINUX_USING_WORKQUEUES) && defined(__linux__) */ return PVRSRV_OK; } #if defined(SYS_OMAP_HAS_DVFS_FRAMEWORK) extern void sgxfreq_notif_sgx_frame_done(void); #endif /* (SYS_OMAP4_HAS_DVFS_FRAMEWORK) */ /*! ****************************************************************************** @Function PVRSRVCommandCompleteKM @Description Updates non-private command complete sync objects @Input hCmdCookie : command cookie @Input bScheduleMISR : boolean to schedule MISR @Return PVRSRV_ERROR ******************************************************************************/ IMG_EXPORT IMG_VOID PVRSRVCommandCompleteKM(IMG_HANDLE hCmdCookie, IMG_BOOL bScheduleMISR) { IMG_UINT32 i; COMMAND_COMPLETE_DATA *psCmdCompleteData = (COMMAND_COMPLETE_DATA *)hCmdCookie; SYS_DATA *psSysData; #if defined(SYS_OMAP_HAS_DVFS_FRAMEWORK) sgxfreq_notif_sgx_frame_done(); #endif /* (SYS_OMAP_HAS_DVFS_FRAMEWORK) */ SysAcquireData(&psSysData); PVR_TTRACE(PVRSRV_TRACE_GROUP_QUEUE, PVRSRV_TRACE_CLASS_CMD_COMP_START, QUEUE_TOKEN_COMMAND_COMPLETE); /* update DST(s) syncs */ for (i=0; iui32DstSyncCount; i++) { psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM->psSyncData->ui32WriteOpsComplete++; #if !((defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE)) && defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)) PVRSRVKernelSyncInfoDecRef(psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM, IMG_NULL); #endif PVR_TTRACE_SYNC_OBJECT(PVRSRV_TRACE_GROUP_QUEUE, QUEUE_TOKEN_UPDATE_DST, psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM, PVRSRV_SYNCOP_COMPLETE); PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVCommandCompleteKM: Dst %u RO-VA:0x%x WO-VA:0x%x ROP:0x%x WOP:0x%x", i, psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr, psCmdCompleteData->psDstSync[i].ui32ReadOps2Pending, psCmdCompleteData->psDstSync[i].ui32WriteOpsPending)); } /* update SRC(s) syncs */ for (i=0; iui32SrcSyncCount; i++) { psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM->psSyncData->ui32ReadOps2Complete++; #if !((defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE)) && defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)) PVRSRVKernelSyncInfoDecRef(psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM, IMG_NULL); #endif PVR_TTRACE_SYNC_OBJECT(PVRSRV_TRACE_GROUP_QUEUE, QUEUE_TOKEN_UPDATE_SRC, psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM, PVRSRV_SYNCOP_COMPLETE); PVR_DPF((PVR_DBG_MESSAGE, "PVRSRVCommandCompleteKM: Src %u RO-VA:0x%x WO-VA:0x%x ROP:0x%x WOP:0x%x", i, psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM->sReadOps2CompleteDevVAddr.uiAddr, psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM->sWriteOpsCompleteDevVAddr.uiAddr, psCmdCompleteData->psSrcSync[i].ui32ReadOps2Pending, psCmdCompleteData->psSrcSync[i].ui32WriteOpsPending)); } #if (defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) || defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE)) && defined(DEBUG_LINUX_MEMORY_ALLOCATIONS) if(psCmdCompleteData->ui32DstSyncCount || psCmdCompleteData->ui32SrcSyncCount) { /* Add work to worker thread for checking and freeing of kernel sync */ queue_work(gpsWorkQueue, &gsWork); } #endif PVR_TTRACE(PVRSRV_TRACE_GROUP_QUEUE, PVRSRV_TRACE_CLASS_CMD_COMP_END, QUEUE_TOKEN_COMMAND_COMPLETE); if (psCmdCompleteData->pfnCommandComplete) { psCmdCompleteData->pfnCommandComplete(psCmdCompleteData->hCallbackData); } #if defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) if(psCmdCompleteData->pvTimeline) { sw_sync_timeline_inc(psCmdCompleteData->pvTimeline, 1); sync_fence_put(psCmdCompleteData->pvCleanupFence); } #elif defined(PVR_ANDROID_NATIVE_WINDOW_HAS_FENCE) if(psCmdCompleteData->pvTimeline) { SyncSWTimelineAdvanceKM(psCmdCompleteData->pvTimeline); SyncSWTimelineReleaseKM(psCmdCompleteData->pvTimeline); psCmdCompleteData->pvTimeline = IMG_NULL; psCmdCompleteData->pvCleanupFence = IMG_NULL; } #endif /* defined(PVR_ANDROID_NATIVE_WINDOW_HAS_SYNC) */ /* free command complete storage */ psCmdCompleteData->bInUse = IMG_FALSE; /* FIXME: This may cause unrelated devices to be woken up. */ PVRSRVScheduleDeviceCallbacks(); if(bScheduleMISR) { OSScheduleMISR(psSysData); } } /*! ****************************************************************************** @Function PVRSRVRegisterCmdProcListKM @Description registers a list of private command processing functions with the Command Queue Manager @Input ui32DevIndex : device index @Input ppfnCmdProcList : function ptr table of private command processors @Input ui32MaxSyncsPerCmd : max number of syncobjects used by command @Input ui32CmdCount : number of entries in function ptr table @Return PVRSRV_ERROR ******************************************************************************/ IMG_EXPORT PVRSRV_ERROR PVRSRVRegisterCmdProcListKM(IMG_UINT32 ui32DevIndex, PFN_CMD_PROC *ppfnCmdProcList, IMG_UINT32 ui32MaxSyncsPerCmd[][2], IMG_UINT32 ui32CmdCount) { SYS_DATA *psSysData; PVRSRV_ERROR eError; IMG_UINT32 ui32CmdCounter, ui32CmdTypeCounter; IMG_SIZE_T ui32AllocSize; DEVICE_COMMAND_DATA *psDeviceCommandData; COMMAND_COMPLETE_DATA *psCmdCompleteData; /* validate device type */ if(ui32DevIndex >= SYS_DEVICE_COUNT) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVRegisterCmdProcListKM: invalid DeviceType 0x%x", ui32DevIndex)); return PVRSRV_ERROR_INVALID_PARAMS; } /* acquire system data structure */ SysAcquireData(&psSysData); /* array of pointers for each command store */ ui32AllocSize = ui32CmdCount * sizeof(*psDeviceCommandData); eError = OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP, ui32AllocSize, (IMG_VOID **)&psDeviceCommandData, IMG_NULL, "Array of Pointers for Command Store"); if (eError != PVRSRV_OK) { PVR_DPF((PVR_DBG_ERROR,"PVRSRVRegisterCmdProcListKM: Failed to alloc CC data")); goto ErrorExit; } psSysData->apsDeviceCommandData[ui32DevIndex] = psDeviceCommandData; for (ui32CmdTypeCounter = 0; ui32CmdTypeCounter < ui32CmdCount; ui32CmdTypeCounter++) { psDeviceCommandData[ui32CmdTypeCounter].pfnCmdProc = ppfnCmdProcList[ui32CmdTypeCounter]; psDeviceCommandData[ui32CmdTypeCounter].ui32CCBOffset = 0; psDeviceCommandData[ui32CmdTypeCounter].ui32MaxDstSyncCount = ui32MaxSyncsPerCmd[ui32CmdTypeCounter][0]; psDeviceCommandData[ui32CmdTypeCounter].ui32MaxSrcSyncCount = ui32MaxSyncsPerCmd[ui32CmdTypeCounter][1]; for (ui32CmdCounter = 0; ui32CmdCounter < DC_NUM_COMMANDS_PER_TYPE; ui32CmdCounter++) { /* allocate storage for the sync update on command complete */ ui32AllocSize = sizeof(COMMAND_COMPLETE_DATA) /* space for one GENERIC_CMD_COMPLETE */ + ((ui32MaxSyncsPerCmd[ui32CmdTypeCounter][0] + ui32MaxSyncsPerCmd[ui32CmdTypeCounter][1]) * sizeof(PVRSRV_SYNC_OBJECT)); /* space for max sync objects */ eError = OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP, ui32AllocSize, (IMG_VOID **)&psCmdCompleteData, IMG_NULL, "Command Complete Data"); if (eError != PVRSRV_OK) { PVR_DPF((PVR_DBG_ERROR,"PVRSRVRegisterCmdProcListKM: Failed to alloc cmd %d", ui32CmdTypeCounter)); goto ErrorExit; } psDeviceCommandData[ui32CmdTypeCounter].apsCmdCompleteData[ui32CmdCounter] = psCmdCompleteData; /* clear memory */ OSMemSet(psCmdCompleteData, 0x00, ui32AllocSize); /* setup sync pointers */ psCmdCompleteData->psDstSync = (PVRSRV_SYNC_OBJECT*) (((IMG_UINTPTR_T)psCmdCompleteData) + sizeof(COMMAND_COMPLETE_DATA)); psCmdCompleteData->psSrcSync = (PVRSRV_SYNC_OBJECT*) (((IMG_UINTPTR_T)psCmdCompleteData->psDstSync) + (sizeof(PVRSRV_SYNC_OBJECT) * ui32MaxSyncsPerCmd[ui32CmdTypeCounter][0])); psCmdCompleteData->ui32AllocSize = (IMG_UINT32)ui32AllocSize; } } return PVRSRV_OK; ErrorExit: /* clean-up if things went wrong */ if (PVRSRVRemoveCmdProcListKM(ui32DevIndex, ui32CmdCount) != PVRSRV_OK) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVRegisterCmdProcListKM: Failed to clean up after error, device 0x%x", ui32DevIndex)); } return eError; } /*! ****************************************************************************** @Function PVRSRVRemoveCmdProcListKM @Description removes a list of private command processing functions and data from the Queue Manager @Input ui32DevIndex : device index @Input ui32CmdCount : number of entries in function ptr table @Return PVRSRV_ERROR ******************************************************************************/ IMG_EXPORT PVRSRV_ERROR PVRSRVRemoveCmdProcListKM(IMG_UINT32 ui32DevIndex, IMG_UINT32 ui32CmdCount) { SYS_DATA *psSysData; IMG_UINT32 ui32CmdTypeCounter, ui32CmdCounter; DEVICE_COMMAND_DATA *psDeviceCommandData; COMMAND_COMPLETE_DATA *psCmdCompleteData; IMG_SIZE_T ui32AllocSize; /* validate device type */ if(ui32DevIndex >= SYS_DEVICE_COUNT) { PVR_DPF((PVR_DBG_ERROR, "PVRSRVRemoveCmdProcListKM: invalid DeviceType 0x%x", ui32DevIndex)); return PVRSRV_ERROR_INVALID_PARAMS; } /* acquire system data structure */ SysAcquireData(&psSysData); psDeviceCommandData = psSysData->apsDeviceCommandData[ui32DevIndex]; if(psDeviceCommandData != IMG_NULL) { for (ui32CmdTypeCounter = 0; ui32CmdTypeCounter < ui32CmdCount; ui32CmdTypeCounter++) { for (ui32CmdCounter = 0; ui32CmdCounter < DC_NUM_COMMANDS_PER_TYPE; ui32CmdCounter++) { psCmdCompleteData = psDeviceCommandData[ui32CmdTypeCounter].apsCmdCompleteData[ui32CmdCounter]; /* free the cmd complete structure array entries */ if (psCmdCompleteData != IMG_NULL) { PVR_ASSERT(psCmdCompleteData->bInUse == IMG_FALSE); OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, psCmdCompleteData->ui32AllocSize, psCmdCompleteData, IMG_NULL); psDeviceCommandData[ui32CmdTypeCounter].apsCmdCompleteData[ui32CmdCounter] = IMG_NULL; } } } /* free the cmd complete structure array for the device */ ui32AllocSize = ui32CmdCount * sizeof(*psDeviceCommandData); OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, ui32AllocSize, psDeviceCommandData, IMG_NULL); psSysData->apsDeviceCommandData[ui32DevIndex] = IMG_NULL; } return PVRSRV_OK; } /****************************************************************************** End of file (queue.c) ******************************************************************************/