Waiting thread and event of fan eventhandler in weblogic server

Hello Mwadhwa,

As per the thread report that you attached, I found only 2 threads related to FAN Event Handler:oracle.simplefan.FanSubscription:kasprod. It contains 52 threads are in waiting state and 125 are runnable state.

[ACTIVE] ExecuteThread: '77' for queue: 'weblogic.kernel.Default (self-tuning)' this thread is throwing an error. Here is the stack trace for it:

stackTrace:
java.lang.Thread.State: RUNNABLE
at java.lang.Throwable.fillInStackTrace(Native Method)
at java.lang.Throwable.fillInStackTrace(Throwable.java:784)
- locked <0x00007faa4462eb10> (a java.lang.Throwable)
at java.lang.Throwable.<init>(Throwable.java:251)
at java.util.logging.LogRecord.inferCaller(LogRecord.java:545)
at java.util.logging.LogRecord.getSourceClassName(LogRecord.java:308)
at java.util.logging.SimpleFormatter.format(SimpleFormatter.java:143)
- locked <0x00007f9dacfa6d98> (a java.util.logging.SimpleFormatter)
at java.util.logging.StreamHandler.publish(StreamHandler.java:211)
- eliminated <0x00007f9dacfa6d00> (a java.util.logging.FileHandler)
at java.util.logging.FileHandler.publish(FileHandler.java:701)
- locked <0x00007f9dacfa6d00> (a java.util.logging.FileHandler)
at org.eclipse.birt.report.engine.api.impl.EngineLogger$AdapterHandler.publish(EngineLogger.java:400)
at java.util.logging.Logger.log(Logger.java:738)
at java.util.logging.Logger.doLog(Logger.java:765)
at java.util.logging.Logger.log(Logger.java:788)
at org.eclipse.birt.data.engine.executor.DataSourceQuery.execute(DataSourceQuery.java:974)
at org.eclipse.birt.data.engine.impl.PreparedOdaDSQuery$OdaDSQueryExecutor.executeOdiQuery(PreparedOdaDSQuery.java:503)
at org.eclipse.birt.data.engine.impl.QueryExecutor.execute(QueryExecutor.java:1208)
at org.eclipse.birt.data.engine.impl.ServiceForQueryResults.executeQuery(ServiceForQueryResults.java:233)
at org.eclipse.birt.data.engine.impl.QueryResults.getResultIterator(QueryResults.java:178)
at org.eclipse.birt.report.engine.data.dte.QueryResultSet.<init>(QueryResultSet.java:98)
at org.eclipse.birt.report.engine.data.dte.DteDataEngine.doExecuteQuery(DteDataEngine.java:168)
at org.eclipse.birt.report.engine.data.dte.DataGenerationEngine.doExecuteQuery(DataGenerationEngine.java:83)
at org.eclipse.birt.report.engine.data.dte.AbstractDataEngine.execute(AbstractDataEngine.java:275)
at org.eclipse.birt.report.engine.executor.ExecutionContext.executeQuery(ExecutionContext.java:1947)
at org.eclipse.birt.report.engine.executor.QueryItemExecutor.executeQuery(QueryItemExecutor.java:80)
at org.eclipse.birt.report.engine.executor.TextItemExecutor.executeHtmlText(TextItemExecutor.java:97)
at org.eclipse.birt.report.engine.executor.TextItemExecutor.execute(TextItemExecutor.java:70)
at org.eclipse.birt.report.engine.internal.executor.wrap.WrappedReportItemExecutor.execute(WrappedReportItemExecutor.java:46)
at org.eclipse.birt.report.engine.internal.executor.emitter.ReportItemEmitterExecutor.execute(ReportItemEmitterExecutor.java:46)
at org.eclipse.birt.report.engine.internal.executor.dup.SuppressDuplicateItemExecutor.execute(SuppressDuplicateItemExecutor.java:43)
at org.eclipse.birt.report.engine.internal.executor.wrap.WrappedReportItemExecutor.execute(WrappedReportItemExecutor.java:46)
at org.eclipse.birt.report.engine.internal.executor.l18n.LocalizedReportItemExecutor.execute(LocalizedReportItemExecutor.java:34)
at org.eclipse.birt.report.engine.layout.html.HTMLBlockStackingLM.layoutNodes(HTMLBlockStackingLM.java:65)
at org.eclipse.birt.report.engine.layout.html.HTMLStackingLM.layoutChildren(HTMLStackingLM.java:26)
at org.eclipse.birt.report.engine.layout.html.HTMLAbstractLM.layout(HTMLAbstractLM.java:140)
at org.eclipse.birt.report.engine.layout.html.HTMLInlineStackingLM.resumeLayout(HTMLInlineStackingLM.java:111)
at org.eclipse.birt.report.engine.layout.html.HTMLInlineStackingLM.layoutNodes(HTMLInlineStackingLM.java:160)
at org.eclipse.birt.report.engine.layout.html.HTMLStackingLM.layoutChildren(HTMLStackingLM.java:26)
at org.eclipse.birt.report.engine.layout.html.HTMLAbstractLM.layout(HTMLAbstractLM.java:140)
at org.eclipse.birt.report.engine.layout.html.HTMLBlockStackingLM.layoutNodes(HTMLBlockStackingLM.java:70)
at org.eclipse.birt.report.engine.layout.html.HTMLStackingLM.layoutChildren(HTMLStackingLM.java:26)
at org.eclipse.birt.report.engine.layout.html.HTMLRepeatHeaderLM.layoutChildren(HTMLRepeatHeaderLM.java:46)
at org.eclipse.birt.report.engine.layout.html.HTMLAbstractLM.layout(HTMLAbstractLM.java:140)
at org.eclipse.birt.report.engine.layout.html.HTMLBlockStackingLM.layoutNodes(HTMLBlockStackingLM.java:70)
at org.eclipse.birt.report.engine.layout.html.HTMLPageLM.layout(HTMLPageLM.java:92)
at org.eclipse.birt.report.engine.layout.html.HTMLReportLayoutEngine.layout(HTMLReportLayoutEngine.java:100)
at org.eclipse.birt.report.engine.presentation.ReportDocumentBuilder.build(ReportDocumentBuilder.java:249)
at org.eclipse.birt.report.engine.api.impl.RunTask.doRun(RunTask.java:269)
at org.eclipse.birt.report.engine.api.impl.RunTask.run(RunTask.java:86)
at org.eclipse.birt.report.service.ReportEngineService.runReport(ReportEngineService.java:1317)
at org.eclipse.birt.report.service.BirtViewerReportService.runReport(BirtViewerReportService.java:158)
at org.eclipse.birt.report.service.actionhandler.BirtRunReportActionHandler.__execute(BirtRunReportActionHandler.java:81)
at org.eclipse.birt.report.service.actionhandler.AbstractBaseActionHandler.execute(AbstractBaseActionHandler.java:90)
at org.eclipse.birt.report.presentation.aggregation.layout.FramesetFragment.doService(FramesetFragment.java:189)
at org.eclipse.birt.report.presentation.aggregation.layout.FramesetFragment.service(FramesetFragment.java:86)
at org.eclipse.birt.report.servlet.ViewerServlet.__doGet(ViewerServlet.java:87)
at org.eclipse.birt.report.servlet.BirtSoapMessageDispatcherServlet.doGet(BirtSoapMessageDispatcherServlet.java:160)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at org.apache.axis.transport.http.AxisServletBase.service(AxisServletBase.java:327)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.birt.report.servlet.BirtSoapMessageDispatcherServlet.service(BirtSoapMessageDispatcherServlet.java:122)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:295)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:260)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:137)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:353)
at weblogic.servlet.internal.TailFilter.doFilter(TailFilter.java:25)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at org.eclipse.birt.report.filter.ViewerFilter.doFilter(ViewerFilter.java:68)
at weblogic.servlet.internal.FilterChainImpl.doFilter(FilterChainImpl.java:78)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3797)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3763)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:344)
at weblogic.security.service.SecurityManager.runAsForUserCode(SecurityManager.java:197)
at weblogic.servlet.provider.WlsSecurityProvider.runAsForUserCode(WlsSecurityProvider.java:203)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:71)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2451)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2299)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2277)
at weblogic.servlet.internal.ServletRequestImpl.runInternal(ServletRequestImpl.java:1720)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1680)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:272)
at weblogic.invocation.ComponentInvocationContextManager._runAs(ComponentInvocationContextManager.java:352)
at weblogic.invocation.ComponentInvocationContextManager.runAs(ComponentInvocationContextManager.java:337)
at weblogic.work.LivePartitionUtility.doRunWorkUnderContext(LivePartitionUtility.java:57)
at weblogic.work.PartitionUtility.runWorkUnderContext(PartitionUtility.java:41)
at weblogic.work.SelfTuningWorkManagerImpl.runWorkUnderContext(SelfTuningWorkManagerImpl.java:655)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:420)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:360)
Locked ownable synchronizers:
- None

There could be several reasons why application is not responding. Some of the reasons are:

• Garbage collection pauses
• Threads getting BLOCKED
• Network connectivity
• Load balancer routing issue
• Heavy CPU consumption of threads
• Operating System running with old patches
• Memory Leak
• DB not responding properly

:

:

 So just thread dump is not enough to diagnose the problem. You have captured only thread dump, that too one snapshot of it. It's always a good practice to capture 3 thread dumps in a gap of 10 seconds between each one. Besides thread dumps you might have to capture other logs/artifacts to do thorough analysis.

 You can use the open source yc data script which will capture 360-degree application level artifacts (like GC logs, 3 snapshots of thread dumps, heap dumps) and system level artifacts (like top, top -H, netstat, vmstat, iostat, dmesg, diskusage, kernel parameters...). Once you have these data, either you can manually analyze them or upload it to yCrash tool, which will analyze all these artifacts and generate one unified root cause analysis marrying all these artifacts. It can indicate the root cause of the problem.

Today I had a close look at your thread dump. I suspect there is some slow down happening at your Oracle Database. Because most of the threads which are making Oracle DB call aren't getting any response. Here with I am sharing stacktrace of the few threads which aren't getting any response from Oracle DB. You might want to see investigate on the database side for the following:

a. Any CPU, Memory bottleneck on the Oracle DB

b. Your application is making DB call from this line of code 'com.infotech.kastle.lending.dao.workflow.Workflow.UpdateWorkflow(Workflow.java:135)' - you might want to investigate whether any SQL change has been introduced recently which is causing this SQL query long time to complete. 

c. OR it could also be a network connectivity issue

[ACTIVE] ExecuteThread: '125' for queue: 'weblogic.kernel.Default (self-tuning)'
PRIORITY : 5
THREAD ID : 0X00007F9C6B262800
NATIVE ID : 0X6F44
NATIVE ID (DECIMAL) : 28484
STATE : RUNNABLE

stackTrace:
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:197)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:378)
- locked <0x00007f9e8bb70970> (a java.lang.Object)
at oracle.net.nt.TimeoutSocketChannel.read(TimeoutSocketChannel.java:174)
at oracle.net.ns.NSProtocolNIO.doSocketRead(NSProtocolNIO.java:555)
at oracle.net.ns.NIOPacket.readHeader(NIOPacket.java:258)
at oracle.net.ns.NIOPacket.readPacketFromSocketChannel(NIOPacket.java:190)
at oracle.net.ns.NIOPacket.readFromSocketChannel(NIOPacket.java:132)
at oracle.net.ns.NIOPacket.readFromSocketChannel(NIOPacket.java:105)
at oracle.net.ns.NIONSDataChannel.readDataFromSocketChannel(NIONSDataChannel.java:91)
at oracle.jdbc.driver.T4CMAREngineNIO.prepareForUnmarshall(T4CMAREngineNIO.java:764)
at oracle.jdbc.driver.T4CMAREngineNIO.unmarshalUB1(T4CMAREngineNIO.java:429)
at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:407)
at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:268)
at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:655)
at oracle.jdbc.driver.T4CCallableStatement.doOall8(T4CCallableStatement.java:265)
at oracle.jdbc.driver.T4CCallableStatement.doOall8(T4CCallableStatement.java:86)
at oracle.jdbc.driver.T4CCallableStatement.executeForRows(T4CCallableStatement.java:965)
at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1205)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3666)
at oracle.jdbc.driver.T4CCallableStatement.executeInternal(T4CCallableStatement.java:1358)
at oracle.jdbc.driver.OraclePreparedStatement.execute(OraclePreparedStatement.java:3778)
- locked <0x00007f9e8ac4d880> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.OracleCallableStatement.execute(OracleCallableStatement.java:4251)
- locked <0x00007f9e8ac4d880> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.OraclePreparedStatementWrapper.execute(OraclePreparedStatementWrapper.java:1081)
at weblogic.jdbc.wrapper.PreparedStatement.execute(PreparedStatement.java:119)
at com.infotech.kastle.lending.dao.workflow.Workflow.UpdateWorkflow(Workflow.java:135)
at com.infotech.kastle.lending.servlet.CommonValidationServlet.onDataEntryComplete(CommonValidationServlet.java:6243)
at com.infotech.kastle.lending.servlet.CommonValidationServlet.doGet(CommonValidationServlet.java:927)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:295)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:260)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:137)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:353)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:250)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3796)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3763)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:344)
at weblogic.security.service.SecurityManager.runAsForUserCode(SecurityManager.java:197)
at weblogic.servlet.provider.WlsSecurityProvider.runAsForUserCode(WlsSecurityProvider.java:203)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:71)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2451)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2299)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2277)
at weblogic.servlet.internal.ServletRequestImpl.runInternal(ServletRequestImpl.java:1720)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1680)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:272)
at weblogic.invocation.ComponentInvocationContextManager._runAs(ComponentInvocationContextManager.java:352)
at weblogic.invocation.ComponentInvocationContextManager.runAs(ComponentInvocationContextManager.java:337)
at weblogic.work.LivePartitionUtility.doRunWorkUnderContext(LivePartitionUtility.java:57)
at weblogic.work.PartitionUtility.runWorkUnderContext(PartitionUtility.java:41)
at weblogic.work.SelfTuningWorkManagerImpl.runWorkUnderContext(SelfTuningWorkManagerImpl.java:655)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:420)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:360)
Locked ownable synchronizers:
- None 

[ACTIVE] ExecuteThread: '132' for queue: 'weblogic.kernel.Default (self-tuning)'
PRIORITY : 5
THREAD ID : 0X00007F9C6B269800
NATIVE ID : 0X7CBB
NATIVE ID (DECIMAL) : 31931
STATE : RUNNABLE

stackTrace:
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
at sun.nio.ch.IOUtil.read(IOUtil.java:197)
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:378)
- locked <0x00007f9f470e1e78> (a java.lang.Object)
at oracle.net.nt.TimeoutSocketChannel.read(TimeoutSocketChannel.java:174)
at oracle.net.ns.NSProtocolNIO.doSocketRead(NSProtocolNIO.java:555)
at oracle.net.ns.NIOPacket.readHeader(NIOPacket.java:258)
at oracle.net.ns.NIOPacket.readPacketFromSocketChannel(NIOPacket.java:190)
at oracle.net.ns.NIOPacket.readFromSocketChannel(NIOPacket.java:132)
at oracle.net.ns.NIOPacket.readFromSocketChannel(NIOPacket.java:105)
at oracle.net.ns.NIONSDataChannel.readDataFromSocketChannel(NIONSDataChannel.java:91)
at oracle.jdbc.driver.T4CMAREngineNIO.prepareForUnmarshall(T4CMAREngineNIO.java:764)
at oracle.jdbc.driver.T4CMAREngineNIO.unmarshalUB1(T4CMAREngineNIO.java:429)
at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:407)
at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:268)
at oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:655)
at oracle.jdbc.driver.T4CCallableStatement.doOall8(T4CCallableStatement.java:265)
at oracle.jdbc.driver.T4CCallableStatement.doOall8(T4CCallableStatement.java:86)
at oracle.jdbc.driver.T4CCallableStatement.executeForRows(T4CCallableStatement.java:965)
at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1205)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3666)
at oracle.jdbc.driver.T4CCallableStatement.executeInternal(T4CCallableStatement.java:1358)
at oracle.jdbc.driver.OraclePreparedStatement.execute(OraclePreparedStatement.java:3778)
- locked <0x00007f9f4700dfc0> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.OracleCallableStatement.execute(OracleCallableStatement.java:4251)
- locked <0x00007f9f4700dfc0> (a oracle.jdbc.driver.T4CConnection)
at oracle.jdbc.driver.OraclePreparedStatementWrapper.execute(OraclePreparedStatementWrapper.java:1081)
at weblogic.jdbc.wrapper.PreparedStatement.execute(PreparedStatement.java:119)
at com.infotech.kastle.lending.dao.workflow.Workflow.UpdateWorkflow(Workflow.java:135)
at com.infotech.kastle.lending.servlet.CommonValidationServlet.onWorkFlowUpdate(CommonValidationServlet.java:6629)
at com.infotech.kastle.lending.servlet.CommonValidationServlet.doGet(CommonValidationServlet.java:1052)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:295)
at weblogic.servlet.internal.StubSecurityHelper$ServletServiceAction.run(StubSecurityHelper.java:260)
at weblogic.servlet.internal.StubSecurityHelper.invokeServlet(StubSecurityHelper.java:137)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:353)
at weblogic.servlet.internal.ServletStubImpl.execute(ServletStubImpl.java:250)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.wrapRun(WebAppServletContext.java:3796)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:3763)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:344)
at weblogic.security.service.SecurityManager.runAsForUserCode(SecurityManager.java:197)
at weblogic.servlet.provider.WlsSecurityProvider.runAsForUserCode(WlsSecurityProvider.java:203)
at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:71)
at weblogic.servlet.internal.WebAppServletContext.doSecuredExecute(WebAppServletContext.java:2451)
at weblogic.servlet.internal.WebAppServletContext.securedExecute(WebAppServletContext.java:2299)
at weblogic.servlet.internal.WebAppServletContext.execute(WebAppServletContext.java:2277)
at weblogic.servlet.internal.ServletRequestImpl.runInternal(ServletRequestImpl.java:1720)
at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1680)
at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:272)
at weblogic.invocation.ComponentInvocationContextManager._runAs(ComponentInvocationContextManager.java:352)
at weblogic.invocation.ComponentInvocationContextManager.runAs(ComponentInvocationContextManager.java:337)
at weblogic.work.LivePartitionUtility.doRunWorkUnderContext(LivePartitionUtility.java:57)
at weblogic.work.PartitionUtility.runWorkUnderContext(PartitionUtility.java:41)
at weblogic.work.SelfTuningWorkManagerImpl.runWorkUnderContext(SelfTuningWorkManagerImpl.java:655)
at weblogic.work.ExecuteThread.execute(ExecuteThread.java:420)
at weblogic.work.ExecuteThread.run(ExecuteThread.java:360)
Locked ownable synchronizers:
- None