Всем привет!
Все потоки моего приложения зависли на методе java.net.SocketInputStream.socketRead0, и израсходовали все доступные потоки на tomcat-сервере.
Среда: Apache Tomcat 5.5.20, java1.6_01, Apache Commons Http Client.
Дамп потока (все остальные такие же) приведу внизу темы.
После залипания всех потоков на этом методе сделал
lsof -p 4613 (4613 - id томкета. Эта команда позволяет посмотреть все файлы и сокеты, открытые данным указанным процессом)
Увидел, что там очень много строк следующего типа:
java 4613 root 1906u IPv6 303362 TCP shrek.office.myorg.net:http-alt->cover.myorg.net:58162 (CLOSE_WAIT)
java 4613 root 1907u IPv6 302767 TCP shrek.office.myorg.net:http-alt->cover.myorg.net:57358 (CLOSE_WAIT)
java 4613 root 1908u IPv6 302774 TCP shrek.office.myorg.net:http-alt->cover.myorg.net:57365 (CLOSE_WAIT)
java 4613 root 1909u IPv6 302783 TCP shrek.office.myorg.net:http-alt->cover.myorg.net:57382 (CLOSE_WAIT)
И вот такого типа:
java 4613 root 689u IPv6 277188 TCP shrek.office.myorg.net:33121->shrek.office.myorg.net:http-alt (ESTABLISHED)
java 4613 root 690u IPv6 277187 TCP shrek.office.myorg.net:http-alt->shrek.office.myorg.net:33121 (ESTABLISHED)
java 4613 root 691u IPv6 277191 TCP shrek.office.myorg.net:58069->shrek.office.myorg.net:http-alt (ESTABLISHED)
java 4613 root 692u IPv6 277190 TCP shrek.office.myorg.net:52543->shrek.office.myorg.net:http-alt (ESTABLISHED)
Здесь shrek - это текущий java-сервер. cover - стоящий впереди Apache Server с которого java иногда забирает страницы для вставки в текущую страницу.
Что можно сделать? Как исправить ситуацию и заставить нормально работать сайт?
Всем заранее спасибо!
Привожу дамп потоков, а вслед за ним код проблемных методов:
"http-8080-Processor1617 been through net.sf.ehcache.constructs.web.filter.SimplePageCachingFilter" daemon prio=10 tid=0x1d81a800 nid=0x1cc7 runnable [0xfad64000..0xfad66150]
java.lang.Thread.State: RUNNABLE
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.read(SocketInputStream.java:129)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
- locked <0xebfd77a0> (a java.io.BufferedInputStream)
at org.apache.commons.httpclient.HttpParser.readRawLine(Unknown Source)
at org.apache.commons.httpclient.HttpParser.readLine(Unknown Source)
at org.apache.commons.httpclient.HttpConnection.readLine(Unknown Source)
at org.apache.commons.httpclient.MultiThreadedHttpConnectionManager$HttpConnectionAdapter.readLine(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.readStatusLine(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.readResponse(Unknown Source)
at org.apache.commons.httpclient.HttpMethodBase.execute(Unknown Source)
at org.apache.commons.httpclient.HttpMethodDirector.executeWithRetry(Unknown Source)
at org.apache.commons.httpclient.HttpMethodDirector.executeMethod(Unknown Source)
at org.apache.commons.httpclient.HttpClient.executeMethod(Unknown Source)
at org.apache.commons.httpclient.HttpClient.executeMethod(Unknown Source)
at net.myorg.commons.HttpGetter.runGetCore(HttpGetter.java:46)
at net.myorg.commons.HttpGetter.getFromURLAsStream(HttpGetter.java:65)
at net.myorg.commons.HttpGetter.getFromCachedUrl(HttpGetter.java:129)
at net.myorg.portal.webClipping.View.pageBeginRender(View.java:36)
at org.apache.tapestry.AbstractPage.firePageBeginRender(AbstractPage.java:478)
at org.apache.tapestry.AbstractPage.renderPage(AbstractPage.java:268)
at org.apache.tapestry.engine.RequestCycle.renderPage(RequestCycle.java:366)
at org.apache.tapestry.portlet.PortletRendererImpl.renderPage(PortletRendererImpl.java:75)
at $PortletRenderer_1156ae882e2.renderPage($PortletRenderer_1156ae882e2.java)
at org.apache.tapestry.portlet.PortletHomeService.service(PortletHomeService.java:80)
at $IEngineService_1156ae882cc.service($IEngineService_1156ae882cc.java)
at org.apache.tapestry.services.impl.EngineServiceOuterProxy.service(EngineServiceOuterProxy.java:66)
at org.apache.tapestry.engine.AbstractEngine.service(AbstractEngine.java:248)
at org.apache.tapestry.services.impl.InvokeEngineTerminator.service(InvokeEngineTerminator.java:60)
at $WebRequestServicer_1156ae8829a.service($WebRequestServicer_1156ae8829a.java)
at $WebRequestServicer_1156ae88298.service($WebRequestServicer_1156ae88298.java)
at org.apache.tapestry.portlet.RenderRequestServicerToWebRequestServicerBridge.service(RenderRequestServicerToWebRequestServicerBridge.java:49)
at $RenderRequestServicer_1156ae88292.service($RenderRequestServicer_1156ae88292.java)
at $RenderRequestServicer_1156ae880a0.service($RenderRequestServicer_1156ae880a0.java)
at org.apache.tapestry.portlet.ApplicationPortlet.render(ApplicationPortlet.java:161)
at org.apache.jetspeed.factory.JetspeedPortletInstance.render(JetspeedPortletInstance.java:103)
at org.apache.jetspeed.container.JetspeedContainerServlet.doGet(JetspeedContainerServlet.java:265)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:689)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:672)
at org.apache.catalina.core.ApplicationDispatcher.doInclude(ApplicationDispatcher.java:574)
at org.apache.catalina.core.ApplicationDispatcher.include(ApplicationDispatcher.java:499)
at org.apache.jetspeed.container.invoker.ServletPortletInvoker.invoke(ServletPortletInvoker.java:245)
at org.apache.jetspeed.container.invoker.ServletPortletInvoker.render(ServletPortletInvoker.java:128)
at org.apache.pluto.PortletContainerImpl.renderPortlet(PortletContainerImpl.java:119)
at org.apache.jetspeed.container.JetspeedPortletContainerWrapper.renderPortlet(JetspeedPortletContainerWrapper.java:121)
at org.apache.jetspeed.aggregator.impl.RenderingJobImpl.execute(RenderingJobImpl.java:242)
at org.apache.jetspeed.aggregator.impl.PortletRendererImpl.renderNow(PortletRendererImpl.java:226)
at org.apache.jetspeed.aggregator.impl.PageAggregatorImpl.aggregateAndRender(PageAggregatorImpl.java:147)
at org.apache.jetspeed.aggregator.impl.PageAggregatorImpl.aggregateAndRender(PageAggregatorImpl.java:143)
at org.apache.jetspeed.aggregator.impl.PageAggregatorImpl.build(PageAggregatorImpl.java:78)
at org.apache.jetspeed.aggregator.AggregatorValve.invoke(AggregatorValve.java:46)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.aggregator.HeaderAggregatorValve.invoke(HeaderAggregatorValve.java:53)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.decoration.DecorationValve.invoke(DecorationValve.java:130)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.resource.ResourceValveImpl.invoke(ResourceValveImpl.java:130)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.pipeline.valve.impl.ActionValveImpl.invoke(ActionValveImpl.java:184)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.container.ContainerValve.invoke(ContainerValve.java:104)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.profiler.impl.ProfilerValveImpl.invoke(ProfilerValveImpl.java:248)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.security.impl.LoginValidationValveImpl.invoke(LoginValidationValveImpl.java:159)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.security.impl.PasswordCredentialValveImpl.invoke(PasswordCredentialValveImpl.java:150)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.localization.impl.LocalizationValveImpl.invoke(LocalizationValveImpl.java:170)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.security.impl.AbstractSecurityValve$1.run(AbstractSecurityValve.java:138)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAsPrivileged(Subject.java:454)
at org.apache.jetspeed.security.JSSubject.doAsPrivileged(JSSubject.java:179)
at org.apache.jetspeed.security.impl.AbstractSecurityValve.invoke(AbstractSecurityValve.java:132)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.container.url.impl.PortalURLValveImpl.invoke(PortalURLValveImpl.java:66)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.capabilities.impl.CapabilityValveImpl.invoke(CapabilityValveImpl.java:126)
at org.apache.jetspeed.pipeline.JetspeedPipeline$Invocation.invokeNext(JetspeedPipeline.java:167)
at org.apache.jetspeed.pipeline.JetspeedPipeline.invoke(JetspeedPipeline.java:146)
at org.apache.jetspeed.engine.JetspeedEngine.service(JetspeedEngine.java:227)
at org.apache.jetspeed.engine.JetspeedServlet.doGet(JetspeedServlet.java:242)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:689)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.jetspeed.engine.servlet.XXSUrlAttackFilter.doFilter(XXSUrlAttackFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at net.sf.ehcache.constructs.web.filter.CachingFilter.buildPage(CachingFilter.java:211)
at net.sf.ehcache.constructs.web.filter.CachingFilter.buildPageInfo(CachingFilter.java:164)
at net.sf.ehcache.constructs.web.filter.CachingFilter.doFilter(CachingFilter.java:130)
at net.sf.ehcache.constructs.web.filter.Filter.doFilter(Filter.java:92)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:202)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:432)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:869)
at org.apache.coyote.http11.Http11BaseProtocol$Http11ConnectionHandler.processConnection(Http11BaseProtocol.java:664)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:80)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684)
at java.lang.Thread.run(Thread.java:619)
Вот код класса HttpGetter:
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
public class HttpGetter {
HttpClient httpClient = new HttpClient( new MultiThreadedHttpConnectionManager());
private GetMethod method;
static CacheDao cacheDao = new CacheDao(CACHE_NAME);
public static String getFromCachedUrl(String url, String encoding) throws IOException {
HttpGetter httpGetter = new HttpGetter();
InputStream stream = null ;
String generatedString = (String) cacheDao.getFromCache((Object) url + "_" + encoding);
if (generatedString == null ) {
try {
stream = httpGetter.getFromURLAsStream(url);
generatedString = HtmlUtils.getPageBody(IOUtils.toString(stream, encoding));
cacheDao.putToCache(url + "_" + encoding, generatedString);
} finally {
IOUtils.closeQuietly(stream);
httpGetter.releaseConnection();
}
}
return generatedString;
}
private InputStream getFromURLAsStream(String url) throws HttpException, IOException {
return runGetCore(url).getResponseBodyAsStream();
}
private GetMethod runGetCore(String url) throws HttpException, IOException {
URI uri = new URI(url, false);
method = new GetMethod(uri.getEscapedURI());
method.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler( 3 , false));
int result = httpClient.executeMethod(method);
if (result != HttpStatus.SC_OK) {
method.releaseConnection();
throw new HttpException(method.getStatusLine().toString());
}
return method;
}
public void releaseConnection() {
method.releaseConnection();
}
}
