Cannot generate access tokens in WSO2 API Manager - wso2

I just set up WSO2 API Manager on a cloud server. Most of it currently works, but unfortunately I cannot generate access tokens. This is the error I observe:
TID: [-1234] [] [2016-01-14 15:12:15,866] ERROR {org.apache.catalina.core.StandardWrapperValve} - Servlet.service() for servlet [OAuth2Endpoints] in context with path [/oauth2] threw exception {org.apache.catalina.core.StandardWrapperValve}
java.lang.RuntimeException: org.apache.cxf.interceptor.Fault
at org.apache.cxf.interceptor.AbstractFaultChainInitiatorObserver.onMessage(AbstractFaultChainInitiatorObserver.java:116)
at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:336)
at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:249)
at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:248)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:222)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:153)
at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:171)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:289)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:209)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:646)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:265)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.wso2.carbon.ui.filters.CSRFPreventionFilter.doFilter(CSRFPreventionFilter.java:88)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.wso2.carbon.ui.filters.CRLFPreventionFilter.doFilter(CRLFPreventionFilter.java:59)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:504)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1074)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1739)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1698)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.cxf.interceptor.Fault
at org.apache.cxf.service.invoker.AbstractInvoker.createFault(AbstractInvoker.java:170)
at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:136)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:204)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:101)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:58)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:94)
at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:272)
... 44 more
Caused by: java.lang.NullPointerException
at org.wso2.carbon.identity.oauth2.OAuth2Service.issueAccessToken(OAuth2Service.java:214)
at org.wso2.carbon.identity.oauth.endpoint.token.OAuth2TokenEndpoint.getAccessToken(OAuth2TokenEndpoint.java:245)
at org.wso2.carbon.identity.oauth.endpoint.token.OAuth2TokenEndpoint.issueAccessToken(OAuth2TokenEndpoint.java:111)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:188)
at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:104)
... 49 more
TID: [-1234] [] [2016-01-14 15:12:15,938] ERROR {JAGGERY.site.blocks.subscription.subscription-add.ajax.subscription-add:jag} - java.lang.RuntimeException: Error occurred while calling token endpoint: HTTP error code : 500 {JAGGERY.site.blocks.subscription.subscription-add.ajax.subscription-add:jag}
Does anyone have a clue what is going on? I didn't observe any startup warnings or errors.
Update: Since I was asked for a bit more configuration information, this is what I changed from the default config files.
I set the HostName in carbon.xml to my address
I changed the ServerURL and the GatewayEndpoint (APIGateway -> Environments -> Environment(type="hybrid")) to contain my address rather than the local IP. This was changed in the api-manager.xml.
The firewall allows connections to port 8280, 8243, 10397, 9763 & 9443.

Related

Intermittent failure while publishing API in wso2 API manager

We are having WSO2 API Manager 2.1.0 setup in a clustered environment. API publication is failing intermittently. API publication request from Publisher to Gateway manager is served by AWS elastic load balancer. When the log file of publisher is analyzed, it shows the following error.
INFO {org.apache.axis2.transport.http.HTTPSender} - Unable to sendViaPost to url[https://LB-DNS:PORT/services/APIGatewayAdmin] {org.apache.axis2.transport.http.HTTPSender}
org.apache.axis2.AxisFault: Transport error: 504 Error: GATEWAY_TIMEOUT
at org.apache.axis2.transport.http.HTTPSender.handleResponse(HTTPSender.java:326)
at org.apache.axis2.transport.http.HTTPSender.sendViaPost(HTTPSender.java:196)
at org.apache.axis2.transport.http.HTTPSender.send(HTTPSender.java:77)
at org.apache.axis2.transport.http.CommonsHTTPTransportSender.writeMessageWithCommons(CommonsHTTPTransportSender.java:451)
at org.apache.axis2.transport.http.CommonsHTTPTransportSender.invoke(CommonsHTTPTransportSender.java:278)
at org.apache.axis2.engine.AxisEngine.send(AxisEngine.java:442)
at org.apache.axis2.description.OutInAxisOperationClient.send(OutInAxisOperation.java:430)
at org.apache.axis2.description.OutInAxisOperationClient.executeImpl(OutInAxisOperation.java:225)
at org.apache.axis2.client.OperationClient.execute(OperationClient.java:149)
at org.wso2.carbon.apimgt.gateway.stub.APIGatewayAdminStub.getApi(APIGatewayAdminStub.java:7012)
at org.wso2.carbon.apimgt.impl.utils.APIGatewayAdminClient.getApi(APIGatewayAdminClient.java:145)
at org.wso2.carbon.apimgt.impl.APIGatewayManager.publishToGateway(APIGatewayManager.java:102)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.publishToGateway(APIProviderImpl.java:1519)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.propergateAPIStatusChangeToGateways(APIProviderImpl.java:1328)
at org.wso2.carbon.apimgt.impl.UserAwareAPIProvider.propergateAPIStatusChangeToGateways(UserAwareAPIProvider.java:36)
at org.wso2.carbon.apimgt.impl.executors.APIExecutor.execute(APIExecutor.java:136)
at org.wso2.carbon.governance.registry.extensions.aspects.DefaultLifeCycle.runCustomExecutorsCode(DefaultLifeCycle.java:712)
at org.wso2.carbon.governance.registry.extensions.aspects.DefaultLifeCycle.invoke(DefaultLifeCycle.java:450)
at org.wso2.carbon.governance.registry.eventing.handlers.GovernanceEventingHandler.invokeAspect(GovernanceEventingHandler.java:338)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerManager.invokeAspect(HandlerManager.java:2792)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerLifecycleManager.invokeAspect(HandlerLifecycleManager.java:518)
at org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.invokeAspect(EmbeddedRegistry.java:2591)
at org.wso2.carbon.registry.core.caching.CacheBackedRegistry.invokeAspect(CacheBackedRegistry.java:510)
at org.wso2.carbon.registry.core.session.UserRegistry.invokeAspect(UserRegistry.java:1952)
at org.wso2.carbon.governance.api.common.dataobjects.GovernanceArtifactImpl.invokeAction(GovernanceArtifactImpl.java:829)
at org.wso2.carbon.governance.api.common.dataobjects.GovernanceArtifactImpl.invokeAction(GovernanceArtifactImpl.java:808)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.changeLifeCycleStatus(APIProviderImpl.java:3755)
at org.wso2.carbon.apimgt.impl.UserAwareAPIProvider.changeLifeCycleStatus(UserAwareAPIProvider.java:148)
at org.wso2.carbon.apimgt.rest.api.publisher.impl.ApisApiServiceImpl.apisChangeLifecyclePost(ApisApiServiceImpl.java:303)
at org.wso2.carbon.apimgt.rest.api.publisher.ApisApi.apisChangeLifecyclePost(ApisApi.java:520)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:180)
at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:189)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:99)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96)
at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308)
at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:254)
at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)
at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:180)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:299)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:218)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:274)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatSessionReplicationValve.invoke(CarbonTomcatSessionReplicationValve.java:45)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:958)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.ha.tcp.ReplicationValve.invoke(ReplicationValve.java:333)
at org.apache.catalina.ha.session.JvmRouteBinderValve.invoke(JvmRouteBinderValve.java:218)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:452)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1087)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1756)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1715)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
ERROR {org.wso2.carbon.apimgt.impl.APIGatewayManager} - Error occurred when publish to gateway Production and Sandbox {org.wso2.carbon.apimgt.impl.APIGatewayManager}
org.apache.axis2.AxisFault: Error while obtaining API information from gateway. Transport error: 504 Error: GATEWAY_TIMEOUT
at org.wso2.carbon.apimgt.impl.utils.APIGatewayAdminClient.getApi(APIGatewayAdminClient.java:149)
at org.wso2.carbon.apimgt.impl.APIGatewayManager.publishToGateway(APIGatewayManager.java:102)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.publishToGateway(APIProviderImpl.java:1519)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.propergateAPIStatusChangeToGateways(APIProviderImpl.java:1328)
at org.wso2.carbon.apimgt.impl.UserAwareAPIProvider.propergateAPIStatusChangeToGateways(UserAwareAPIProvider.java:36)
at org.wso2.carbon.apimgt.impl.executors.APIExecutor.execute(APIExecutor.java:136)
at org.wso2.carbon.governance.registry.extensions.aspects.DefaultLifeCycle.runCustomExecutorsCode(DefaultLifeCycle.java:712)
at org.wso2.carbon.governance.registry.extensions.aspects.DefaultLifeCycle.invoke(DefaultLifeCycle.java:450)
at org.wso2.carbon.governance.registry.eventing.handlers.GovernanceEventingHandler.invokeAspect(GovernanceEventingHandler.java:338)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerManager.invokeAspect(HandlerManager.java:2792)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerLifecycleManager.invokeAspect(HandlerLifecycleManager.java:518)
at org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.invokeAspect(EmbeddedRegistry.java:2591)
at org.wso2.carbon.registry.core.caching.CacheBackedRegistry.invokeAspect(CacheBackedRegistry.java:510)
at org.wso2.carbon.registry.core.session.UserRegistry.invokeAspect(UserRegistry.java:1952)
at org.wso2.carbon.governance.api.common.dataobjects.GovernanceArtifactImpl.invokeAction(GovernanceArtifactImpl.java:829)
at org.wso2.carbon.governance.api.common.dataobjects.GovernanceArtifactImpl.invokeAction(GovernanceArtifactImpl.java:808)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.changeLifeCycleStatus(APIProviderImpl.java:3755)
at org.wso2.carbon.apimgt.impl.UserAwareAPIProvider.changeLifeCycleStatus(UserAwareAPIProvider.java:148)
at org.wso2.carbon.apimgt.rest.api.publisher.impl.ApisApiServiceImpl.apisChangeLifecyclePost(ApisApiServiceImpl.java:303)
at org.wso2.carbon.apimgt.rest.api.publisher.ApisApi.apisChangeLifecyclePost(ApisApi.java:520)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:180)
at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:189)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:99)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96)
at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308)
at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:254)
at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)
at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:180)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:299)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:218)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:274)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatSessionReplicationValve.invoke(CarbonTomcatSessionReplicationValve.java:45)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:958)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.ha.tcp.ReplicationValve.invoke(ReplicationValve.java:333)
at org.apache.catalina.ha.session.JvmRouteBinderValve.invoke(JvmRouteBinderValve.java:218)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:452)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1087)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1756)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1715)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.axis2.AxisFault: Transport error: 504 Error: GATEWAY_TIMEOUT
at org.apache.axis2.transport.http.HTTPSender.handleResponse(HTTPSender.java:326)
at org.apache.axis2.transport.http.HTTPSender.sendViaPost(HTTPSender.java:196)
at org.apache.axis2.transport.http.HTTPSender.send(HTTPSender.java:77)
at org.apache.axis2.transport.http.CommonsHTTPTransportSender.writeMessageWithCommons(CommonsHTTPTransportSender.java:451)
at org.apache.axis2.transport.http.CommonsHTTPTransportSender.invoke(CommonsHTTPTransportSender.java:278)
at org.apache.axis2.engine.AxisEngine.send(AxisEngine.java:442)
at org.apache.axis2.description.OutInAxisOperationClient.send(OutInAxisOperation.java:430)
at org.apache.axis2.description.OutInAxisOperationClient.executeImpl(OutInAxisOperation.java:225)
at org.apache.axis2.client.OperationClient.execute(OperationClient.java:149)
at org.wso2.carbon.apimgt.gateway.stub.APIGatewayAdminStub.getApi(APIGatewayAdminStub.java:7012)
at org.wso2.carbon.apimgt.impl.utils.APIGatewayAdminClient.getApi(APIGatewayAdminClient.java:145)
... 71 more
Below error messages found in Gateway manager
ERROR {org.apache.synapse.transport.passthru.PassThroughHttpSSLSender} - IO Error sending response message {org.apache.synapse.transport.passthru.PassThroughHttpSSLSender}
org.apache.axis2.AxisFault: java.io.IOException: Broken pipe
at org.apache.axis2.AxisFault.makeFault(AxisFault.java:430)
at org.apache.axis2.transport.http.SOAPMessageFormatter.writeTo(SOAPMessageFormatter.java:78)
at org.apache.synapse.transport.passthru.PassThroughHttpSender.sendUsingOutputStream(PassThroughHttpSender.java:307)
at org.apache.synapse.transport.passthru.PassThroughHttpSender.invoke(PassThroughHttpSender.java:271)
at org.apache.axis2.engine.AxisEngine.sendFault(AxisEngine.java:525)
at org.apache.axis2.transport.http.AxisServlet.handleFault(AxisServlet.java:433)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:216)
at org.wso2.carbon.core.transports.CarbonServlet.doPost(CarbonServlet.java:231)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.eclipse.equinox.http.servlet.internal.ServletRegistration.service(ServletRegistration.java:61)
at org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:128)
at org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:68)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.wso2.carbon.tomcat.ext.servlet.DelegationServlet.service(DelegationServlet.java:68)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.owasp.csrfguard.CsrfGuardFilter.doFilter(CsrfGuardFilter.java:88)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.wso2.carbon.tomcat.ext.filter.CharacterSetFilter.doFilter(CharacterSetFilter.java:61)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:120)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:958)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:452)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1087)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1756)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1715)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: com.ctc.wstx.exc.WstxIOException: java.io.IOException: Broken pipe
at com.ctc.wstx.sw.BaseStreamWriter._finishDocument(BaseStreamWriter.java:1406)
at com.ctc.wstx.sw.BaseStreamWriter.close(BaseStreamWriter.java:247)
at org.apache.axiom.om.impl.MTOMXMLStreamWriter.close(MTOMXMLStreamWriter.java:188)
at org.apache.axiom.om.impl.llom.OMSerializableImpl.serializeAndConsume(OMSerializableImpl.java:197)
at org.apache.axis2.transport.http.SOAPMessageFormatter.writeTo(SOAPMessageFormatter.java:74)
... 50 more
Caused by: org.apache.catalina.connector.ClientAbortException: java.io.IOException: Broken pipe
at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:370)
at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:334)
at org.apache.catalina.connector.CoyoteOutputStream.flush(CoyoteOutputStream.java:101)
at com.compuware.apm.agent.introspection.uem.impl.AgentInjectingServletOutputStream.flush(AgentInjectingServletOutputStream.java:168)
at com.ctc.wstx.io.UTF8Writer.flush(UTF8Writer.java:103)
at com.ctc.wstx.sw.BufferingXmlWriter.flush(BufferingXmlWriter.java:225)
at com.ctc.wstx.sw.BufferingXmlWriter.close(BufferingXmlWriter.java:198)
at com.ctc.wstx.sw.BaseStreamWriter._finishDocument(BaseStreamWriter.java:1404)
... 54 more
Caused by: java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:51)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.apache.tomcat.util.net.SecureNioChannel.flush(SecureNioChannel.java:141)
at org.apache.tomcat.util.net.SecureNioChannel.write(SecureNioChannel.java:502)
at org.apache.tomcat.util.net.NioBlockingSelector.write(NioBlockingSelector.java:101)
at org.apache.tomcat.util.net.NioSelectorPool.write(NioSelectorPool.java:174)
at org.apache.coyote.http11.InternalNioOutputBuffer.writeToSocket(InternalNioOutputBuffer.java:164)
at org.apache.coyote.http11.InternalNioOutputBuffer.flushBuffer(InternalNioOutputBuffer.java:243)
at org.apache.coyote.http11.InternalNioOutputBuffer.flush(InternalNioOutputBuffer.java:95)
at org.apache.coyote.http11.AbstractHttp11Processor.action(AbstractHttp11Processor.java:800)
at org.apache.coyote.Response.action(Response.java:172)
at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:366)
at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:334)
at org.apache.catalina.connector.CoyoteOutputStream.flush(CoyoteOutputStream.java:101)
at com.compuware.apm.agent.introspection.uem.impl.AgentInjectingServletOutputStream.flush(AgentInjectingServletOutputStream.java:168)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:141)
at org.apache.axis2.transport.http.AxisServlet.doPost(AxisServlet.java:182)
... 45 more
We were able to resolve this problem by analysing possible scenarios request can hang. We are using AWS EFS for sharing synapse files(APIM_HOME/repository/deployment/server/synapse-configs/default) between the gateway manager and gateway worker. While Publishing an API, Publisher will call APIGateway admin service which includes EFS file creation operation. We had created EFS with Burst mode. In burst mode, throughput is determined using burst credit. Since our EFS burst credit reduced to zero, EFS doesn't have enough throughput to do i/o operations in EFS. Changing the throughput mode to Provisioned mode solved the issue.
Useful links
https://docs.aws.amazon.com/efs/latest/ug/performance.html#throughput-modes
https://aws.amazon.com/premiumsupport/knowledge-center/efs-burst-credits/

ERROR occured when I call APIM Rest API. (in multiple user)

I use WSO2 API-Manager 2.1.0 with MaribDB.
Recently I've done some test with APIM Rest APIs such as create api, change life-cycle, and create application.
The point of my test was to check how to APIM work when receive multiple requests.
so I made a situation that 10 users call same RestAPI at the same time.
During the test, I faced to unexpected ERROR. However I coundn't found out what cause that kind of ERROR.
so I changed some code(am#api#publisher) to make them work synchronized.
I got the result with less ERROR but them still occrued.
Can somebody give me some advise? ( Please check the ERROR Log)
TID: [-1234] [] [2018-07-03 13:41:49,736] ERROR {org.wso2.carbon.apimgt.impl.executors.APIExecutor} - Failed to get the generic artifact while executing APIExecutor. {org.wso2.carbon.apimgt.impl.executors.APIExecutor}
org.wso2.carbon.governance.api.exception.GovernanceException: Invalid check list item.
at org.wso2.carbon.governance.api.common.dataobjects.GovernanceArtifactImpl.isLCItemChecked(GovernanceArtifactImpl.java:897)
at org.wso2.carbon.apimgt.impl.executors.APIExecutor.execute(APIExecutor.java:170)
at org.wso2.carbon.governance.registry.extensions.aspects.DefaultLifeCycle.runCustomExecutorsCode(DefaultLifeCycle.java:712)
at org.wso2.carbon.governance.registry.extensions.aspects.DefaultLifeCycle.invoke(DefaultLifeCycle.java:450)
at org.wso2.carbon.governance.registry.eventing.handlers.GovernanceEventingHandler.invokeAspect(GovernanceEventingHandler.java:338)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerManager.invokeAspect(HandlerManager.java:2792)
at org.wso2.carbon.registry.core.jdbc.handlers.HandlerLifecycleManager.invokeAspect(HandlerLifecycleManager.java:518)
at org.wso2.carbon.registry.core.jdbc.EmbeddedRegistry.invokeAspect(EmbeddedRegistry.java:2591)
at org.wso2.carbon.registry.core.caching.CacheBackedRegistry.invokeAspect(CacheBackedRegistry.java:510)
at org.wso2.carbon.registry.core.session.UserRegistry.invokeAspect(UserRegistry.java:1952)
at org.wso2.carbon.governance.api.common.dataobjects.GovernanceArtifactImpl.invokeAction(GovernanceArtifactImpl.java:829)
at org.wso2.carbon.governance.api.common.dataobjects.GovernanceArtifactImpl.invokeAction(GovernanceArtifactImpl.java:808)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.changeLifeCycleStatus(APIProviderImpl.java:3755)
at org.wso2.carbon.apimgt.impl.UserAwareAPIProvider.changeLifeCycleStatus(UserAwareAPIProvider.java:148)
at org.wso2.carbon.apimgt.rest.api.publisher.impl.ApisApiServiceImpl.apisChangeLifecyclePost(ApisApiServiceImpl.java:303)
at org.wso2.carbon.apimgt.rest.api.publisher.ApisApi.apisChangeLifecyclePost(ApisApi.java:520)
at sun.reflect.GeneratedMethodAccessor241.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:180)
at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:189)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:99)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96)
at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308)
at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:254)
at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)
at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:180)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:299)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doPost(AbstractHTTPServlet.java:218)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:274)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.event.receiver.core.internal.tenantmgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:48)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:958)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:452)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1087)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1756)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1715)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)

WSO2 IoT 3.1.0 failed to retreive oauth token using jwt error

I am installing WSO2 IOT Server 3.1.0 on a server. I just want to run it with default basic configurations. After Uploading to my server, I configured it to run with the IP using the script as mentioned in this link.
After running the script, I ran the broker and the iotserver profiles. I am able to log in the API store and publisher but when I tries to log in the devicemgt web console. I am getting the following error.
Error Snapshot:
Exception In Logs:
An exception thrown when executing the script '/app/modules/login.js'. {auth-module}
TID: [-1234] [] [2017-12-04 00:32:43,944] ERROR {auth-module} - org.wso2.carbon.apimgt.integration.client.exception.APIMClientOAuthException: failed to retrieve oauth token using jwt {auth-module}
org.mozilla.javascript.WrappedException: Wrapped org.wso2.carbon.apimgt.integration.client.exception.APIMClientOAuthException: failed to retrieve oauth token using jwt (/devicemgt/app/modules/login.js#31)
at org.mozilla.javascript.Context.throwAsScriptRuntimeEx(Context.java:1754)
at org.mozilla.javascript.MemberBox.invoke(MemberBox.java:148)
at org.mozilla.javascript.NativeJavaMethod.call(NativeJavaMethod.java:225)
at org.mozilla.javascript.optimizer.OptRuntime.callProp0(OptRuntime.java:85)
at org.jaggeryjs.rhino.devicemgt.app.modules.c3._c_anonymous_2(/devicemgt/app/modules/login.js:31)
at org.jaggeryjs.rhino.devicemgt.app.modules.c3.call(/devicemgt/app/modules/login.js)
at org.mozilla.javascript.optimizer.OptRuntime.call1(OptRuntime.java:32)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c0._c_executeScript_10(/devicemgt/lib/modules/auth/auth.js:234)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c0.call(/devicemgt/lib/modules/auth/auth.js)
at org.mozilla.javascript.optimizer.OptRuntime.callName(OptRuntime.java:63)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c0._c_handleEvent_11(/devicemgt/lib/modules/auth/auth.js:259)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c0.call(/devicemgt/lib/modules/auth/auth.js)
at org.mozilla.javascript.optimizer.OptRuntime.callName(OptRuntime.java:63)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c0._c_anonymous_20(/devicemgt/lib/modules/auth/auth.js:547)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c0.call(/devicemgt/lib/modules/auth/auth.js)
at org.mozilla.javascript.optimizer.OptRuntime.call2(OptRuntime.java:42)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c2._c_anonymous_1(/devicemgt/lib/modules/auth/acs.jag:34)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c2.call(/devicemgt/lib/modules/auth/acs.jag)
at org.mozilla.javascript.optimizer.OptRuntime.call0(OptRuntime.java:23)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c2._c_script_0(/devicemgt/lib/modules/auth/acs.jag:22)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c2.call(/devicemgt/lib/modules/auth/acs.jag)
at org.mozilla.javascript.ContextFactory.doTopCall(ContextFactory.java:394)
at org.mozilla.javascript.ScriptRuntime.doTopCall(ScriptRuntime.java:3091)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c2.call(/devicemgt/lib/modules/auth/acs.jag)
at org.jaggeryjs.rhino.devicemgt.lib.modules.auth.c2.exec(/devicemgt/lib/modules/auth/acs.jag)
at org.jaggeryjs.scriptengine.engine.RhinoEngine.execScript(RhinoEngine.java:567)
at org.jaggeryjs.scriptengine.engine.RhinoEngine.exec(RhinoEngine.java:273)
at org.jaggeryjs.jaggery.core.manager.WebAppManager.exec(WebAppManager.java:588)
at org.jaggeryjs.jaggery.core.manager.WebAppManager.execute(WebAppManager.java:508)
at org.jaggeryjs.jaggery.core.JaggeryServlet.doPost(JaggeryServlet.java:29)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:747)
at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:485)
at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:377)
at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:337)
at org.jaggeryjs.jaggery.core.JaggeryFilter.doFilter(JaggeryFilter.java:21)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.wso2.carbon.ui.filters.cache.ContentTypeBasedCachePreventionFilter.doFilter(ContentTypeBasedCachePreventionFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.owasp.csrfguard.CsrfGuardFilter.doFilter(CsrfGuardFilter.java:88)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:120)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:506)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.webapp.authenticator.framework.WebappAuthenticationValve.invoke(WebappAuthenticationValve.java:45)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:962)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:452)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1087)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1760)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1719)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.wso2.carbon.apimgt.integration.client.exception.APIMClientOAuthException: failed to retrieve oauth token using jwt
at org.wso2.carbon.apimgt.integration.client.OAuthRequestInterceptor.apply(OAuthRequestInterceptor.java:118)
at feign.SynchronousMethodHandler.targetRequest(SynchronousMethodHandler.java:158)
at feign.SynchronousMethodHandler.executeAndDecode(SynchronousMethodHandler.java:88)
at feign.SynchronousMethodHandler.invoke(SynchronousMethodHandler.java:76)
at feign.ReflectiveFeign$FeignInvocationHandler.invoke(ReflectiveFeign.java:103)
at com.sun.proxy.$Proxy38.tiersTierLevelTierNameGet(Unknown Source)
at org.wso2.carbon.apimgt.application.extension.APIManagementProviderServiceImpl.isTierLoaded(APIManagementProviderServiceImpl.java:56)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.mozilla.javascript.MemberBox.invoke(MemberBox.java:126)
... 75 more
Caused by: org.wso2.carbon.identity.jwt.client.extension.exception.JWTClientException: Error when parsing the response <am:fault xmlns:am="http://wso2.org/apimanager"><am:code>101500</am:code><am:type>Status report</am:type><am:message>Runtime Error</am:message><am:description>Error in Sender</am:description></am:fault>
at org.wso2.carbon.identity.jwt.client.extension.JWTClient.getTokenInfo(JWTClient.java:169)
at org.wso2.carbon.identity.jwt.client.extension.JWTClient.getAccessToken(JWTClient.java:79)
at org.wso2.carbon.apimgt.integration.client.OAuthRequestInterceptor.apply(OAuthRequestInterceptor.java:99)
... 86 more
Caused by: Unexpected character (<) at position 0.
at org.json.simple.parser.Yylex.yylex(Unknown Source)
at org.json.simple.parser.JSONParser.nextToken(Unknown Source)
at org.json.simple.parser.JSONParser.parse(Unknown Source)
at org.json.simple.parser.JSONParser.parse(Unknown Source)
at org.json.simple.parser.JSONParser.parse(Unknown Source)
at org.wso2.carbon.identity.jwt.client.extension.JWTClient.getTokenInfo(JWTClient.java:153)
... 88 more
Can anybody help pointing out what is going wrong and what is happening?
Are you using the same IP as the one you entered when asked for the common name when running the change IP script, to access the server?
Appreciate if you could add here the entire log for server startup. That way we would be able to figure out what exactly is causing the problem.

WSO2 API Manager issue while publishing Soap Endpoint

Iam new to WSO2 API manager.Iam trying to publish soap endpoint in APi publisher and iam getting below exception.When i open the wsdl in browser iam able to open it successfully. In Design->Api Definition when i click on Test URL it shows as valid.
Can you please help me with the issue.
Wso2 server logs:
[2017-06-14 16:27:58,802] ERROR - add:jag org.mozilla.javascript.WrappedException: Wrapped org.wso2.carbon.apimgt.api.APIManagementException: Failed to process the WSDL : http://system.dcm.allianz:25001/company/esb/api/pricing/eu/deferred/RatingPricingService?wsdl (/publisher/modules/api/add.jag#134)
at org.mozilla.javascript.Context.throwAsScriptRuntimeEx(Context.java:1754)
at org.mozilla.javascript.MemberBox.invoke(MemberBox.java:148)
at org.mozilla.javascript.FunctionObject.call(FunctionObject.java:386)
at org.mozilla.javascript.optimizer.OptRuntime.call1(OptRuntime.java:32)
at org.jaggeryjs.rhino.publisher.modules.api.c3._c_anonymous_6(/publisher/modules/api/add.jag:134)
at org.jaggeryjs.rhino.publisher.modules.api.c3.call(/publisher/modules/api/add.jag)
at org.mozilla.javascript.ScriptRuntime.applyOrCall(ScriptRuntime.java:2430)
at org.mozilla.javascript.BaseFunction.execIdCall(BaseFunction.java:269)
at org.mozilla.javascript.IdFunctionObject.call(IdFunctionObject.java:97)
at org.mozilla.javascript.optimizer.OptRuntime.call2(OptRuntime.java:42)
at org.jaggeryjs.rhino.publisher.modules.api.c0._c_anonymous_40(/publisher/modules/api/module.jag:127)
at org.jaggeryjs.rhino.publisher.modules.api.c0.call(/publisher/modules/api/module.jag)
at org.mozilla.javascript.optimizer.OptRuntime.call1(OptRuntime.java:32)
at org.jaggeryjs.rhino.publisher.site.blocks.item_design.ajax.c0._c_anonymous_3(/publisher/site/blocks/item-design/ajax/add.jag:189)
at org.jaggeryjs.rhino.publisher.site.blocks.item_design.ajax.c0.call(/publisher/site/blocks/item-design/ajax/add.jag)
at org.mozilla.javascript.optimizer.OptRuntime.call0(OptRuntime.java:23)
at org.jaggeryjs.rhino.publisher.site.blocks.item_design.ajax.c0._c_script_0(/publisher/site/blocks/item-design/ajax/add.jag:21)
at org.jaggeryjs.rhino.publisher.site.blocks.item_design.ajax.c0.call(/publisher/site/blocks/item-design/ajax/add.jag)
at org.mozilla.javascript.ContextFactory.doTopCall(ContextFactory.java:394)
at org.mozilla.javascript.ScriptRuntime.doTopCall(ScriptRuntime.java:3091)
at org.jaggeryjs.rhino.publisher.site.blocks.item_design.ajax.c0.call(/publisher/site/blocks/item-design/ajax/add.jag)
at org.jaggeryjs.rhino.publisher.site.blocks.item_design.ajax.c0.exec(/publisher/site/blocks/item-design/ajax/add.jag)
at org.jaggeryjs.scriptengine.engine.RhinoEngine.execScript(RhinoEngine.java:567)
at org.jaggeryjs.scriptengine.engine.RhinoEngine.exec(RhinoEngine.java:273)
at org.jaggeryjs.jaggery.core.manager.WebAppManager.exec(WebAppManager.java:588)
at org.jaggeryjs.jaggery.core.manager.WebAppManager.execute(WebAppManager.java:508)
at org.jaggeryjs.jaggery.core.JaggeryServlet.doPost(JaggeryServlet.java:29)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:650)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:747)
at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:485)
at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:377)
at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:337)
at org.jaggeryjs.jaggery.core.JaggeryFilter.doFilter(JaggeryFilter.java:21)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.wso2.carbon.ui.filters.cache.ContentTypeBasedCachePreventionFilter.doFilter(ContentTypeBasedCachePreventionFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:120)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:218)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.event.receiver.core.internal.tenantmgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:48)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:958)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:452)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1087)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1756)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1715)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.wso2.carbon.apimgt.api.APIManagementException: Failed to process the WSDL : http://system.dcm.allianz:25001/company/esb/api/pricing/eu/deferred/RatingPricingService?wsdl
at org.wso2.carbon.apimgt.impl.utils.APIUtil.createWSDL(APIUtil.java:1400)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.updateWsdl(APIProviderImpl.java:731)
at org.wso2.carbon.apimgt.impl.APIProviderImpl.updateAPI(APIProviderImpl.java:836)
at org.wso2.carbon.apimgt.impl.UserAwareAPIProvider.manageAPI(UserAwareAPIProvider.java:72)
at org.wso2.carbon.apimgt.hostobjects.APIProviderHostObject.saveAPI(APIProviderHostObject.java:1061)
at org.wso2.carbon.apimgt.hostobjects.APIProviderHostObject.jsFunction_updateAPIDesign(APIProviderHostObject.java:900)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.mozilla.javascript.MemberBox.invoke(MemberBox.java:126)
... 66 more
Caused by: org.wso2.carbon.apimgt.api.APIManagementException: Error occurs when change the addres URL of the WSDL
at org.wso2.carbon.apimgt.impl.utils.APIMWSDLReader.readAndCleanWsdl(APIMWSDLReader.java:113)
at org.wso2.carbon.apimgt.impl.utils.APIUtil.createWSDL(APIUtil.java:1358)
... 76 more
Caused by: org.wso2.carbon.apimgt.api.APIManagementException: Error occurred while getting the wsdl address location
at org.wso2.carbon.apimgt.impl.utils.APIMWSDLReader.setServiceDefinition(APIMWSDLReader.java:316)
at org.wso2.carbon.apimgt.impl.utils.APIMWSDLReader.readAndCleanWsdl(APIMWSDLReader.java:97)
... 77 more
Caused by: java.net.MalformedURLException: no protocol: tba
at java.net.URL.<init>(URL.java:589)
at java.net.URL.<init>(URL.java:486)
at java.net.URL.<init>(URL.java:435)
at org.wso2.carbon.apimgt.impl.utils.APIMWSDLReader.setServiceDefinition(APIMWSDLReader.java:307)
... 78 more

wso2 am 1.10.0: Error occurred while executing the action generateApplicationKey

I got this error before (That one is related to APIAuthenticationHandler API authentication failure due to Invalid Credentials)
, but this time the error messages from the server log are different. When I try to generate keys in store/subscription screen, I got "Error occurred while executing the action generateApplicationKey" error.
Server log:
==> wso2carbon.log <==
TID: [-1234] [] [2016-08-11 01:21:14,600] ERROR {org.wso2.carbon.apimgt.hostobjects.APIStoreHostObject} - **Error while obtaining the application access token for the application:…** {org.wso2.carbon.apimgt.hostobjects.APIStoreHostObject}
org.wso2.carbon.apimgt.api.APIManagementException: Could not execute Workflow
at org.wso2.carbon.apimgt.impl.APIConsumerImpl.requestApprovalForApplicationRegistration(APIConsumerImpl.java:2651)
at org.wso2.carbon.apimgt.impl.UserAwareAPIConsumer.requestApprovalForApplicationRegistration(UserAwareAPIConsumer.java:36)
at org.wso2.carbon.apimgt.hostobjects.APIStoreHostObject.jsFunction_getApplicationKey(APIStoreHostObject.java:380)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.mozilla.javascript.MemberBox.invoke(MemberBox.java:126)
at org.mozilla.javascript.FunctionObject.call(FunctionObject.java:386)
at org.mozilla.javascript.optimizer.OptRuntime.callN(OptRuntime.java:52)
at org.jaggeryjs.rhino.store.modules.subscription.c2._c_anonymous_2(/store/modules/subscription/key.jag:39)
at org.jaggeryjs.rhino.store.modules.subscription.c2.call(/store/modules/subscription/key.jag)
at org.mozilla.javascript.ScriptRuntime.applyOrCall(ScriptRuntime.java:2430)
at org.mozilla.javascript.BaseFunction.execIdCall(BaseFunction.java:269)
at org.mozilla.javascript.IdFunctionObject.call(IdFunctionObject.java:97)
at org.mozilla.javascript.optimizer.OptRuntime.call2(OptRuntime.java:42)
at org.jaggeryjs.rhino.store.modules.subscription.c0._c_anonymous_9(/store/modules/subscription/module.jag:32)
at org.jaggeryjs.rhino.store.modules.subscription.c0.call(/store/modules/subscription/module.jag)
at org.mozilla.javascript.optimizer.OptRuntime.callN(OptRuntime.java:52)
at org.jaggeryjs.rhino.store.site.blocks.subscription.subscription_add.ajax.c0._c_anonymous_1(/store/site/blocks/subscription/subscription-add/ajax/subscription-add.jag:240)
at org.jaggeryjs.rhino.store.site.blocks.subscription.subscription_add.ajax.c0.call(/store/site/blocks/subscription/subscription-add/ajax/subscription-add.jag)
at org.mozilla.javascript.optimizer.OptRuntime.call0(OptRuntime.java:23)
at org.jaggeryjs.rhino.store.site.blocks.subscription.subscription_add.ajax.c0._c_script_0(/store/site/blocks/subscription/subscription-add/ajax/subscription-add.jag:3)
at org.jaggeryjs.rhino.store.site.blocks.subscription.subscription_add.ajax.c0.call(/store/site/blocks/subscription/subscription-add/ajax/subscription-add.jag)
at org.mozilla.javascript.ContextFactory.doTopCall(ContextFactory.java:394)
at org.mozilla.javascript.ScriptRuntime.doTopCall(ScriptRuntime.java:3091)
at org.jaggeryjs.rhino.store.site.blocks.subscription.subscription_add.ajax.c0.call(/store/site/blocks/subscription/subscription-add/ajax/subscription-add.jag)
at org.jaggeryjs.rhino.store.site.blocks.subscription.subscription_add.ajax.c0.exec(/store/site/blocks/subscription/subscription-add/ajax/subscription-add.jag)
at org.jaggeryjs.scriptengine.engine.RhinoEngine.execScript(RhinoEngine.java:567)
at org.jaggeryjs.scriptengine.engine.RhinoEngine.exec(RhinoEngine.java:273)
at org.jaggeryjs.jaggery.core.manager.WebAppManager.exec(WebAppManager.java:587)
at org.jaggeryjs.jaggery.core.manager.WebAppManager.execute(WebAppManager.java:507)
at org.jaggeryjs.jaggery.core.JaggeryServlet.doPost(JaggeryServlet.java:29)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:646)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.ApplicationDispatcher.invoke(ApplicationDispatcher.java:748)
at org.apache.catalina.core.ApplicationDispatcher.processRequest(ApplicationDispatcher.java:486)
at org.apache.catalina.core.ApplicationDispatcher.doForward(ApplicationDispatcher.java:378)
at org.apache.catalina.core.ApplicationDispatcher.forward(ApplicationDispatcher.java:338)
at org.jaggeryjs.jaggery.core.JaggeryFilter.doFilter(JaggeryFilter.java:21)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:504)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.continueInvocation(CompositeValve.java:99)
at org.wso2.carbon.tomcat.ext.valves.CarbonTomcatValve$1.invoke(CarbonTomcatValve.java:47)
at org.wso2.carbon.webapp.mgt.TenantLazyLoaderValve.invoke(TenantLazyLoaderValve.java:57)
at org.wso2.carbon.tomcat.ext.valves.TomcatValveContainer.invokeValves(TomcatValveContainer.java:47)
at org.wso2.carbon.tomcat.ext.valves.CompositeValve.invoke(CompositeValve.java:62)
at org.wso2.carbon.tomcat.ext.valves.CarbonStuckThreadDetectionValve.invoke(CarbonStuckThreadDetectionValve.java:159)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
at org.wso2.carbon.tomcat.ext.valves.CarbonContextCreatorValve.invoke(CarbonContextCreatorValve.java:57)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:421)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1074)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:611)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1739)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run(NioEndpoint.java:1698)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.wso2.carbon.apimgt.impl.workflow.WorkflowException: Error occured when updating the status of the Application creation process
at org.wso2.carbon.apimgt.impl.workflow.ApplicationRegistrationSimpleWorkflowExecutor.complete(ApplicationRegistrationSimpleWorkflowExecutor.java:82)
at org.wso2.carbon.apimgt.impl.workflow.ApplicationRegistrationSimpleWorkflowExecutor.execute(ApplicationRegistrationSimpleWorkflowExecutor.java:54)
at org.wso2.carbon.apimgt.impl.APIConsumerImpl.requestApprovalForApplicationRegistration(APIConsumerImpl.java:2626)
... 66 more
Caused by: org.wso2.carbon.apimgt.api.APIManagementException: Error occurred while executing SubscriberKeyMgtClient.
at org.wso2.carbon.apimgt.impl.utils.APIUtil.handleException(APIUtil.java:1195)
at org.wso2.carbon.apimgt.impl.workflow.AbstractApplicationRegistrationWorkflowExecutor.dogenerateKeysForApplication(AbstractApplicationRegistrationWorkflowExecutor.java:186)
at org.wso2.carbon.apimgt.impl.workflow.AbstractApplicationRegistrationWorkflowExecutor.generateKeysForApplication(AbstractApplicationRegistrationWorkflowExecutor.java:121)
at org.wso2.carbon.apimgt.impl.workflow.ApplicationRegistrationSimpleWorkflowExecutor.complete(ApplicationRegistrationSimpleWorkflowExecutor.java:78)
... 68 more
Caused by: org.wso2.carbon.apimgt.api.APIManagementException: Can not create OAuth application : …
at org.wso2.carbon.apimgt.impl.AMDefaultKeyManagerImpl.handleException(AMDefaultKeyManagerImpl.java:592)
at org.wso2.carbon.apimgt.impl.AMDefaultKeyManagerImpl.createApplication(AMDefaultKeyManagerImpl.java:106)
at org.wso2.carbon.apimgt.impl.workflow.AbstractApplicationRegistrationWorkflowExecutor.dogenerateKeysForApplication(AbstractApplicationRegistrationWorkflowExecutor.java:150)
... 70 more
**Caused by: org.wso2.carbon.apimgt.keymgt.stub.subscriber.APIKeyMgtSubscriberServiceAPIManagementException: APIKeyMgtSubscriberServiceAPIManagementException**
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at java.lang.Class.newInstance(Class.java:383)
at org.wso2.carbon.apimgt.keymgt.stub.subscriber.APIKeyMgtSubscriberServiceStub.createOAuthApplicationByApplicationInfo(APIKeyMgtSubscriberServiceStub.java:2124)
at org.wso2.carbon.apimgt.keymgt.client.SubscriberKeyMgtClient.createOAuthApplicationbyApplicationInfo(SubscriberKeyMgtClient.java:62)
at org.wso2.carbon.apimgt.impl.AMDefaultKeyManagerImpl.createApplication(AMDefaultKeyManagerImpl.java:104)
... 71 more
The database is MSSQL, if that makes a difference.
From the log, the issue is with carbon component. WSO2 APIM 1.10.0 is based on WSO2 Carbon 4.4.3.
In my case, I found in the log that thrift server/key manager cannot be started due to port conflicts - I have two wso instances running in the same box. The gateway port offset did NOT apply to thrift server/client ports. After I changed it to a different port, all works like a charm.