I get a confusing ArrayIndexOutOfBoundsException at every SSL-secured Webservice call and I have no idea what's the reason. The exception comes without any relevant change on the code over night and I am baffled.
Can anyone give me a hint how I can fix this shitty bug?
Environment:
Tomcat7
IBM JDK 6
Stacktrace:
com.sun.xml.ws.client.ClientTransportException: HTTP transport error: javax.net.ssl.SSLException: java.lang.ArrayIndexOutOfBoundsException
at com.sun.xml.ws.transport.http.client.HttpClientTransport.getOutput(HttpClientTransport.java:132)
at com.sun.xml.ws.transport.http.client.HttpTransportPipe.process(HttpTransportPipe.java:256)
at com.sun.xml.ws.transport.http.client.HttpTransportPipe.processRequest(HttpTransportPipe.java:184)
at com.sun.xml.ws.transport.DeferredTransportPipe.processRequest(DeferredTransportPipe.java:137)
at com.sun.xml.ws.api.pipe.Fiber.__doRun(Fiber.java:641)
at com.sun.xml.ws.api.pipe.Fiber._doRun(Fiber.java:600)
at com.sun.xml.ws.api.pipe.Fiber.doRun(Fiber.java:585)
at com.sun.xml.ws.api.pipe.Fiber.runSync(Fiber.java:482)
at com.sun.xml.ws.client.Stub.process(Stub.java:323)
at com.sun.xml.ws.client.sei.SEIStub.doProcess(SEIStub.java:161)
at com.sun.xml.ws.client.sei.SyncMethodHandler.invoke(SyncMethodHandler.java:113)
at com.sun.xml.ws.client.sei.SyncMethodHandler.invoke(SyncMethodHandler.java:93)
at com.sun.xml.ws.client.sei.SEIStub.invoke(SEIStub.java:144)
at $Proxy294.liefereKonfiguration(Unknown Source)
at at.gv.bmf.efsz.vkps.client.lieferekonfiguration.impl.LiefereKonfigurationServiceClientImpl.updateKonfReader(LiefereKonfigurationServiceClientImpl.java:279)
at at.gv.bmf.efsz.vkps.client.lieferekonfiguration.impl.LiefereKonfigurationServiceClientImpl.initiateUpdate(LiefereKonfigurationServiceClientImpl.java:205)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
at java.lang.reflect.Method.invoke(Method.java:611)
at org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:318)
at org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:110)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)
at $Proxy293.initiateUpdate(Unknown Source)
at at.gv.bmf.efsz.vkps.client.lieferekonfiguration.impl.VKPSClientInitializer.initVKPSClient(VKPSClientInitializer.java:23)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
at java.lang.reflect.Method.invoke(Method.java:611)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeCustomInitMethod(AbstractAutowireCapableBeanFactory.java:1581)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1522)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1452)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:519)
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:456)
at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:294)
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:225)
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:291)
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:193)
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:585)
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:913)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:464)
at org.springframework.web.context.ContextLoader.configureAndRefreshWebApplicationContext(ContextLoader.java:385)
at org.springframework.web.context.ContextLoader.initWebApplicationContext(ContextLoader.java:284)
at org.springframework.web.context.ContextLoaderListener.contextInitialized(ContextLoaderListener.java:111)
at org.apache.catalina.core.StandardContext.listenerStart(StandardContext.java:4723)
at org.apache.catalina.core.StandardContext$1.call(StandardContext.java:5226)
at org.apache.catalina.core.StandardContext$1.call(StandardContext.java:5221)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:314)
at java.util.concurrent.FutureTask.run(FutureTask.java:149)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:897)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:919)
at java.lang.Thread.run(Thread.java:736)
Caused by: javax.net.ssl.SSLException: java.lang.ArrayIndexOutOfBoundsException
at com.ibm.jsse2.n.a(n.java:27)
at com.ibm.jsse2.tc.a(tc.java:328)
at com.ibm.jsse2.tc.a(tc.java:386)
at com.ibm.jsse2.tc.a(tc.java:278)
at com.ibm.jsse2.tc.a(tc.java:298)
at com.ibm.jsse2.tc.startHandshake(tc.java:337)
at com.ibm.net.ssl.www2.protocol.https.c.afterConnect(c.java:23)
at com.ibm.net.ssl.www2.protocol.https.d.connect(d.java:57)
at sun.net.www.protocol.http.HttpURLConnection.getOutputStream(HttpURLConnection.java:1024)
at com.ibm.net.ssl.www2.protocol.https.b.getOutputStream(b.java:56)
at com.sun.xml.ws.transport.http.client.HttpClientTransport.getOutput(HttpClientTransport.java:120)
... 54 more
Caused by: java.lang.ArrayIndexOutOfBoundsException
at com.ibm.crypto.provider.TlsPrfGenerator.a(Unknown Source)
at com.ibm.crypto.provider.TlsPrfGenerator.a(Unknown Source)
at com.ibm.crypto.provider.TlsPrfGenerator.a(Unknown Source)
at com.ibm.crypto.provider.TlsMasterSecretGenerator.engineGenerateKey(Unknown Source)
at javax.crypto.KeyGenerator.generateKey(Unknown Source)
at com.ibm.jsse2.gb.b(gb.java:268)
at com.ibm.jsse2.gb.a(gb.java:131)
at com.ibm.jsse2.hb.a(hb.java:216)
at com.ibm.jsse2.hb.a(hb.java:71)
at com.ibm.jsse2.gb.n(gb.java:142)
at com.ibm.jsse2.gb.a(gb.java:95)
at com.ibm.jsse2.tc.a(tc.java:244)
at com.ibm.jsse2.tc.g(tc.java:206)
at com.ibm.jsse2.tc.a(tc.java:165)
... 60 more
It seems you are using IBMJSSE2 Provider for SSL Communication.
There is a very similar problem which is already fixed:
http://www-01.ibm.com/support/docview.wss?crawler=1&uid=swg1IV73472
APAR Name: IV73472: LARGE PRE-MASTER SECRET GENERATED FROM 2048 BIT DH KEY NOT DIGESTED IN TLSV1 AND TLSV1.1
Problem summary: When TLSv1 or TLSv1.1 was used, the large pre-master secret
generated from 2048 bit DH key was not properly hashed to derive
the master secret.
Local fix: Disable cipher suites which uses DH/DHE key exchange.
Problem conclusion: A fix is made to IBMJSSE2 and IBMJCE provider to handle large pre-master secret in TLSv1 and TLSv1.1
The associated Hursley RTC Problem Report is 93670 and 94644
The associated Austin CMVC defect is 116692 and 116765
JVMs affected: Java 6.0, Java 626, Java 7.0, Java 727 and Java 8
The fix was delivered for Java 6.0 SR16FP7, Java 626 SR8FP7,
Java 7.0 SR9FP10, Java 727 SR3FP10 and Java 8 SR1FP10
The affected jar are "ibmjsseprovider2.jar" and
"ibmjceprovider.jar".
The build level of ibmjceprovider.jar for the affected releases
is "20150604".
The build level of ibmjsseprovider2.jar for the affected
releases is "20150604"
Try to upgrade ibmjsseprovider2.jar library.
Related
Good day everyone.
We have wso2 implemented on three Docker containers. Everything worked fine until after adding the next flow, the entrance to the Micro Integration Dashboard stopped working. Moreover, the system itself continues to work, only you can not enter the panel.
Error image
login failed internal server error
No changes were made to the users or to the container itself.
I've searched and read everything I can, but I can't even grasp where the problem is.
Please push me in the right direction.
Here is the error from the logs:
[2023-01-12 07:40:08,602] ERROR {LoginDelegate} - Error logging into dashboard server java.lang.IndexOutOfBoundsException: Index 0 out of bounds for length 0
at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64)
at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70)
at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248)
at java.base/java.util.Objects.checkIndex(Objects.java:372)
at java.base/java.util.ArrayList.get(ArrayList.java:459)
at org.wso2.ei.dashboard.core.rest.delegates.auth.LoginDelegate.getTokenFromMI(LoginDelegate.java:80)
at org.wso2.ei.dashboard.core.rest.delegates.auth.LoginDelegate.authenticateUser(LoginDelegate.java:51)
at org.wso2.ei.dashboard.core.rest.api.LoginApi.receiveLogin(LoginApi.java:56)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:76)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:148)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:191)
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:200)
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:103)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:493)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:415)
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:104)
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:277)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:272)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:268)
at org.glassfish.jersey.internal.Errors.process(Errors.java:316)
at org.glassfish.jersey.internal.Errors.process(Errors.java:298)
at org.glassfish.jersey.internal.Errors.process(Errors.java:268)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:289)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:256)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:703)
at org.glassfish.jersey.servlet.WebComponent.serviceImpl(WebComponent.java:416)
at org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:370)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:389)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:342)
at org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:229)
at org.eclipse.jetty.servlet.ServletHolder$NotAsync.service(ServletHolder.java:1450)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:550)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:516)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:555)
at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:410)
at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:164)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
at java.base/java.lang.Thread.run(Thread.java:829)
i just set up an AWS EMR Cluster (EMR Version 5.18 with Spark 2.3.2). I ssh into the master maschine and run spark-shell or pyspark and get the following error:
$ spark-shell
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /stderr (Permission denied)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580)
at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
at org.apache.spark.internal.Logging$class.initializeLogging(Logging.scala:120)
at org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:108)
at org.apache.spark.deploy.SparkSubmit$.initializeLogIfNecessary(SparkSubmit.scala:71)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:128)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
log4j:ERROR Either File or DatePattern options are not set for appender [DRFA-stderr].
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /stdout (Permission denied)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:580)
at org.apache.log4j.helpers.OptionConverter.selectAndConfigure(OptionConverter.java:526)
at org.apache.log4j.LogManager.<clinit>(LogManager.java:127)
at org.apache.spark.internal.Logging$class.initializeLogging(Logging.scala:120)
at org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:108)
at org.apache.spark.deploy.SparkSubmit$.initializeLogIfNecessary(SparkSubmit.scala:71)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:128)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
log4j:ERROR Either File or DatePattern options are not set for appender [DRFA-stdout].
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
18/11/04 12:24:32 ERROR SparkContext: Error initializing SparkContext.
java.lang.IllegalArgumentException: Required executor memory (4608+460 MB) is above the max threshold (3072 MB) of this cluster! Please check the values of 'yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb'.
at org.apache.spark.deploy.yarn.Client.verifyClusterResources(Client.scala:318)
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:166)
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57)
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:164)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:500)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2493)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:934)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:925)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:925)
at org.apache.spark.repl.Main$.createSparkSession(Main.scala:103)
at $line3.$read$$iw$$iw.<init>(<console>:15)
at $line3.$read$$iw.<init>(<console>:43)
at $line3.$read.<init>(<console>:45)
at $line3.$read$.<init>(<console>:49)
at $line3.$read$.<clinit>(<console>)
at $line3.$eval$.$print$lzycompute(<console>:7)
at $line3.$eval$.$print(<console>:6)
at $line3.$eval.$print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:79)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1$$anonfun$apply$mcV$sp$1$$anonfun$apply$mcV$sp$2.apply(SparkILoop.scala:79)
at scala.collection.immutable.List.foreach(List.scala:381)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(SparkILoop.scala:79)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:79)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1$$anonfun$apply$mcV$sp$1.apply(SparkILoop.scala:79)
at scala.tools.nsc.interpreter.ILoop.savingReplayStack(ILoop.scala:91)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply$mcV$sp(SparkILoop.scala:78)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply(SparkILoop.scala:78)
at org.apache.spark.repl.SparkILoop$$anonfun$initializeSpark$1.apply(SparkILoop.scala:78)
at scala.tools.nsc.interpreter.IMain.beQuietDuring(IMain.scala:214)
at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:77)
at org.apache.spark.repl.SparkILoop.loadFiles(SparkILoop.scala:110)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:920)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
at org.apache.spark.repl.Main$.doMain(Main.scala:76)
at org.apache.spark.repl.Main$.main(Main.scala:56)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
18/11/04 12:24:33 WARN YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!
18/11/04 12:24:33 WARN MetricsSystem: Stopping a MetricsSystem that is not running
java.lang.IllegalArgumentException: Required executor memory (4608+460 MB) is above the max threshold (3072 MB) of this cluster! Please check the values of 'yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb'.
at org.apache.spark.deploy.yarn.Client.verifyClusterResources(Client.scala:318)
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:166)
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57)
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:164)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:500)
at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2493)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:934)
at org.apache.spark.sql.SparkSession$Builder$$anonfun$7.apply(SparkSession.scala:925)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:925)
at org.apache.spark.repl.Main$.createSparkSession(Main.scala:103)
... 55 elided
<console>:14: error: not found: value spark
import spark.implicits._
^
<console>:14: error: not found: value spark
import spark.sql
^
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/___/ .__/\_,_/_/ /_/\_\ version 2.3.2
/_/
Using Scala version 2.11.8 (OpenJDK 64-Bit Server VM, Java 1.8.0_181)
Type in expressions to have them evaluated.
Type :help for more information.
I am new to Spark and to EMR and dont know what to do. Is there some configuration step i missed or anything else i have to provide to make it work?
Thank you for your help!
If you look into /etc/spark/conf/log4j.properties file, you'll find that there's new setup allowing to roll Spark Streaming logs hourly (probably as it's suggested here).
The problem occurs because ${spark.yarn.app.container.log.dir} system property is not set in Spark driver process. The property is set eventually to Yarn's container log directory, but this happens later (look here and here).
In order to fix this error in Spark driver, add the following to your spark-submit or spark-shell command:
--driver-java-options='-Dspark.yarn.app.container.log.dir=/mnt/var/log/hadoop'
Please note that /mnt/var/log/hadoop/stderr and /mnt/var/log/hadoop/stdout files will be reused by all the (Spark Streaming) processes started on the same node.
We have also run into this issue and hope some AWS or Spark engineers are reading this. I've narrowed this down to the /etc/spark/conf/log4j.properties file and how the loggers are configured using the ${spark.yarn.app.container.log.dir} system property. That value is evaluating to null and so the logging directory now evaluates to /stdout and /stderr instead of the desired /mnt/var/log/hadoop-yarn/containers/<app_id>/<container_id>/(stdout|stderr) which is how it worked in EMR < 5.18.0.
Workaround #1 (not ideal): If you set that property to a static path which the hadoop user has access to like /var/log/hadoop-yarn/stderr things work fine. This probably breaks things like the history server and an unknown number of other things, but spark-shell and pyspark can start without errors.
UPDATE Workaround #2 (revert): Not sure why I didn't do this earlier but comparing this to a 5.13 cluster, the entirety of the DRFA-stderr and DRFA-stdout appenders were non-existent. If you comment those sections out, delete them, or simply copy the log4j.properties file from the template this problem also goes away (again, unknown impact to the rest of the services). I'm not sure where that section originated from, the master repo configs do not have those appenders so it appears to be proprietary to AWS distros.
in order to fix this issue you can add configuration in json format on emr provisioning. We use code like this
{
"Classification": "yarn-site",
"Configurations": [
],
"Properties": {
"spark.yarn.app.container.log.dir": "/var/log/hadoop-yarn"
}
}
I am trying to run one my process through callgrind. One of the child process (that I need to trace) calls into libhdfs and when running through callgrind this throws an exception:
java.util.ServiceConfigurationError: org.apache.hadoop.fs.FileSystem: Provider org.apache.hadoop.fs.viewfs.ViewFileSystem could not be instantiated
at java.util.ServiceLoader.fail(ServiceLoader.java:224)
at java.util.ServiceLoader.access$100(ServiceLoader.java:181)
at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:377)
at java.util.ServiceLoader$1.next(ServiceLoader.java:445)
at org.apache.hadoop.fs.FileSystem.loadFileSystems(FileSystem.java:2602)
Caused by: java.lang.ExceptionInInitializerError
at org.apache.hadoop.fs.viewfs.ViewFileSystem.<init>(ViewFileSystem.java:134)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
at java.lang.Class.newInstance(Class.java:374)
at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:373)
... 2 more
Caused by: java.lang.IllegalArgumentException: java.lang.ClassFormatError: Illegal UTF8 string in constant pool in class file com/sun/proxy/$Proxy7
at java.lang.reflect.Proxy$ProxyClassFactory.apply(Proxy.java:685)
at java.lang.reflect.Proxy$ProxyClassFactory.apply(Proxy.java:592)
at java.lang.reflect.WeakCache$Factory.get(WeakCache.java:244)
at java.lang.reflect.WeakCache.get(WeakCache.java:141)
at java.lang.reflect.Proxy.getProxyClass0(Proxy.java:455)
at java.lang.reflect.Proxy.newProxyInstance(Proxy.java:738)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.getProxyForCallback(MetricsSystemImpl.java:315)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.register(MetricsSystemImpl.java:311)
at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.register(MetricsSystemImpl.java:237)
at org.apache.hadoop.metrics2.MetricsSystem.register(MetricsSystem.java:60)
at org.apache.hadoop.security.UserGroupInformation$UgiMetrics.create(UserGroupInformation.java:120)
at org.apache.hadoop.security.UserGroupInformation.<clinit>(UserGroupInformation.java:236)
... 9 more
When running normally I do not get this exception. Any idea what's going on? I am new to valgrind/callgrind.
It appears that this is an issue with JVM which generates a lot of SIGSEGVs during it's normal course of operation which valgrind doesn't like. I overcame that with the following parameter to callgrind --vex-iropt-register-updates=allregs-at-mem-access.
I'm running Wowza 3.6.2 on Windows 8.1 (64 bit) and have enabled the Transcoder add-on. I'm using the transcoder to take JPEG-snapshots from the live stream. I've built a custom HTTPProvider, similar to what is described here. This works fine and I can get JPG-snapshots from the stream through my HTTPProvider.
The problem is that since I enabled the transcoder I get irritating error messages in my server log on onPublish and onUnPublish of any stream.
As a transcoder template I used the default transrate.xml that comes with the Wowza installation, without any modifications made to it.
When i publish to a stream asdf I get errors similar to this:
ERROR server comment - TranscoderSessionDestination.init[livereceiver/_definst_/asdf]: [asdf_160p]:java.lang.ArrayIndexOutOfBoundsException: 1
java.lang.ArrayIndexOutOfBoundsException: 1
at com.foo.wms.module.IncomingStreamEventHandler.getQueryStringMap(IncomingStreamEventHandler.java:191)
at com.foo.wms.module.IncomingStreamEventHandler.onPublish(IncomingStreamEventHandler.java:83)
at com.wowza.wms.stream.MediaStream.notifyActionPublish(Unknown Source)
at com.wowza.wms.stream.publish.Publisher.publish(Unknown Source)
at com.wowza.wms.stream.publish.Publisher.publish(Unknown Source)
at com.wowza.wms.transcoder.model.TranscoderSessionDestination.init(Unknown Source)
at com.wowza.wms.transcoder.model.TranscoderSession.a(Unknown Source)
at com.wowza.wms.transcoder.model.TranscoderSession.handleOnMetadata(Unknown Source)
at com.wowza.wms.transcoder.model.LiveStreamTranscoder.handleOnMetadata(Unknown Source)
at com.wowza.wms.stream.live.LiveStreamTranscoderRunner.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
And when I unpublish the stream I get this:
ERROR server comment - TranscoderSessionDestination.shutdown: [asdf_160p]:java.lang.NullPointerException
java.lang.NullPointerException
at com.foo.wms.module.IncomingStreamEventHandler.onUnPublish(IncomingStreamEventHandler.java:166)
at com.wowza.wms.stream.MediaStream.notifyActionUnPublish(Unknown Source)
at com.wowza.wms.stream.publish.Publisher.publish(Unknown Source)
at com.wowza.wms.stream.publish.Publisher.unpublish(Unknown Source)
at com.wowza.wms.transcoder.model.TranscoderSessionDestination.shutdown(Unknown Source)
at com.wowza.wms.transcoder.model.TranscoderSession.c(Unknown Source)
at com.wowza.wms.transcoder.model.TranscoderSession.shutdown(Unknown Source)
at com.wowza.wms.transcoder.model.LiveStreamTranscoder.shutdown(Unknown Source)
at com.wowza.wms.stream.live.LiveStreamTranscoderRunner.run(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)
I get three of each exception when I publish/unpublish (one for each Encode block that is enabled in the transrate.xml file).
Does anyone have an idea on what might be causing this?
Here is a better way to determine if the stream is a transcoded stream and not a source stream
if(stream.isTranscodeResult()) return;
After posting my question I had another look at the stacktrace and realized what the problem was - I had been looking in the wrong direction the whole time. Since the problem appeared when I enabled the transcoder, I concluded that's where the problem must be. What I didn't realize was that onPublish and onUnPublish fires multiple times when you use the transcoder - one time for the incoming stream, and one time for every transcoded stream.
Within the onPublish and onUnPublish methods of my module I do stuff, like read querystring-paramters, which are not present on the transcoded streams. That is why the exceptions are thrown when the onPublish and onUnPublish methods are called for the transcoded streams.
To remedy this, I added two lines at the beginning of onPublish and onUnPublish methods.
if (streamName.contains("_"))
return;
A somewhat ugly solution, but I am in control of all stream names and do not allow underscore in them anyway, so in my case this works fine.
Update:
#flux has provided a much nicer solution for how to check if the stream is the result of a transcode operation. See his answer for more info.
I am trying to get the Handviewer java sample project running. When running the application crashes after (1-2 sec) and firing the following exceptions!
Please note that I have managed to run the UserViewers.java sample without problems. and I am running on OSX 10.8.3, NiTE-MacOSX-x64-2.2, OpenNI-MacOSX-x64-2.2, and Asus XtionPro Live.
Exception in thread "Thread-17" java.util.NoSuchElementException: Unknown pixel format: 0
at org.openni.PixelFormat.fromNative(PixelFormat.java:30)
at org.openni.VideoMode.<init>(VideoMode.java:38)
at com.primesense.nite.NativeMethods.niteReadHandTrackerFrame(Native Method)
at com.primesense.nite.HandTracker.readFrame(HandTracker.java:139)
at com.primesense.nite.Samples.HandViewer.HandViewer.onNewFrame(HandViewer.java:69)
at com.primesense.nite.HandTracker.onFrameReady(HandTracker.java:360)
Exception in thread "AWT-EventQueue-0" java.lang.NullPointerException
at com.primesense.nite.Samples.HandViewer.HandViewer.paint(HandViewer.java:55)
at javax.swing.JComponent.paintChildren(JComponent.java:884)
at javax.swing.JComponent.paint(JComponent.java:1046)
at javax.swing.JComponent._paintImmediately(JComponent.java:5106)
at javax.swing.JComponent.paintImmediately(JComponent.java:4890)
at javax.swing.RepaintManager$3.run(RepaintManager.java:814)
at javax.swing.RepaintManager$3.run(RepaintManager.java:802)
at java.security.AccessController.doPrivileged(Native Method)
at java.security.AccessControlContext$1.doIntersectionPrivilege(AccessControlContext.java:87)
at javax.swing.RepaintManager.paintDirtyRegions(RepaintManager.java:802)
at javax.swing.RepaintManager.paintDirtyRegions(RepaintManager.java:745)
at javax.swing.RepaintManager.prePaintDirtyRegions(RepaintManager.java:725)
at javax.swing.RepaintManager.access$1000(RepaintManager.java:46)
at javax.swing.RepaintManager$ProcessingRunnable.run(RepaintManager.java:1684)
at java.awt.event.InvocationEvent.dispatch(InvocationEvent.java:209)
at java.awt.EventQueue.dispatchEventImpl(EventQueue.java:708)
at java.awt.EventQueue.access$400(EventQueue.java:82)
at java.awt.EventQueue$2.run(EventQueue.java:669)
at java.awt.EventQueue$2.run(EventQueue.java:667)
at java.security.AccessController.doPrivileged(Native Method)
at java.security.AccessControlContext$1.doIntersectionPrivilege(AccessControlContext.java:87)
at java.awt.EventQueue.dispatchEvent(EventQueue.java:678)
at java.awt.EventDispatchThread.pumpOneEventForFilters(EventDispatchThread.java:296)
at java.awt.EventDispatchThread.pumpEventsForFilter(EventDispatchThread.java:211)
at java.awt.EventDispatchThread.pumpEventsForHierarchy(EventDispatchThread.java:201)
at java.awt.EventDispatchThread.pumpEvents(EventDispatchThread.java:196)
at java.awt.EventDispatchThread.pumpEvents(EventDispatchThread.java:188)
at java.awt.EventDispatchThread.run(EventDispatchThread.java:122)
This probably won't help you much because it's already been a while... but I have had a similar problem with my Kinect. I could resolve everything by reinstalling everything and switching to OpenNI 1.5.4. Everything worked after that. Good luck.