Got this error running a custom jar on EMR.
Exception in thread "main" com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.model.AmazonS3Exception: Bad Request (Service: Amazon S3; Status Code: 400; Error Code: 400 Bad Request; Request ID: B042BB0B40A75966), S3 Extended Request ID: vr/DUr8HD3xjomauyzqvVdGuW3fHBP8PDUmTIAoVLUxrmsxh9H+OS9+cgo4OmHxaz/b8CSPGmuc=
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.handleErrorResponse(AmazonHttpClient.java:1389)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:902)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:607)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.doExecute(AmazonHttpClient.java:376)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.executeWithTimer(AmazonHttpClient.java:338)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:287)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:3826)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.headBucket(AmazonS3Client.java:1071)
at com.amazon.ws.emr.hadoop.fs.shaded.com.amazonaws.services.s3.AmazonS3Client.doesBucketExist(AmazonS3Client.java:1029)
at com.amazon.ws.emr.hadoop.fs.s3n.Jets3tNativeFileSystemStore.ensureBucketExists(Jets3tNativeFileSystemStore.java:138)
at com.amazon.ws.emr.hadoop.fs.s3n.Jets3tNativeFileSystemStore.initialize(Jets3tNativeFileSystemStore.java:116)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy23.initialize(Unknown Source)
at com.amazon.ws.emr.hadoop.fs.s3n.S3NativeFileSystem.initialize(S3NativeFileSystem.java:461)
at com.amazon.ws.emr.hadoop.fs.EmrFileSystem.initialize(EmrFileSystem.java:110)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2703)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2737)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2719)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:375)
at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(FileInputFormat.java:485)
at SentimentsDriver.run(SentimentsDriver.java:21)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:84)
at SentimentsDriver.main(SentimentsDriver.java:33)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
x
arguments:
s3://sentimentproj/input/tweet.txt
s3://sentimentproj/output
SentimentsDriver:
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
/*This class is responsible for running map reduce job*/
public class SentimentsDriver extends Configured implements Tool{
public int run(String[] args) throws Exception{
if(args.length !=2) {
System.err.println("Usage: MaxTemperatureDriver <input path> <outputpath>");
System.exit(-1);
}
#SuppressWarnings("deprecation")
Job job = new Job();
job.setJarByClass(SentimentsDriver.class);
job.setJobName("SentimentAnalysis");
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
job.setMapperClass(SentimentsMapper.class);
job.setReducerClass(SentimentsReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
System.exit(job.waitForCompletion(true) ? 0:1);
boolean success = job.waitForCompletion(true);
return success ? 0 : 1;
}
public static void main(String[] args) throws Exception {
SentimentsDriver driver = new SentimentsDriver();
int exitCode = ToolRunner.run(driver, args);
System.exit(exitCode);
}
}
The issue may be with the S3 bucket. Make sure to keep the S3 bucket region in same Region of your cluster while creating bucket. If it will be different then it usually throw the Bad Request 400 error.
Related
I have the following two classes -
Service class
#Service
public class ProfileInfoCacheService {
public ProfileInfoResponse getProfileInfo(String uuid) {
ProfileInfoResponse response = getUserInfoFromIdentity(uuid);
return response;
}
public ProfileInfoResponse getUserInfoFromIdentity(String identity) {
LinkedHashMap<String, List<UserInfo>> profileInfo = userService.getUserInfoFromIdentity(identityList);
return profcache.getProfileInfo();
}
And component
#Component
public class UserInfoServiceImpl implements UserInfoService {
#SuppressWarnings("unchecked")
#Override
public LinkedHashMap<String, List<UserInfo>> getUserInfoFromIdentity(List<String> identityList) {
//Calls an API and returns data in LinkedHashMap.
}
Now, here is my test case file. Please bear with me as I am still new at it.
package com.citruspay.prepaid;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
//class file imports
import org.mockito.runners.MockitoJUnitRunner;
#RunWith(MockitoJUnitRunner.class)
public class ProfileInfoCacheServiceTest {
private PrepaidAccount account;
private ProfileInfoCacheService pics;
#Before
public void setUp() throws Exception {
account = new PrepaidAccount("8588844251");
//pics = new ProfileInfoCacheService();
ProfileInfoCacheService pics = mock(ProfileInfoCacheService.class);
}
#Test
public void testConstructor() {
assertEquals("8588888888", account.owner());
}
#Test
public void testNumber() throws Exception {
ProfileInfoResponse response = pics.getProfileInfo("7266704751953077361"); //pics is null here
assertEquals("", response.getEmail() );
assertEquals("8588888888", response.getMobile() );
}
}
The response of getProfileInfo() call is expected to be in this format -
{
"email": "some.email#something.in",
"mobile": "8888888888",
}
Every mobile/email pair has a number (like 7266704751953077361), using which, when the API is called, it returns the same.
However, I am getting a Null Pointer exception, due to pics being null where I call pics.getProfileInfo("7266704751953077361").
Please help me correct my mistake here.
Update
Upon changing mock() to new class call, I get the ProfileInfoCacheService object but then on moving the debug to the last line of the setup() method, getting the following -
java.lang.ExceptionInInitializerError
at org.slf4j.impl.StaticLoggerBinder.<init>(StaticLoggerBinder.java:72)
at org.slf4j.impl.StaticLoggerBinder.<clinit>(StaticLoggerBinder.java:45)
at org.slf4j.LoggerFactory.bind(LoggerFactory.java:150)
at org.slf4j.LoggerFactory.performInitialization(LoggerFactory.java:124)
at org.slf4j.LoggerFactory.getILoggerFactory(LoggerFactory.java:412)
at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:357)
at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:383)
at com.citruspay.prepaid.common.user.beans.ProfileInfoCacheService.<init>(ProfileInfoCacheService.java:42)
at com.citruspay.prepaid.ProfileInfoCacheServiceTest.setUp(ProfileInfoCacheServiceTest.java:26)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:71)
at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:288)
at org.junit.runners.ParentRunner.access$000(ParentRunner.java:58)
at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:268)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.mockito.internal.runners.JUnit45AndHigherRunnerImpl.run(JUnit45AndHigherRunnerImpl.java:37)
at org.mockito.runners.MockitoJUnitRunner.run(MockitoJUnitRunner.java:62)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
at com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:47)
at com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:242)
at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
Caused by: java.lang.IllegalStateException: Detected both log4j-over-slf4j.jar AND bound slf4j-log4j12.jar on the class path, preempting StackOverflowError. See also http://www.slf4j.org/codes.html#log4jDelegationLoop for more details.
at org.slf4j.impl.Log4jLoggerFactory.<clinit>(Log4jLoggerFactory.java:54)
... 33 more
Update 2
Adding more details of ProfileInfoCacheService class -
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.PostConstruct;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import redis.clients.jedis.exceptions.JedisException;
import com.aerospike.client.AerospikeException;
import com.aerospike.client.Key;
import com.citruspay.prepaid.common.cache.dao.ProfileInfoCacheDao;
import com.citruspay.prepaid.common.cache.entity.ProfileInfoCache;
import com.citruspay.prepaid.common.cache.factory.DaoCacheFactory;
import com.citruspay.prepaid.common.user.process.UserInfoServiceImpl;
import com.spikeify.ResultSet;
#Service
public class ProfileInfoCacheService {
#Autowired
UserInfoServiceImpl userService;
#Autowired
DaoCacheFactory<ProfileInfoCacheDao> daoCacheFactory;
#PostConstruct
public void init() {
ProfileInfoCacheDao profileInfoCacheDao = daoCacheFactory.getDaoForRead(ProfileInfoCacheDao.class);
profileInfoCacheDao.init();
}
Logger logger = LoggerFactory.getLogger(ProfileInfoCacheService.class);
I am using HttpClient of Java 11 to post the request to an HTTP2 server. The HttpClient Object is created as a Singleton Spring bean as shown below.
#Bean
public HttpClient getClient() {
return HttpClient.newBuilder().version(Version.HTTP_2).executor(Executors.newFixedThreadPool(20)).followRedirects(Redirect.NORMAL)
.connectTimeout(Duration.ofSeconds(20)).build();
}
I am using the sendAsync method to send the requests asynchronously.
When I try to hit the server continuously, I am receiving the error after certain time "java.io.IOException: too many concurrent streams". I used Fixed threadpool in the Client building to try to overcome this error, but it is still giving the same error.
The Exception stack is..
java.util.concurrent.CompletionException: java.io.IOException: too many concurrent streams
at java.base/java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1108) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2235) ~[?:?]
at java.net.http/jdk.internal.net.http.MultiExchange.responseAsyncImpl(MultiExchange.java:345) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.MultiExchange.lambda$responseAsync0$2(MultiExchange.java:250) ~[java.net.http:?]
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]
at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1705) ~[?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
at java.base/java.lang.Thread.run(Thread.java:834) [?:?]
Caused by: java.io.IOException: too many concurrent streams
at java.net.http/jdk.internal.net.http.Http2Connection.reserveStream(Http2Connection.java:440) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.Http2ClientImpl.getConnectionFor(Http2ClientImpl.java:103) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.ExchangeImpl.get(ExchangeImpl.java:88) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.Exchange.establishExchange(Exchange.java:293) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.Exchange.responseAsyncImpl0(Exchange.java:425) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.Exchange.responseAsyncImpl(Exchange.java:330) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.Exchange.responseAsync(Exchange.java:322) ~[java.net.http:?]
at java.net.http/jdk.internal.net.http.MultiExchange.responseAsyncImpl(MultiExchange.java:304) ~[java.net.http:?]
Can someone help me in fixing this issue?
The server is Tomcat9 and its max concurrent streams are the default.
When I try to hit the server continuously
The server has a setting for max_concurrent_streams that is communicated to the client during the initial establishment of a HTTP/2 connection.
If you blindly "hit the server continuously" using sendAsync you are not waiting for previous requests to finish and eventually you exceed the max_concurrent_streams value and receive the error above.
The solution is to send concurrently a number of requests that is less than max_concurrent_streams; after that, you only send a new request when a previous one completes.
This can easily implemented on the client using a Semaphore or something similar.
Unfortunately, the approach with Semaphore, suggested by #sbordet, didn't work for me. I tried this:
var semaphore = semaphores.computeIfAbsent(getRequestKey(request), k -> new Semaphore(MAX_CONCURRENT_REQUESTS_NUMBER));
CompletableFuture.runAsync(semaphore::acquireUninterruptibly, WAITING_POOL)
.thenComposeAsync(ignored -> httpClient.sendAsync(request, responseBodyHandler), ASYNC_POOL)
.whenComplete((response, e) -> semaphore.release());
There's no guarantee that a connection stream is released by the time the execution is passed to the next CompletableFuture, where the semaphore is released. For me the approach worked in case of normal execution, however if there're any exceptions, it seems that the connection stream may be closed after semaphore.release() is invoked.
Finally, I ended up by using OkHttp. It handles the problem (it just waits until some streams are freed up if the number of concurrent streams reaches max_concurrent_streams). It also handles the GOAWAY frame. In case of Java HttpClient I had to implement retry logic to handle this as it just throws IOException if the server sends GOAWAY frame.
I think #sbordet's answer is incorrect and this error does not occur because your requests per second is exceeding MAX_CONCURRENT_STREAMS, but because the number of open HTTP streams (per HTTP 2 connection?) exceeds that number.
For example, I have a server at work that has a MAX_CONCURRENT_STREAMS setting of 128:
$ curl -iv -H "Content-Type: application/json" https://example.local
...
* Connection state changed (MAX_CONCURRENT_STREAMS == 128)!
But I seem to be able to hit it with up to ~1000 requests per second without getting any errors back:
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.util.ArrayList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
public class TooManyConcurrentStreams1 {
private static final int CONCURRENCY = 1000;
public static void main(String[] args) {
final var counter = new AtomicInteger();
final var singletonHttpClient = newHttpClient();
final var singletonRequest = newRequest();
final var responses = new ArrayList<CompletableFuture<HttpResponse<Void>>>(CONCURRENCY);
for (int i = 0; i < CONCURRENCY; i++) {
responses.add(singletonHttpClient.sendAsync(singletonRequest, BodyHandlers.discarding()));
}
for (CompletableFuture<HttpResponse<Void>> response : responses) {
response.thenAccept(x -> {});
response.join();
System.out.println(counter.incrementAndGet());
}
singletonHttpClient.executor().ifPresent(executor -> {
if (executor instanceof ExecutorService executorService) {
executorService.shutdown();
}
});
}
public static HttpRequest newRequest() {
return HttpRequest.newBuilder()
.uri(Constants.TEST_URI)
.header("Content-Type", Constants.CONTENT_TYPE)
.header("Accept", Constants.CONTENT_TYPE)
.POST(HttpRequest.BodyPublishers.ofString(Constants.BODY))
.build();
}
public static HttpClient newHttpClient() {
return HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_2)
.executor(Executors.newFixedThreadPool(CONCURRENCY))
.build();
}
}
When I increase CONCURRENCY to an absurd number like 2000,
I get this error, and not java.io.IOException: too many concurrent streams:
Exception in thread "main" java.util.concurrent.CompletionException: java.net.SocketException: Connection reset
at java.base/java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:368)
at java.base/java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:377)
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1152)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162)
at java.net.http/jdk.internal.net.http.Stream.completeResponseExceptionally(Stream.java:1153)
at java.net.http/jdk.internal.net.http.Stream.cancelImpl(Stream.java:1238)
at java.net.http/jdk.internal.net.http.Stream.connectionClosing(Stream.java:1212)
at java.net.http/jdk.internal.net.http.Http2Connection.shutdown(Http2Connection.java:710)
at java.net.http/jdk.internal.net.http.Http2Connection$Http2TubeSubscriber.processQueue(Http2Connection.java:1323)
at java.net.http/jdk.internal.net.http.common.SequentialScheduler$LockingRestartableTask.run(SequentialScheduler.java:205)
at java.net.http/jdk.internal.net.http.common.SequentialScheduler$CompleteRestartableTask.run(SequentialScheduler.java:149)
at java.net.http/jdk.internal.net.http.common.SequentialScheduler$SchedulableTask.run(SequentialScheduler.java:230)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
However, I can reproduce your error with this code (I hit this error first, and then found your question here!)
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.util.ArrayList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class TooManyConcurrentStreams2 {
public static void main(String[] args) {
final var singletonHttpClient = newHttpClient();
final var singletonRequest = newRequest();
final var counter = new AtomicInteger();
final var scheduler = Executors.newScheduledThreadPool(2);
scheduler.schedule(scheduler::shutdown, 1, TimeUnit.HOURS);
scheduler.scheduleAtFixedRate(() -> {
final var batchSize = counter.incrementAndGet();
final var responses = new ArrayList<CompletableFuture<HttpResponse<Void>>>(batchSize);
try {
for (int i = 0; i < batchSize; i++) {
responses.add(
singletonHttpClient.sendAsync(
singletonRequest,
BodyHandlers.discarding()
)
);
}
for (CompletableFuture<HttpResponse<Void>> response : responses) {
response.thenAccept(x -> {
});
response.join();
}
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("batchSize = " + batchSize);
}, 0, 500, TimeUnit.MILLISECONDS);
}
public static HttpRequest newRequest() {
return HttpRequest.newBuilder()
.uri(Constants.TEST_URI)
.header("Content-Type", Constants.CONTENT_TYPE)
.header("Accept", Constants.CONTENT_TYPE)
.POST(HttpRequest.BodyPublishers.ofString(Constants.BODY))
.build();
}
public static HttpClient newHttpClient() {
return HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_2)
.build();
}
}
This one fails on 128th (!) execution of my once per 500ms runnable:
java.util.concurrent.CompletionException: java.io.IOException: too many concurrent streams
at java.base/java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:368)
at java.base/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1189)
at java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2309)
at java.net.http/jdk.internal.net.http.MultiExchange.responseAsyncImpl(MultiExchange.java:453)
at java.net.http/jdk.internal.net.http.MultiExchange.lambda$responseAsync0$2(MultiExchange.java:341)
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1773)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.io.IOException: too many concurrent streams
So the problem is not number of requests per second, but something else, which seems to be the number of concurrent open streams per http connection/client.
We can verify this by NOT sharing the same http client (and request) for all batch requests:
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.util.ArrayList;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
public class TooManyConcurrentStreams2 {
public static void main(String[] args) {
final var counter = new AtomicInteger();
final var scheduler = Executors.newScheduledThreadPool(2);
scheduler.schedule(scheduler::shutdown, 1, TimeUnit.HOURS);
scheduler.scheduleAtFixedRate(() -> {
final var httpClient = newHttpClient();
final var request = newRequest();
final var batchSize = counter.incrementAndGet();
final var responses = new ArrayList<CompletableFuture<HttpResponse<Void>>>(batchSize);
try {
for (int i = 0; i < batchSize; i++) {
responses.add(
httpClient.sendAsync(
request,
BodyHandlers.discarding()
)
);
}
for (CompletableFuture<HttpResponse<Void>> response : responses) {
response.thenAccept(x -> {
});
response.join();
}
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("batchSize = " + batchSize);
}, 0, 500, TimeUnit.MILLISECONDS);
}
public static HttpRequest newRequest() {
return HttpRequest.newBuilder()
.uri(Constants.TEST_URI)
.header("Content-Type", Constants.CONTENT_TYPE)
.header("Accept", Constants.CONTENT_TYPE)
.POST(HttpRequest.BodyPublishers.ofString(Constants.BODY))
.build();
}
public static HttpClient newHttpClient() {
return HttpClient.newBuilder()
.version(HttpClient.Version.HTTP_2)
.build();
}
}
For me, this one fails at 143rd try with this error message:
java.util.concurrent.CompletionException: java.lang.InternalError: java.net.SocketException: Too many open files
at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:315)
at java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:320)
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1159)
at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1773)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.InternalError: java.net.SocketException: Too many open files
at java.net.http/jdk.internal.net.http.PlainHttpConnection.<init>(PlainHttpConnection.java:293)
at java.net.http/jdk.internal.net.http.AsyncSSLConnection.<init>(AsyncSSLConnection.java:49)
at java.net.http/jdk.internal.net.http.HttpConnection.getSSLConnection(HttpConnection.java:293)
at java.net.http/jdk.internal.net.http.HttpConnection.getConnection(HttpConnection.java:279)
at java.net.http/jdk.internal.net.http.Http2Connection.createAsync(Http2Connection.java:369)
at java.net.http/jdk.internal.net.http.Http2ClientImpl.getConnectionFor(Http2ClientImpl.java:128)
at java.net.http/jdk.internal.net.http.ExchangeImpl.get(ExchangeImpl.java:93)
at java.net.http/jdk.internal.net.http.Exchange.establishExchange(Exchange.java:343)
at java.net.http/jdk.internal.net.http.Exchange.responseAsyncImpl0(Exchange.java:475)
at java.net.http/jdk.internal.net.http.Exchange.responseAsyncImpl(Exchange.java:380)
at java.net.http/jdk.internal.net.http.Exchange.responseAsync(Exchange.java:372)
at java.net.http/jdk.internal.net.http.MultiExchange.responseAsyncImpl(MultiExchange.java:408)
at java.net.http/jdk.internal.net.http.MultiExchange.lambda$responseAsync0$2(MultiExchange.java:341)
at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1150)
... 5 more
This one is most likely due to my laptop's relatively low ulimit of 12544.
I am getting the following exception while trying to download a file from AmazonS3 bucket. This is working fine from Java IDE but when I run the same code from other java based product it is failing. I have loaded all the client needed jar files,
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/aws-java-sdk-core-1.11.250.jar
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/aws-java-sdk-s3-1.11.250.jar
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/httpclient-4.5.2.jar
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/httpcore-4.4.4.jar
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/jackson-annotations-2.6.0.jar
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/jackson-core-2.6.7.jar
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/jackson-databind-2.6.7.1.jar
VENDOR_JAR=/u01/app/sterling/jar/amazons3/1_0/joda-time-2.8.1.jar
Caused by: java.net.SocketException: Connection reset
at java.net.SocketInputStream.read(SocketInputStream.java:168)
at com.sun.net.ssl.internal.ssl.InputRecord.readFully(InputRecord.java:293)
at com.sun.net.ssl.internal.ssl.InputRecord.read(InputRecord.java:331)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:798)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1138)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1165)
at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1149)
at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
at com.amazonaws.http.conn.ssl.SdkTLSSocketFactory.connectSocket(SdkTLSSocketFactory.java:132)
at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:141)
at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:76)
at com.amazonaws.http.conn.$Proxy15.connect(Unknown Source)
at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1236)
at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1056)
... 24 more
Java Code:
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 s3Client = new AmazonS3Client(credentials);
System.out.println("Initialized Client Finally!!");
S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, fileName));
displayTextInputStream(object.getObjectContent());
public void displayTextInputStream(InputStream input)throws IOException {
System.out.println("Inside displayTextInputStream()..........");
// Read one text line at a time and display.
BufferedReader reader = new BufferedReader(new InputStreamReader(input));
while (true) {
String line = reader.readLine();
if (line == null) break;
System.out.println(" " + line);
}
System.out.println();
}
I've problem with initiation of JAXWSProperties in MessageContext of example webservice described on following blog
There is the helper class initiating HeaderList object in getHeaders() method:
import com.sun.xml.internal.ws.api.SOAPVersion;
import com.sun.xml.internal.ws.api.addressing.AddressingVersion;
import com.sun.xml.internal.ws.api.message.HeaderList;
import com.sun.xml.internal.ws.api.message.Headers;
import com.sun.xml.internal.ws.developer.JAXWSProperties;
import com.sun.xml.internal.ws.developer.WSBindingProvider;
import javax.xml.ws.EndpointReference;
import javax.xml.ws.Service;
import javax.xml.ws.WebServiceContext;
public final class CorrelationHelper<S extends Service> {
private WebServiceContext wsc;
private S service;
public CorrelationHelper(S service, WebServiceContext wsc) {
this.service = service;
this.wsc = wsc;
}
private HeaderList getHeaders() {
return (HeaderList)wsc.getMessageContext().get(JAXWSProperties.INBOUND_HEADER_LIST_PROPERTY);
}
public <P> P getCorrelatedPort(Class<P> portType) {
P port = service.getPort(getReplyTo(), portType);
((WSBindingProvider)port).setOutboundHeaders(Headers.create(AddressingVersion.W3C.relatesToTag,
getMessageId()));
return port;
}
private EndpointReference getReplyTo() {
return getHeaders().getReplyTo(AddressingVersion.W3C,
SOAPVersion.SOAP_11).toSpec();
}
private String getMessageId() {
return getHeaders().getMessageID(AddressingVersion.W3C,
SOAPVersion.SOAP_11);
}
}
this helper is called from service implementation as:
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import javax.jws.HandlerChain;
import javax.jws.WebService;
import javax.xml.ws.WebServiceContext;
import javax.xml.ws.handler.MessageContext;
import javax.xml.ws.soap.Addressing;
import dev.home.examples.jobprocessor.client.JobProcessorNotify;
import dev.home.examples.jobprocessor.client.JobProcessorNotify_Service;
import dev.home.examples.jobprocessor.client.JobReplyType;
import dev.home.examples.jobprocessor.handlers.CorrelationHelper;
import dev.home.examples.jobprocessor.types.JobType;
#WebService(serviceName = "JobProcessor",
targetNamespace = "http://examples.home.dev/jobprocessor",
portName = "jobProcessor",
endpointInterface = "dev.home.examples.jobprocessor.ws.JobProcessor")
#HandlerChain(file = "JobProcessor-HandlerChain.xml")
#Addressing(required = true)
public class JobProcessorImpl {
//...
public void processJob(JobType job) {
// do processing
int seconds = doJob();
// prepare reply message
JobReplyType jobReply = new JobReplyType();
jobReply.setJobId(job.getJobId());
jobReply.setResult(String.format("Job payload %s processed in %d seconds!",
job.getPayload(), seconds));
// do correlation and perform the callback
JobProcessorNotify jobProcessorNotify =
correlationHelper.getCorrelatedPort(JobProcessorNotify.class);
jobProcessorNotify.replyFinishedJob(jobReply);
}
}
HeaderList is not initiated during getReplyTo() and than is returned as null:
(HeaderList)wsc.getMessageContext().get(JAXWSProperties.INBOUND_HEADER_LIST_PROPERTY)
Caused by: java.lang.NullPointerException
at dev.home.examples.jobprocessor.handlers.CorrelationHelper.getReplyTo(CorrelationHelper.java:67)
at dev.home.examples.jobprocessor.handlers.CorrelationHelper.getCorrelatedPort(CorrelationHelper.java:56)
at dev.home.examples.jobprocessor.ws.JobProcessorImpl.processJob(JobProcessorImpl.java:61)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:180)
at org.apache.cxf.jaxws.JAXWSMethodInvoker.performInvocation(JAXWSMethodInvoker.java:66)
at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
... 17 more
although the SOAP data Header contains all data:
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:typ="http://examples.home.dev/jobprocessor/types">
<soapenv:Header xmlns:wsa="http://www.w3.org/2005/08/addressing"><wsa:Action>http://examples.home.dev/jobprocessor/processJob</wsa:Action><wsa:ReplyTo><wsa:Address>http://CZ407032:8088/mockJobProcessorNotify</wsa:Address></wsa:ReplyTo><wsa:MessageID>uuid:96dd09e2-c448-47c0-902f-7eb95421e232</wsa:MessageID><wsa:To>http://CZ407032:8088/JobProcessor</wsa:To></soapenv:Header>
<soapenv:Body>
...
</soapenv:Body></soapenv:Envelope>
You are using the property INBOUND_HEADER_LIST_PROPERTY. Reading JAX-WS documentation, shows an unpleasant warning
THIS PROPERTY IS EXPERIMENTAL AND IS SUBJECT TO CHANGE WITHOUT NOTICE IN FUTURE.
HeaderList and JAXWSProperties are classes in com.sun.xml.internal.ws.* package. Do you really want to use this? The blog is dated in 2012, may be the behaviour has changed
Check if this simple code return a not null object (after injecting messageContext as #Resource in your service)
HeaderList hl = (HeaderList) messageContext.get(JAXWSProperties.INBOUND_HEADER_LIST_PROPERTY);
I have a jar uploaded to aws lambda but it keeps throwing below error:
{
"errorMessage": "java.lang.NullPointerException",
"errorType": "java.lang.NullPointerException",
"stackTrace": [
"com.amazonaws.auth.profile.ProfilesConfigFile.<init>(ProfilesConfigFile.java:143)",
"com.amazonaws.auth.profile.ProfilesConfigFile.<init>(ProfilesConfigFile.java:132)",
"com.amazonaws.auth.profile.ProfilesConfigFile.<init>(ProfilesConfigFile.java:99)",
"com.amazonaws.auth.profile.ProfileCredentialsProvider.getCredentials(ProfileCredentialsProvider.java:135)",
"com.amazonaws.http.AmazonHttpClient.getCredentialsFromContext(AmazonHttpClient.java:802)",
"com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:828)",
"com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:723)",
"com.amazonaws.http.AmazonHttpClient.doExecute(AmazonHttpClient.java:475)",
"com.amazonaws.http.AmazonHttpClient.executeWithTimer(AmazonHttpClient.java:437)",
"com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:386)",
"com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:2074)",
"com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:2044)",
"com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.putItem(AmazonDynamoDBClient.java:1580)",
"com.amazonaws.services.dynamodbv2.document.internal.PutItemImpl.doPutItem(PutItemImpl.java:85)",
"com.amazonaws.services.dynamodbv2.document.internal.PutItemImpl.putItem(PutItemImpl.java:41)",
"com.amazonaws.services.dynamodbv2.document.Table.putItem(Table.java:144)",
"augury.api.SaveAuguryApi.handleRequest(SaveAuguryApi.java:46)",
"sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)",
"sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)",
"sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)",
"java.lang.reflect.Method.invoke(Method.java:498)"
]
}
and stack trace:
java.lang.NullPointerException: java.lang.NullPointerException
java.lang.NullPointerException
at com.amazonaws.auth.profile.ProfilesConfigFile.<init>(ProfilesConfigFile.java:143)
at com.amazonaws.auth.profile.ProfilesConfigFile.<init>(ProfilesConfigFile.java:132)
at com.amazonaws.auth.profile.ProfilesConfigFile.<init>(ProfilesConfigFile.java:99)
at com.amazonaws.auth.profile.ProfileCredentialsProvider.getCredentials(ProfileCredentialsProvider.java:135)
at com.amazonaws.http.AmazonHttpClient.getCredentialsFromContext(AmazonHttpClient.java:802)
at com.amazonaws.http.AmazonHttpClient.executeOneRequest(AmazonHttpClient.java:828)
at com.amazonaws.http.AmazonHttpClient.executeHelper(AmazonHttpClient.java:723)
at com.amazonaws.http.AmazonHttpClient.doExecute(AmazonHttpClient.java:475)
at com.amazonaws.http.AmazonHttpClient.executeWithTimer(AmazonHttpClient.java:437)
at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:386)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.doInvoke(AmazonDynamoDBClient.java:2074)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.invoke(AmazonDynamoDBClient.java:2044)
at com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient.putItem(AmazonDynamoDBClient.java:1580)
at com.amazonaws.services.dynamodbv2.document.internal.PutItemImpl.doPutItem(PutItemImpl.java:85)
at com.amazonaws.services.dynamodbv2.document.internal.PutItemImpl.putItem(PutItemImpl.java:41)
at com.amazonaws.services.dynamodbv2.document.Table.putItem(Table.java:144)
at augury.api.SaveAuguryApi.handleRequest(SaveAuguryApi.java:46)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
END RequestId: b2b9807e-6a09-11e6-9873-2588e6cfa497
REPORT RequestId: b2b9807e-6a09-11e6-9873-2588e6cfa497 Duration: 305.85 ms Billed Duration: 400 ms Memory Size: 512 MB Max Memory Used: 61 MB
And also my lambda java code:
package augury.api;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import augury.pojo.AuguryResponse;
import augury.pojo.AuguryResult;
public class SaveAuguryApi implements RequestHandler<AuguryResult, AuguryResponse> {
// Initialize the Log4j logger.
static final Logger log = Logger.getLogger(SaveAuguryApi.class);
static DynamoDB dynamoDB = new DynamoDB(new AmazonDynamoDBClient(new ProfileCredentialsProvider()));
static String tableName = "tarot_history";
public AuguryResponse handleRequest(AuguryResult result, Context context) {
String userId = result.getUserId();
List<Integer> tarotIds = result.getTarotIds();
String createTime = result.getCreate_time();
if (log.isDebugEnabled()) {
log.debug("requestId = " + context.getAwsRequestId() + ", userId = " + userId + ", tarotIds = " + tarotIds
+ ", create_time = " + createTime);
}
if (StringUtils.isBlank(userId) || tarotIds == null || tarotIds.isEmpty() || StringUtils.isBlank(createTime)) {
return new AuguryResponse(400, "this request doesn't contain rightful parameters, please check log");
}
Table table = dynamoDB.getTable(tableName);
Item item = new Item();
item.withString("create_time", createTime);
item.withString("user_id", userId);
item.withList("tarot_ids", tarotIds);
item.withInt("id", 1);
table.putItem(item);
return new AuguryResponse(201, "tarot history created");
}
}
i tried but i still couldn't locate the problem. I'm new to aws lambda and I was trying to learn from the link
The example you are looking at assumes you have credential properties saved to a file, which isn't going to be the case in your Lambda environment. To use the IAM role assigned to the Lambda function change this:
static DynamoDB dynamoDB = new DynamoDB(new AmazonDynamoDBClient(new ProfileCredentialsProvider()));
To this:
static DynamoDB dynamoDB = new DynamoDB(new AmazonDynamoDBClient());