SOAPConnection not handling gzip response - web-services

Question

I am using SOAPConnection to to invoke a SOAP based web service. The request is sent with "Accept-Encoding: gzip,deflate" in the header.
I used fiddler to grab the response, it is gzipped compressed, but while deserializing the message, the SOAPConnection is giving an error saying "invalid utf-8" message.
I tried normal http post and the http response is able to unzip the response correctly. Do I need to set some attributes on SOAPConnection to get it to handle the gzip message?

Answer 1

I found this snippet doing the job
SOAPMessage response = conn.call(finalRequest, aUrl);
// The response is gzip encoded, so decompress the response.
ByteArrayOutputStream out = new ByteArrayOutputStream();
response.writeTo(out);
byte[] barr = out.toByteArray();
InputStream gzipStream = new GZIPInputStream(new ByteArrayInputStream(barr));
Reader decoder = new InputStreamReader(gzipStream, "UTF-8");
BufferedReader buffered = new BufferedReader(decoder);
int n = 0;
char[] cbuf = new char[1024];
Writer w = new StringWriter();
while ((n = buffered.read(cbuf)) != -1) {
w.write(cbuf,0,n);
}
// the writer now contains unzipped message.
System.out.println(w.toString());

Answer 2

GZIP Request,Response form Server:
If the server is enable with GZip then the server sends gzip-compress text data. For this we need over a request with Http headers as:
Request: We must send a HTTP request containing the Accept-Encoding: gzip header.
Response: If gzip is enabled, the server should return the Content-Encoding: gzip header.
Soap WebServies:
a - Grahical Weather Endpoint HTTPS and its WSDL URL OnlineClient
Response Headers:
Content-Type:text/xml; charset=ISO-8859-1
Vary:Accept-Encoding
Content-Encoding:gzip
Plain Response - DemoOnline Webserice HTTP. for example see stackpostResponse Headers: Content-Type:application/soap+xml; charset=utf-8
Request the WebService using SOAPConnectiona and get response in GZIP compressed format:
public static String getGZIP(byte[] zipBytes) {
try {
GZIPInputStream gzipInput = new GZIPInputStream( new ByteArrayInputStream(zipBytes) );
return IOUtils.toString(gzipInput);
} catch (IOException e) {
throw new UncheckedIOException("Error while decompression!", e);
}
}
public static void getSOAPConnection(SOAPMessage soapMsg) throws Exception {
System.out.println("\n===== SOAPConnection =====");
MimeHeaders headers = soapMsg.getMimeHeaders();
headers.addHeader("SoapBinding", serverDetails.get("SoapBinding") );
headers.addHeader("MethodName", serverDetails.get("MethodName") );
headers.addHeader("SOAPAction", serverDetails.get("SOAPAction") );
headers.addHeader("Content-Type", serverDetails.get("Content-Type")); // InBound
headers.addHeader("Accept-Encoding", serverDetails.get("Accept-Encoding")); // OutBound
if (soapMsg.saveRequired()) soapMsg.saveChanges();
/*SOAPMessage message = MessageFactory.newInstance().createMessage(headers, new ByteArrayInputStream(TSOXML.getBytes()));*/
SOAPConnectionFactory newInstance = SOAPConnectionFactory.newInstance();
javax.xml.soap.SOAPConnection connection = newInstance.createConnection();
SOAPMessage resp = connection.call(soapMsg, getURL( serverDetails.get("SoapServerURI") ));
MimeHeaders mimeHeaders = resp.getMimeHeaders();
String[] header = mimeHeaders.getHeader("Content-Encoding");
String contentEoncoding = "";
if (header != null && header.length > 0) contentEoncoding = header[0].toString();
System.out.println("Content:"+contentEoncoding);
if (contentEoncoding.equalsIgnoreCase("GZIP")) {
System.out.println("SOAP Message in GZIP");
ByteArrayOutputStream out = new ByteArrayOutputStream();
resp.writeTo(out);
byte[] zipBytes = out.toByteArray();
String gZipString= getGZIP(zipBytes);
System.out.println("Response:"+ gZipString);
SOAPMessage soapMessage = getSOAPMessagefromDataXML(gZipString);
System.out.println("SOAP Message Object:\n"+soapMessage);
getSOAPXMLasString(soapMessage);
} else {
getSOAPXMLasString(resp);
}
}
Requesting SOAP WS using HTTPCline. The way SOAPUI requests:
public static void getHttpURLConnection_Core(SOAPMessage soapMsg) throws Exception {
System.out.println("\n===== java.net.HttpURLConnection =====");
URL url = new URL(null, serverDetails.get("SoapServerURI"));
String protocol = url.getProtocol();
System.out.println("Protocol: "+protocol);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(PROXY_ADDRESS, PROXY_PORT));
HttpURLConnection connection = (HttpURLConnection) url.openConnection(proxy);
connection.setReadTimeout(5 * 1000);
connection.setConnectTimeout(5 * 1000);
connection.setDoInput(true); connection.setDoOutput(true); connection.setUseCaches(true);
//String authString = username + ":" + password; // Authorization: Basic ZW9uMDE5XzAxOkVsaWFfMTIz
//String authMsg = "Basic " + Base64.encode(authString.getBytes());
//connection.setRequestProperty(javax.ws.rs.core.HttpHeaders.AUTHORIZATION, authMsg);
((HttpURLConnection) connection).setRequestMethod("POST");
connection.setRequestProperty(javax.ws.rs.core.HttpHeaders.ACCEPT, "text/xml");
connection.setRequestProperty(javax.ws.rs.core.HttpHeaders.ACCEPT_LANGUAGE, "en-US,en;q=0.9");
connection.setRequestProperty("MethodName", serverDetails.get("MethodName") );
connection.setRequestProperty("SOAPAction", serverDetails.get("SOAPAction") );
connection.setRequestProperty("HTTP_ACCEPT_ENCODING", "gzip, deflate, br");
connection.setRequestProperty("Accept-Encoding", serverDetails.get("Accept-Encoding"));
String soapxmLasString = getSOAPXMLasString(soapMsg);
connection.setRequestProperty(javax.ws.rs.core.HttpHeaders.CONTENT_TYPE, "text/xml");// serverDetails.get("Content-Type")
connection.setRequestProperty( "Content-Length", String.valueOf(soapxmLasString.length()));
DataOutputStream dataOutputStream = new DataOutputStream(connection.getOutputStream());
try {
dataOutputStream.writeBytes(soapxmLasString);
} finally {
dataOutputStream.close();
}
long start = System.currentTimeMillis();
long end = System.currentTimeMillis();
String date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").format(Calendar.getInstance().getTime());
System.out.println("TIme taken Date:"+date+", Time:"+ (end-start));
String contentEncoding = connection.getContentEncoding();
System.out.println("Encoding:"+ contentEncoding);
int responseCode = connection.getResponseCode();
String responseMessage = connection.getResponseMessage();
System.out.println("Response Code: " + responseCode + " " + responseMessage);
String xmlRply = null ;
String requestStatus = "Fail";
if (responseCode == HttpURLConnection.HTTP_OK) {
requestStatus = "Pass";
InputStream inputStream = connection.getInputStream();
xmlRply = getStreamContent(inputStream, contentEncoding);
} else { // Response Code: 500 Internal Server Error
InputStream errorStream = connection.getErrorStream();
xmlRply = getStreamContent(errorStream, contentEncoding);
}
System.out.println("Reply: " + xmlRply);
System.out.println("Request Status:"+ requestStatus);
}
public static String getStreamContent(InputStream input, String encoding) throws IOException {
byte[] httpRply;
String rply;
httpRply = IOUtils.toByteArray(input);
System.out.println("Byte Array:"+httpRply.toString());
if (encoding == null) {
rply = new String(httpRply);
} else if ( encoding.equalsIgnoreCase("GZIP") ) {
rply = getGZIP(httpRply);
} else { // "ISO-8859-1", ";TF-8"
rply = new String(httpRply, encoding);
}
return rply;
}
Working example where the Server response with GZIP format.
public class SOAP_Weather {
static final String PROXY_ADDRESS = "Proxy_*****.net";
static final int PROXY_PORT = 9400;
static HashMap<String, String> serverDetails = new HashMap<>();
static {
// https://graphical.weather.gov/xml/ : conus
serverDetails.put("SoapServerURI", "https://graphical.weather.gov:443/xml/SOAP_server/ndfdXMLserver.php");
serverDetails.put("SoapWSDL", "https://graphical.weather.gov/xml/SOAP_server/ndfdXMLserver.php?wsdl");
serverDetails.put("SoapXML", "<ndf:CornerPoints xmlns:ndf=\"https://graphical.weather.gov/xml/DWMLgen/wsdl/ndfdXML.wsdl\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"><sector xsi:type=\"xsd:string\">conus</sector></ndf:CornerPoints>");
serverDetails.put("SoapBinding", "ndfdXMLBinding");
serverDetails.put("MethodName", "CornerPoints");
serverDetails.put("SOAPAction", "https://graphical.weather.gov/xml/DWMLgen/wsdl/ndfdXML.wsdl#CornerPoints");
serverDetails.put("User-Agent", "Apache-HttpClient");
serverDetails.put("Accept-Encoding", "gzip,deflate,sdch");
serverDetails.put("Content-Type", "text/xml;charset=UTF-8");
}
public static void main(String[] args) throws Exception {
callSoapService();
}
public static void callSoapService( ) throws Exception{
String xmlData = serverDetails.get("SoapXML");
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
dbFactory.setNamespaceAware(true);
dbFactory.setIgnoringComments(true);
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
InputSource ips = new org.xml.sax.InputSource(new StringReader(xmlData));
Document docBody = dBuilder.parse(ips);
//docBody.createElementNS(DSIG_NS, "ds");
System.out.println("Data Document: "+docBody.getDocumentElement());
// Protocol 1.1=SOAP-ENV Content-Type:text/xml; charset=utf-8, 1.2=env Content-Type:application/soap+xml; charset=utf-8
MessageFactory messageFactory = MessageFactory.newInstance(SOAPConstants.SOAP_1_2_PROTOCOL);
SOAPMessage soapMsg = messageFactory.createMessage();
SOAPPart soapPart = soapMsg.getSOAPPart();
SOAPEnvelope soapEnv = soapPart.getEnvelope();
SOAPBody soapBody = soapEnv.getBody();
soapBody.addDocument(docBody);
// Invoke the webService.
System.out.println("Request SOAP Message:");
soapMsg.writeTo(System.out);
System.out.println("\n");
// Protocol 1.1=SOAP-ENV Content-Type:text/xml; charset=utf-8, 1.2=env Content-Type:application/soap+xml; charset=utf-8
SOAPEnvelope envelope = soapMsg.getSOAPPart().getEnvelope();
if (envelope.getElementQName().getNamespaceURI().equals("http://schemas.xmlsoap.org/soap/envelope/")) {
System.out.println("SOAP 1.1 NamespaceURI: http://schemas.xmlsoap.org/soap/envelope/");
serverDetails.put("Content-Type", "text/xml; charset=utf-8");
} else {
System.out.println("SOAP 1.2 NamespaceURI: http://www.w3.org/2003/05/soap-envelope");
serverDetails.put("Content-Type", "application/soap+xml; charset=utf-8");
}
getSOAPConnection(soapMsg); // Disadvantage 1:
getHttpURLConnection_Core(soapMsg);
}
private static URL getURL(String endPointUrl) throws MalformedURLException {
URL endpoint = new URL(null, endPointUrl, new URLStreamHandler() {
protected URLConnection openConnection(URL url) throws IOException {
URL clone = new URL(url.toString());
URLConnection connection = null;
if (PROXY_ADDRESS != null && PROXY_PORT != 0 ) { // https://stackoverflow.com/a/22533464/5081877
Socket socket = new Socket();
SocketAddress sockaddr = new InetSocketAddress(PROXY_ADDRESS, PROXY_PORT);
socket.connect(sockaddr, 10000);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(socket.getInetAddress(), PROXY_PORT));
if (proxy.address().toString().equals("0.0.0.0/0.0.0.0:80") || proxy.address().toString() != null) {
System.out.println("Connection through proxy ...");
connection = clone.openConnection(proxy);
} else {
connection = clone.openConnection();
}
} else {
connection = clone.openConnection();
}
connection.setConnectTimeout(5 * 1000); // 5 sec
connection.setReadTimeout(5 * 1000); // 5 sec
return connection;
}
});
return endpoint;
}
public static String getSOAPXMLasString(SOAPMessage soapMsg) throws SOAPException, IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
soapMsg.writeTo(out);
// resp.writeTo(System.out);
String strMsg = new String(out.toByteArray());
System.out.println("Soap XML: "+ strMsg);
return strMsg;
}
public static SOAPMessage getSOAPMessagefromDataXML(String saopBodyXML) throws Exception {
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
dbFactory.setNamespaceAware(true);
dbFactory.setIgnoringComments(true);
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
InputSource ips = new org.xml.sax.InputSource(new StringReader(saopBodyXML));
Document docBody = dBuilder.parse(ips);
System.out.println("Data Document: "+docBody.getDocumentElement());
MessageFactory messageFactory = MessageFactory.newInstance(SOAPConstants.SOAP_1_2_PROTOCOL);
SOAPMessage soapMsg = messageFactory.createMessage();
SOAPBody soapBody = soapMsg.getSOAPPart().getEnvelope().getBody();
soapBody.addDocument(docBody);
return soapMsg;
}
// SOPAConneciton and HTTPClien connection functions
}

Answer 3

did you see this?
SOAPMessage - SOAPConnection - gzip - how to
Also, if you use AXIS2, it has built-in features to enable compression of base64-encoded binary content using MTOM:
http://axis.apache.org/axis2/java/core/docs/mtom-guide.html

Answer 4

Just to complete that snippet - if you want to work with the decompressed SOAPMessage you need to load it into a new message.
SOAPMessage responseCompressed = connection.call(reqMessage, endpoint);
ByteArrayOutputStream out = new ByteArrayOutputStream();
responseCompressed.writeTo(out);
byte[] barr = out.toByteArray();
InputStream gzipStream = new GZIPInputStream(new ByteArrayInputStream(barr));
Here is the magic line
SOAPMessage response = factory.createMessage(responseCompressed.getMimeHeaders(), gzipStream);
Where factory is your MessageFactory.
Now response will function like it did without the gzip headers. You just drop it in.