Extracting binary as base64 String results in incomplete file? - wso2

Question

I am trying to extract binary from soap response and convert it to base64 string but what happens is that I get part of the binary converted to base64 not the full binary data.
Integration Studio : 8.1.0
the soap response is as following :
--A-B--MIME-BOUNDARY--27d99311536fa8c4-18606176189--Y-Z
Content-Transfer-Encoding: binary
Content-Type: application/xop+xml; type="application/soap+xml"; charset=UTF-8
Content-ID:
<v0-27d99311536fa8c4-18606176189#mtom.p8ce.filenet.com>
<?xml version="1.0" encoding="UTF-8"?>
<e:Envelope xmlns:d="http://www.w3.org/2001/XMLSchema" xmlns:i="http://www.w3.org/2001/XMLSchema-instance" xmlns:fn35="http://www.filenet.com/ns/fnce/2005/02/ws/schema" xmlns:fn40m="http://www.filenet.com/ns/fnce/2006/11/ws/MTOM/schema" xmlns:fn40="http://www.filenet.com/ns/fnce/2006/11/ws/schema" xmlns:xop="http://www.w3.org/2004/08/xop/include" xmlns:e="http://schemas.xmlsoap.org/soap/envelope/">
<e:Body>
<GetContentResponse xmlns="http://www.filenet.com/ns/fnce/2006/11/ws/schema">
<ContentResponse id="1" i:type="fn40:ContentElementResponse" retrievalName="test.pdf" totalSize="4586622" bufferedSize="1048576" continueFrom="offset=1048576;id={B3795692-DA06-468C-AEC9-523A3715455D};">
<SourceSpecification i:type="fn40:ObjectReference" classId="Document" objectId="{E408981D-FF67-4D91-B7A9-CEBEC4630AB0}" objectStore="MCIDEV"></SourceSpecification>
<ElementSpecification elementSequenceNumber="0"></ElementSpecification>
<Content i:type="fn40:InlineContent">
<Binary>
<xop:Include href="cid:v1-27d99311536fa8c4-18606176189#mtom.p8ce.filenet.com"></xop:Include>
</Binary>
</Content>
</ContentResponse>
</GetContentResponse>
</e:Body>
</e:Envelope>
--A-B--MIME-BOUNDARY--27d99311536fa8c4-18606176189--Y-Z
Content-Transfer-Encoding: binary
Content-Type: application/octet-stream
Content-Id:
<v1-27d99311536fa8c4-18606176189#mtom.p8ce.filenet.com>
%PDF-1.4
......
......
......
--A-B--MIME-BOUNDARY--27d99311536fa8c4-18606176189--Y-Z--
The code to extract the binary and convert it to base64 string is as follows :
<property expression="$body/fn:GetContentResponse/fn:ContentResponse/fn:Content/fn:Binary" name="DataBase64" scope="default" type="STRING" xmlns:fn="http://www.filenet.com/ns/fnce/2006/11/ws/schema"/>
My service code is as follows :
<?xml version="1.0" encoding="UTF-8"?>
<sequence name="StorageGetDocumentSeq" trace="disable" xmlns="http://ws.apache.org/ns/synapse">
<propertyGroup>
<property expression="json-eval($.username)" name="Username" scope="default" type="STRING"/>
<property expression="json-eval($.password)" name="Password" scope="default" type="STRING"/>
<property expression="json-eval($.objectStore)" name="ObjectStore" scope="default" type="STRING"/>
<property name="DocumentClass" scope="default" type="STRING" value="Document"/>
<property expression="json-eval($.documentId)" name="DocumentId" scope="default" type="STRING"/>
<property expression="json-eval($.version)" name="Version" scope="default" type="STRING"/>
</propertyGroup>
<sequence key="StorageRequestValidatorSeq"/>
<script description="Response Builder" language="js"><![CDATA[var errors = [];
var errorProperties = mc.getPropertyKeySet();
var it = errorProperties.iterator();
var i = 1;
while(it.hasNext()){
var prop = it.next();
if (prop.contains('Error')) {
print(prop.toString());
errors.push(mc.getProperty(prop));
}
}
mc.setProperty('errorList', "[" + errors.toString() + "]");]]></script>
<filter regex="false" source="get-property('IsValid')">
<then>
<payloadFactory description="Response Payload" media-type="json">
<format>
{"errorCode":400,"errorMessage":"One or more validation error","errorList":$1}</format>
<args>
<arg evaluator="xml" expression="get-property('errorList')"/>
</args>
</payloadFactory>
<property description="Set 400 response code" name="HTTP_SC" scope="axis2" type="STRING" value="400"/>
<respond description="Response to client"/>
</then>
<else/>
</filter>
<sequence key="StorageGetMimeTypeSeq"/>
<payloadFactory media-type="xml">
<format>
<soapenv:Envelope xmlns:soapenc="http://schemas.xmlsoap.org/soap/encoding/" xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<soapenv:Header>
<Security xmlns="http://schemas.xmlsoap.org/ws/2002/12/secext">
<hd:UsernameToken xmlns:hd="http://schemas.xmlsoap.org/ws/2002/12/secext">
<hd:Username>$1</hd:Username>
<hd:Password>$2</hd:Password>
</hd:UsernameToken>
</Security>
</soapenv:Header>
<soapenv:Body>
<p857:GetContentRequest validateOnly="0" xmlns:p857="http://www.filenet.com/ns/fnce/2006/11/ws/schema">
<p857:ContentRequest continueFrom="" cacheAllowed="1" id="1" startOffset="0">
<p857:SourceSpecification classId="$4" itemIndex="0" objectId="$5" objectStore="$3"
serializationDuplicate="0" xsi:type="p857:ObjectSpecification"/>
<p857:ElementSpecification elementSequenceNumber="0" itemIndex="0"/>
</p857:ContentRequest>
</p857:GetContentRequest>
</soapenv:Body>
</soapenv:Envelope>
</format>
<args>
<arg evaluator="xml" expression="get-property('Username')"/>
<arg evaluator="xml" expression="get-property('Password')"/>
<arg evaluator="xml" expression="get-property('ObjectStore')"/>
<arg evaluator="xml" expression="get-property('DocumentClass')"/>
<arg evaluator="xml" expression="get-property('DocumentId')"/>
</args>
</payloadFactory>
<call-template target="StorageEndpointTemplate"/>
<propertyGroup>
<property expression="$axis2:HTTP_SC" name="HTTPStatus" scope="default" type="STRING"/>
<property expression="$body/e:Fault/e:Reason/e:Text" name="ErrorMessage" scope="default" type="STRING" xmlns:e="http://www.w3.org/2003/05/soap-envelope"/>
<property expression="$body/fn:GetContentResponse/fn:ContentResponse/#retrievalName" name="FileName" scope="default" type="STRING" xmlns:fn="http://www.filenet.com/ns/fnce/2006/11/ws/schema"/>
<property expression="$body/fn:GetContentResponse/fn:ContentResponse/#totalSize" name="FileSize" scope="default" type="STRING" xmlns:fn="http://www.filenet.com/ns/fnce/2006/11/ws/schema"/>
<property expression="$body/fn:GetContentResponse/fn:ContentResponse/fn:Content/fn:Binary" name="DataBase64" scope="default" type="STRING" xmlns:fn="http://www.filenet.com/ns/fnce/2006/11/ws/schema"/>
</propertyGroup>
<log level="headers"/>
<log level="custom">
<property expression="get-property('FileName')" name="FileNameLog"/>
<property expression="get-property('FileSize')" name="FileSizeLog"/>
</log>
<filter regex="true" source="boolean(get-property('ErrorMessage'))=''">
<then>
<payloadFactory media-type="json">
<format>
{
"fileName":"$1",
"fileSize": $2,
"mimeType":"$3",
"dataBase64":"$4"
}
</format>
<args>
<arg evaluator="xml" expression="get-property('FileName')"/>
<arg evaluator="xml" expression="get-property('FileSize')"/>
<arg evaluator="xml" expression="get-property('MimeType')"/>
<arg evaluator="xml" expression="get-property('DataBase64')"/>
</args>
</payloadFactory>
</then>
<else>
<property name="HTTP_SC" scope="axis2" type="STRING" value="400"/>
<payloadFactory description="Response Payload" media-type="json">
<format>{"errorCode": "$1","errorMessage": "$2"}</format>
<args>
<arg value="400"/>
<arg evaluator="xml" expression="get-property('ErrorMessage')"/>
</args>
</payloadFactory>
</else>
</filter>
</sequence>
EDIT : I also tried the base64Encode as follows :
<property expression="base64Encode($body/fn:GetContentResponse/fn:ContentResponse/fn:Content/fn:Binary)" name="DataBase64" scope="default" type="STRING" xmlns:fn="http://www.filenet.com/ns/fnce/2006/11/ws/schema"/>
And it also results in corrupted base64String

Answer 1

I found out that the issue was not related to WSO2 at all but a filenet limitation
Inline Content Retrieval Limit default is 1 MB so that filenet API itself was returning 1 MB only of the file.
https://www.ibm.com/docs/en/filenet-p8-platform/5.5.x?topic=engine-improving-content-uploads-downloads