WSO2 Aggregate Mediator premature/incomplete/unknown completion - wso2

Question

I'm having a confusing issue with my aggregate mediator inside an out Sequence of a proxy.
Configuration :
A sequence implementing the iterate mediator Iterated the following message :
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/">
<soapenv:Body>
<productSearchRs xmlns="SHC">
<productDetails>
<Product_ID>1487326</Product_ID>
<Product_Name>SECRET</Product_Name>
<Product_Size>M</Product_Size>
<Product_Colour>BLACK</Product_Colour>
<Product_Type>SOCKS</Product_Type>
<Source>SHC</Source>
</productDetails>
<productDetails>
<Product_ID>1985211</Product_ID>
<Product_Name>SECRET</Product_Name>
<Product_Size>M</Product_Size>
<Product_Colour>BLACK</Product_Colour>
<Product_Type>SOCKS</Product_Type>
<Source>SHC</Source>
</productDetails>
</productSearchRs>
</soapenv:Body>
</soapenv:Envelope>
Into these (after a payloadFactory):
<pfPadding>
<productDetails>
<productID>1487326</productID>
<productName>SECRET</productName>
<productSize>M</productSize>
<productColour>BLACK</productColour>
<productType>SOCKS</productType>
<sourceID>SHC</sourceID>
</productDetails>
</pfPadding>
and
<pfPadding>
<productDetails>
<productID>1985211</productID>
<productName>SECRET</productName>
<productSize>M</productSize>
<productColour>BLACK</productColour>
<productType>SOCKS</productType>
<sourceID>SHC</sourceID>
</productDetails>
</pfPadding>
Full sequence:
<sequence xmlns="http://ws.apache.org/ns/synapse" name="shcServiceSearchSeqRs">
<log level="custom">
<property name="Below response recieved from Shc-DS" value="=================="></property>
</log>
<log level="full"></log>
<log level="custom">
<property name="Splitting product_Detail/s up" value="=================="></property>
</log>
<property xmlns:ns1="SHC" xmlns:ns="http://org.apache.synapse/xsd" name="Product_ID" expression="//ns1:productDetails/ns1:Product_ID/text()" scope="default" type="STRING"></property>
<filter xmlns:ns="http://org.apache.synapse/xsd" xpath="boolean(get-property('Product_ID'))">
<then>
<iterate xmlns:ns1="SHC" expression="//ns1:productDetails">
<target>
<sequence>
<property name="Product_ID" expression="//ns1:productDetails/ns1:Product_ID/text()" scope="default" type="STRING"></property>
<property name="Product_Name" expression="//ns1:productDetails/ns1:Product_Name/text()" scope="default" type="STRING"></property>
<property name="Product_Size" expression="//ns1:productDetails/ns1:Product_Size/text()" scope="default" type="STRING"></property>
<property name="Product_Colour" expression="//ns1:productDetails/ns1:Product_Colour/text()" scope="default" type="STRING"></property>
<property name="Product_Type" expression="//ns1:productDetails/ns1:Product_Type/text()" scope="default" type="STRING"></property>
<property name="Source" expression="//ns1:productDetails/ns1:Source/text()" scope="default" type="STRING"></property>
<log level="full"></log>
<log level="custom">
<property name="shcRs : P_ID : " expression="$ctx:Product_ID"></property>
<property name="shcRs : P_Name : " expression="$ctx:Product_Name"></property>
<property name="shcRs : P_Size : " expression="$ctx:Product_Size"></property>
<property name="shcRs : P_Colour : " expression="$ctx:Product_Colour"></property>
<property name="shcRs : P_Type : " expression="$ctx:Product_Type"></property>
<property name="shcRs : P_Source : " expression="$ctx:Source"></property>
</log>
<payloadFactory media-type="xml">
<format>
<productDetails xmlns="">
<productID>$1</productID>
<productName>$2</productName>
<productSize>$3</productSize>
<productColour>$4</productColour>
<productType>$5</productType>
<sourceID>$6</sourceID>
</productDetails>
</format>
<args>
<arg expression="$ctx:Product_ID" evaluator="xml"></arg>
<arg expression="$ctx:Product_Name" evaluator="xml"></arg>
<arg expression="$ctx:Product_Size" evaluator="xml"></arg>
<arg expression="$ctx:Product_Colour" evaluator="xml"></arg>
<arg expression="$ctx:Product_Type" evaluator="xml"></arg>
<arg expression="$ctx:Source" evaluator="xml"></arg>
</args>
</payloadFactory>
<log level="custom">
<property name="Iterated Message : : :" value="++++++++++++++===================================================+++++++++++++++++=="></property>
</log>
<log level="full"></log>
<sequence key="conf:/pocSearchRsHandlerSeq"></sequence>
</sequence>
</target>
</iterate>
</then>
<else>
<payloadFactory media-type="xml">
<format>
<productDetails xmlns="">
<noItemFound>No item in ShoeCity</noItemFound>
</productDetails>
</format>
</payloadFactory>
<sequence key="conf:/pocSearchRsHandlerSeq"></sequence>
</else>
</filter>
</sequence>
Here's my Response handler :
<?xml version="1.0" encoding="UTF-8"?>
<sequence xmlns="http://ws.apache.org/ns/synapse" name="pocSearchRsHandlerSeq">
<log level="custom">
<property name="pocSearchRsHandlerSeq reached" value="++++++++++++++=========Aggregating now========+++++++++++++++++=="/>
</log>
<aggregate>
<completeCondition>
<messageCount min="-1" max="-1"/>
</completeCondition>
<onComplete xmlns:ns="http://org.apache.synapse/xsd" expression="//productDetails">
<log level="custom" separator=",">
<property name="::::" value="======================= Formatting the Aggregated Responses. ==============="/>
</log>
<log level="full"/>
</onComplete>
</aggregate>
<payloadFactory media-type="xml">
<format>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:prod="http://za.co.pepkor/product_service/">
<soapenv:Body>
<prod:productSearchResp> $1 </prod:productSearchResp>
</soapenv:Body>
</soapenv:Envelope>
</format>
<args>
<arg xmlns:ns="http://org.apache.synapse/xsd" evaluator="xml" expression="//productDetails"/>
</args>
</payloadFactory>
<log level="custom">
<property name="Full Rs Message : : : " value="++++++++++++++===================================================+++++++++++++++++=="/>
</log>
<respond/>
</sequence>
This sequence also receives another message from a different sequence which it does aggregate (sometimes)
Here are some responses (all from same request)
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:prod="http://za.co.pepkor/product_service/">
<soapenv:Body>
<prod:productSearchResp>
<productDetails>
<productID>2145627</productID>
<productName>NIKE_SHIRTS</productName>
<productSize>7</productSize>
<productColour>RED</productColour>
<productType>SHIRT</productType>
<sourceID>ACK</sourceID>
</productDetails>
<productDetails>
<productID>1452168</productID>
<productName>PUMPS</productName>
<productSize>7</productSize>
<productColour>ORANGE</productColour>
<productType>SHOE</productType>
<sourceID>SHC</sourceID>
</productDetails>
</prod:productSearchResp>
</soapenv:Body>
</soapenv:Envelope>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:prod="http://za.co.pepkor/product_service/">
<soapenv:Body>
<prod:productSearchResp></prod:productSearchResp>
</soapenv:Body>
</soapenv:Envelope>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:prod="http://za.co.pepkor/product_service/">
<soapenv:Body>
<prod:productSearchResp>
<productDetails>
<productID>1452168</productID>
<productName>PUMPS</productName>
<productSize>7</productSize>
<productColour>ORANGE</productColour>
<productType>SHOE</productType>
<sourceID>SHC</sourceID>
</productDetails>
<productDetails>
<productID>2145627</productID>
<productName>NIKE_SHIRTS</productName>
<productSize>7</productSize>
<productColour>RED</productColour>
<productType>SHIRT</productType>
<sourceID>ACK</sourceID>
</productDetails>
<productDetails>
<productID>1124596</productID>
<productName>REEBOK_SNEAKERS</productName>
<productSize>7</productSize>
<productColour>BROWN</productColour>
<productType>SHOES</productType>
<sourceID>SHC</sourceID>
</productDetails>
</prod:productSearchResp>
</soapenv:Body>
</soapenv:Envelope>
I mainly get the empty response back , I noticed in the logs that these have the following line in the log where the others(non-empty) don't:
TID: [0] [ESB] [2015-01-16 16:00:15,770] DEBUG {org.apache.synapse.registry.AbstractRegistry} - Cached object has expired for key : conf:/ackServiceSearchSeqRs {org.apache.synapse.registry.AbstractRegistry}
TID: [0] [ESB] [2015-01-16 16:00:15,770] DEBUG {org.apache.synapse.registry.AbstractRegistry} - Expired version number is same as current version in registry {org.apache.synapse.registry.AbstractRegistry}
EDIT :
What is the difference between using a "send" mediator with a sequence key and a "sequence" mediator?
Also , can I use a single "aggregate" node to aggregate responses from an "iterate" in one sequnece AND responses from a different sequence , reliably? It seems my aggregate node is prematurely "completing" before collecting all of my responses , and therefore executing my send/respond(any difference here?) mediator. Because I noticed this in the documentation :
Note that when the Iterate mediator is used to split the requests and produces only an n number of fragmented messages, the Aggregate mediator will terminate as soon as it receives n responses, even if you have specified a higher minimum limit.
Enabling reliable messaging gives me all sorts of other problems.

Answer 1

I solved it like such :
Turns out I needed a specific aggregator to agg my iterated messages (linked through the id attribute on the iterate/aggregate nodes respectively), then another to aggregate the other response as well as the above (already aggregated responses)
My Response handler:
<sequence xmlns="http://ws.apache.org/ns/synapse">
<log level="custom">
<property name="pocSearchRsHandlerSeq reached" value="++++++++++++++=========Aggregating now========+++++++++++++++++=="></property>
</log>
<aggregate id="iT">
<completeCondition>
<messageCount min="-1" max="-1"></messageCount>
</completeCondition>
<onComplete xmlns:ns="http://org.apache.synapse/xsd" expression="//productDetails">
<log level="custom" separator=",">
<property name="::::" value="======================= Formatting the Aggregated Responses. ==============="></property>
</log>
<log level="full"></log>
</onComplete>
</aggregate>
<aggregate>
<completeCondition>
<messageCount min="-1" max="-1"></messageCount>
</completeCondition>
<onComplete xmlns:ns="http://org.apache.synapse/xsd" expression="//productDetails">
<payloadFactory media-type="xml">
<format>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:prod="http://za.co.pepkor/product_service/">
<soapenv:Body>
<prod:productSearchResp> $1 </prod:productSearchResp>
</soapenv:Body>
</soapenv:Envelope>
</format>
<args>
<arg expression="//productDetails" evaluator="xml"></arg>
</args>
</payloadFactory>
<respond></respond>
</onComplete>
</aggregate>
</sequence>

Answer 2

In your sequence "pocSearchRsHandlerSeq", you should send back your aggregated response inside the aggregate's onComplete node :
<?xml version="1.0" encoding="UTF-8"?>
<sequence xmlns="http://ws.apache.org/ns/synapse" name="pocSearchRsHandlerSeq">
<aggregate>
<completeCondition>
<messageCount min="-1" max="-1"/>
</completeCondition>
<onComplete xmlns:ns="http://org.apache.synapse/xsd" expression="//productDetails">
<payloadFactory media-type="xml">
<format>
<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:prod="http://za.co.pepkor/product_service/">
<soapenv:Body>
<prod:productSearchResp>$1</prod:productSearchResp>
</soapenv:Body>
</soapenv:Envelope>
</format>
<args>
<arg xmlns:ns="http://org.apache.synapse/xsd" evaluator="xml" expression="//productDetails"/>
</args>
</payloadFactory>
<send/>
</onComplete>
</aggregate>
</sequence>