RPC failure code 14 in our simple GRPC example program - c++

Question

We've made good progress in getting GRPC running under RHEL 7.
Our application has one rather complicated structure with three levels of nesting with the outer level implementing a "oneof" keyword.
We find that all our other structures run fine, but this one gives us an RPC failure with code=14.
We've simplified this part of the application as much as possible so it can hopefully be recompiled and run easily.
Here's the .proto file, updated to accommodate Uli's question:
syntax = "proto3";
option java_multiple_files = true;
option java_package = "io.grpc.examples.debug";
option java_outer_classname = "DebugProto";
option objc_class_prefix = "DEBUG";
package DEBUGpackage;
service DEBUGservice {
rpc DEBUG_val_container_get (input_int32_request) returns (outer_container) {}
}
message input_int32_request {
int32 ival = 1;
}
message inner_container {
repeated uint32 val_array = 1;
}
message middle_container {
inner_container vac = 1;
}
message other_container {
int32 other_val = 1;
}
message outer_container {
oneof reply {
middle_container r1 = 1;
other_container r2 = 2;
}
}
(Please note that the java lines in this prototype code are just in there because they are in the GRPC website examples. Our code is entirely C++, with no java. Don't know if that means we can do without some of these "option java..." lines).
Here's our client source code:
#include <iostream>
#include <memory>
#include <string>
#include <grpc++/grpc++.h>
#include <grpc/support/log.h>
#include <thread>
#include <unistd.h>
#include "debug.grpc.pb.h"
using grpc::Channel;
using grpc::ClientAsyncResponseReader;
using grpc::ClientContext;
using grpc::CompletionQueue;
using grpc::Status;
using DEBUGpackage::input_int32_request;
using DEBUGpackage::inner_container;
using DEBUGpackage::middle_container;
using DEBUGpackage::outer_container;
using DEBUGpackage::DEBUGservice;
class DEBUGClient {
public:
explicit DEBUGClient(std::shared_ptr<Channel> channel)
: stub_(DEBUGservice::NewStub(channel)) {}
void DEBUG_val_container_get() {
std::cout << "in DEBUG_val_container_get" << std::endl;
// Data we are sending to the server
input_int32_request val;
val.set_ival(0);
AsyncClientCall* call = new AsyncClientCall;
call->response_reader = stub_->AsyncDEBUG_val_container_get(&call->context, val, &cq_);
call->response_reader->Finish(&call->reply_, &call->status, (void*)call);
}
void AsyncCompleteRpc() {
void* got_tag;
bool ok = false;
while (cq_.Next(&got_tag, &ok)) {
AsyncClientCall* call = static_cast<AsyncClientCall*>(got_tag);
GPR_ASSERT(ok);
if (call->status.ok()) {
if (call->reply_.has_r1()) {
std::cout << call << " DEBUG received: "
<< call->reply_.r1().vac().val_array(0) << std::endl;
}
}
else {
std::cout << call << " RPC failed" << std::endl;
std::cout << " RPC failure code = " << call->status.error_code() << std::endl;
std::cout << " RPC failure message = " << call->status.error_message() << std::endl;
}
delete call;
}
}
private:
struct AsyncClientCall {
outer_container reply_;
ClientContext context;
Status status;
std::unique_ptr<ClientAsyncResponseReader<outer_container>> response_reader;
};
std::unique_ptr<DEBUGservice::Stub> stub_;
CompletionQueue cq_;
};
int main(int argc, char** argv) {
DEBUGClient DEBUG0(grpc::CreateChannel("172.16.17.46:50050", grpc::InsecureChannelCredentials()));
std::thread thread0_ = std::thread(&DEBUGClient::AsyncCompleteRpc, &DEBUG0);
DEBUG0.DEBUG_val_container_get();
sleep(1);
std::cout << "Press control-c to quit" << std::endl << std::endl;
thread0_.join(); //blocks forever
return 0;
}
And, here's our server source code:
#include <memory>
#include <iostream>
#include <string>
#include <thread>
#include <grpc++/grpc++.h>
#include <grpc/support/log.h>
#include "debug.grpc.pb.h"
#include <time.h>
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
using grpc::Server;
using grpc::ServerAsyncResponseWriter;
using grpc::ServerBuilder;
using grpc::ServerContext;
using grpc::ServerCompletionQueue;
using grpc::Status;
using DEBUGpackage::inner_container;
using DEBUGpackage::input_int32_request;
using DEBUGpackage::middle_container;
using DEBUGpackage::outer_container;
using DEBUGpackage::DEBUGservice;
std::string save_server_address;
class ServerImpl final {
public:
~ServerImpl() {
server_->Shutdown();
cq_->Shutdown();
}
void Run() {
std::string server_address("0.0.0.0:50050");
ServerBuilder builder;
builder.AddListeningPort(server_address, grpc::InsecureServerCredentials());
builder.RegisterService(&service_);
cq_ = builder.AddCompletionQueue();
server_ = builder.BuildAndStart();
std::cout << "Server listening on " << server_address << std::endl;
save_server_address = server_address;
HandleRpcs();
}
private:
class CallData {
public:
virtual void Proceed() = 0;
};
class DebugGetCallData final : public CallData{
public:
DebugGetCallData(DEBUGservice::AsyncService* service, ServerCompletionQueue* cq)
: service_(service), cq_(cq), responder_(&ctx_), status_(CREATE) {
Proceed();
}
void Proceed() {
if (status_ == CREATE) {
status_ = PROCESS;
service_->RequestDEBUG_val_container_get(&ctx_, &request_, &responder_, cq_, cq_, this);
} else if (status_ == PROCESS) {
new DebugGetCallData(service_, cq_);
char *portchar;
portchar = (char *) save_server_address.c_str();
long cq_addr = (long) cq_;
int cq_addr32 = (int) (cq_addr & 0xfffffff);
srand(cq_addr32);
fprintf(stderr, "%s task started\n", portchar); fflush(stderr);
unsigned int return_val = 10;
inner_container ic;
ic.add_val_array(return_val);
middle_container reply_temp;
reply_temp.set_allocated_vac(&ic);
reply_.set_allocated_r1(&reply_temp);
fprintf(stderr, "%s %s task done\n", portchar, "val_container_get"); fflush(stderr);
status_ = FINISH;
responder_.Finish(reply_, Status::OK, this);
} else {
GPR_ASSERT(status_ == FINISH);
}
}
private:
DEBUGservice::AsyncService* service_;
ServerCompletionQueue* cq_;
ServerContext ctx_;
input_int32_request request_;
outer_container reply_;
ServerAsyncResponseWriter<outer_container> responder_;
enum CallStatus { CREATE, PROCESS, FINISH };
CallStatus status_;
};
void HandleRpcs() {
new DebugGetCallData(&service_, cq_.get());
void* tag;
bool ok;
while (true) {
GPR_ASSERT(cq_->Next(&tag, &ok));
GPR_ASSERT(ok);
static_cast<CallData*>(tag)->Proceed();
}
}
std::unique_ptr<ServerCompletionQueue> cq_;
DEBUGservice::AsyncService service_;
std::unique_ptr<Server> server_;
};
int main() {
ServerImpl server;
server.Run();
return 0;
}
The output when I run it looks like this:
[fossum#netsres46 debug]$ DEBUG_client2
in DEBUG_val_container_get
0xb73ff0 RPC failed
RPC failure code = 14
RPC failure message = Endpoint read failed
Press control-c to quit
We ran the server under gdb, and found a place in the generated
file "debug.pb.cc" where if we just comment out one line, it all starts working.
Here's the pertinent piece of the generated file "debug.pb.cc":
middle_container::~middle_container() {
// ##protoc_insertion_point(destructor:DEBUGpackage.middle_container)
SharedDtor();
}
void middle_container::SharedDtor() {
if (this != internal_default_instance()) {
delete vac_; // comment out this one line, to make the problem go away
}
}
The "delete vac_" line appears to be an attempt to delete storage that either has already been deleted, or is about to be deleted somewhere else. Please, can someone look into this? [The files below are still the files we use to generate this code, and to debug the problem to this point]
I have no idea whether I've uncovered a bug in GRPC, or whether I've coded something wrong.

Answer 1

The issue is that you are allocated middle_container reply_tmp on the stack in your server. As a result it gets destructed as soon as you pass out of the scope. At that time, you have called Finish but not yet waited for its result. Since this is an async server, the data must remain alive until you've received the tag back for it. This is why manually editing your destructor works in your case; you're basically nullifying the destructor (and leaking memory as a result).