Method PeerConnection::SetLocalDescription in NativeAPI crashes - c++

I'm trying to set up a data channel between a server written in C++ and a Python client. The server crashes with SIGSEGV error when it tries to set a local session description created in method "CreateAnswer"
The server and client exchange SDP information via WebSocket and should open the data channel without video and audio streams. Both programs are working under docker-compose in different services. So no audio or video devices are provided. I use WebRTC Native API from m76 branch.
Crashing handler:
static void OnAnswerCreated(WebRTCManagerImpl* impl_, webrtc::SessionDescriptionInterface* desc) {
LOG4CPLUS_INFO_FMT(impl_->logger_, "Answer created session_id %s", desc->session_id().c_str());
std::string offer_string;
desc->ToString(&offer_string);
LOG4CPLUS_DEBUG_FMT(impl_->logger_, "Offer string: %s", offer_string.c_str());
impl_->peer_connection_->SetLocalDescription(&impl_->set_session_description_observer_, desc);
impl_->signaling_->SendSessionDescription(*desc);
};
I create my connection with this factory:
webrtc::PeerConnectionFactoryDependencies CreatePeerConnectionFactoryDependencies() {
webrtc::PeerConnectionFactoryDependencies dependencies;
dependencies.network_thread = nullptr;
dependencies.worker_thread = nullptr;
dependencies.signaling_thread = nullptr;
dependencies.call_factory = webrtc::CreateCallFactory();
dependencies.task_queue_factory = webrtc::CreateDefaultTaskQueueFactory();
dependencies.event_log_factory = absl::make_unique<webrtc::RtcEventLogFactory>(dependencies.task_queue_factory.get());
cricket::MediaEngineDependencies mediaDependencies;
mediaDependencies.task_queue_factory = dependencies.task_queue_factory.get();
mediaDependencies.adm = rtc::scoped_refptr<webrtc::FakeAudioDeviceModule>(new webrtc::FakeAudioDeviceModule);
mediaDependencies.audio_encoder_factory = webrtc::CreateBuiltinAudioEncoderFactory();
mediaDependencies.audio_decoder_factory = webrtc::CreateBuiltinAudioDecoderFactory();
mediaDependencies.audio_processing = webrtc::AudioProcessingBuilder().Create();
mediaDependencies.video_encoder_factory = webrtc::CreateBuiltinVideoEncoderFactory();
mediaDependencies.video_decoder_factory = webrtc::CreateBuiltinVideoDecoderFactory();
dependencies.media_engine = cricket::CreateMediaEngine(std::move(mediaDependencies));
return dependencies;
}
webrtc::PeerConnectionFactoryDependencies deps = CreatePeerConnectionFactoryDependencies();
deps.signaling_thread = signaling_thread_.get();
// deps.network_thread = network_thread.get();
// deps.worker_thread = worker_thread.get();
peer_connection_factory_ = webrtc::CreateModularPeerConnectionFactory(std::move(deps));
The call stack:
<unknown> 0x0000000001e798f7
webrtc::PeerConnection::ValidateSessionDescription(webrtc::SessionDescriptionInterface const*, cricket::ContentSource) 0x00000000005e74dc
webrtc::PeerConnection::SetLocalDescription(webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*) 0x00000000005bb677
void webrtc::ReturnType<void>::Invoke<webrtc::PeerConnectionInterface, void (webrtc::PeerConnectionInterface::*)(webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*), webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*>(webrtc::PeerConnectionInterface*, void (webrtc::PeerConnectionInterface::*)(webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*), webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*) 0x000000000059b814
webrtc::MethodCall2<webrtc::PeerConnectionInterface, void, webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*>::OnMessage(rtc::Message*) 0x0000000000598f5f
webrtc::internal::SynchronousMethodCall::Invoke(rtc::Location const&, rtc::Thread*) 0x00000000007198fc
webrtc::MethodCall2<webrtc::PeerConnectionInterface, void, webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*>::Marshal(rtc::Location const&, rtc::Thread*) 0x0000000000593706
webrtc::PeerConnectionProxyWithInternal<webrtc::PeerConnectionInterface>::SetLocalDescription(webrtc::SetSessionDescriptionObserver*, webrtc::SessionDescriptionInterface*) 0x000000000058c982
preprocessor::p2p::WebRTCManager::WebRTCManagerImpl::OnAnswerCreated webrtc_manager.cpp:226
std::__invoke_impl<void, void (*&)(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, webrtc::SessionDescriptionInterface*), preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*&, webrtc::SessionDescriptionInterface*> invoke.h:60
std::__invoke<void (*&)(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, webrtc::SessionDescriptionInterface*), preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*&, webrtc::SessionDescriptionInterface*> invoke.h:95
std::_Bind<void (*(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, std::_Placeholder<1>))(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, webrtc::SessionDescriptionInterface*)>::__call<void, webrtc::SessionDescriptionInterface*&&, 0ul, 1ul>(std::tuple<webrtc::SessionDescriptionInterface*&&>&&, std::_Index_tuple<0ul, 1ul>) functional:467
std::_Bind<void (*(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, std::_Placeholder<1>))(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, webrtc::SessionDescriptionInterface*)>::operator()<webrtc::SessionDescriptionInterface*, void>(webrtc::SessionDescriptionInterface*&&) functional:549
std::_Function_handler<void (webrtc::SessionDescriptionInterface*), std::_Bind<void (*(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, std::_Placeholder<1>))(preprocessor::p2p::WebRTCManager::WebRTCManagerImpl*, webrtc::SessionDescriptionInterface*)> >::_M_invoke(std::_Any_data const&, webrtc::SessionDescriptionInterface*&&) std_function.h:316
std::function<void (webrtc::SessionDescriptionInterface*)>::operator()(webrtc::SessionDescriptionInterface*) const std_function.h:706
preprocessor::p2p::CreateSessionDescriptionObserver::OnSuccess webrtc_manager.cpp:79
webrtc::WebRtcSessionDescriptionFactory::OnMessage(rtc::Message*) 0x0000000000b90785
rtc::MessageQueue::Dispatch(rtc::Message*) 0x00000000005712f8
rtc::Thread::ProcessMessages(int) 0x0000000000553398
rtc::Thread::Run() 0x0000000000552993
rtc::Thread::PreRun(void*) 0x0000000000552950
start_thread 0x00007ffff76536db
clone 0x00007ffff608a88f
WebRTC logs:
(audio_processing_impl.cc:435): Capture analyzer activated: 0
Capture post processor activated: 0
Render pre processor activated: 0
(webrtc_voice_engine.cc:196): WebRtcVoiceEngine::WebRtcVoiceEngine
(webrtc_video_engine.cc:479): WebRtcVideoEngine::WebRtcVideoEngine()
(webrtc_voice_engine.cc:219): WebRtcVoiceEngine::Init
(webrtc_voice_engine.cc:227): Supported send codecs in order of preference:
(webrtc_voice_engine.cc:230): opus/48000/2 { minptime=10 useinbandfec=1 } (111)
(webrtc_voice_engine.cc:230): ISAC/16000/1 (103)
(webrtc_voice_engine.cc:230): ISAC/32000/1 (104)
(webrtc_voice_engine.cc:230): G722/8000/1 (9)
(webrtc_voice_engine.cc:230): ILBC/8000/1 (102)
(webrtc_voice_engine.cc:230): PCMU/8000/1 (0)
(webrtc_voice_engine.cc:230): PCMA/8000/1 (8)
(webrtc_voice_engine.cc:230): CN/32000/1 (106)
(webrtc_voice_engine.cc:230): CN/16000/1 (105)
(webrtc_voice_engine.cc:230): CN/8000/1 (13)
(webrtc_voice_engine.cc:230): telephone-event/48000/1 (110)
(webrtc_voice_engine.cc:230): telephone-event/32000/1 (112)
(webrtc_voice_engine.cc:230): telephone-event/16000/1 (113)
(webrtc_voice_engine.cc:230): telephone-event/8000/1 (126)
(webrtc_voice_engine.cc:233): Supported recv codecs in order of preference:
(webrtc_voice_engine.cc:236): opus/48000/2 { minptime=10 useinbandfec=1 } (111)
(webrtc_voice_engine.cc:236): ISAC/16000/1 (103)
(webrtc_voice_engine.cc:236): ISAC/32000/1 (104)
(webrtc_voice_engine.cc:236): G722/8000/1 (9)
(webrtc_voice_engine.cc:236): ILBC/8000/1 (102)
(webrtc_voice_engine.cc:236): PCMU/8000/1 (0)
(webrtc_voice_engine.cc:236): PCMA/8000/1 (8)
(webrtc_voice_engine.cc:236): CN/32000/1 (106)
(webrtc_voice_engine.cc:236): CN/16000/1 (105)
(webrtc_voice_engine.cc:236): CN/8000/1 (13)
(webrtc_voice_engine.cc:236): telephone-event/48000/1 (110)
(webrtc_voice_engine.cc:236): telephone-event/32000/1 (112)
(webrtc_voice_engine.cc:236): telephone-event/16000/1 (113)
(webrtc_voice_engine.cc:236): telephone-event/8000/1 (126)
(apm_helpers.cc:32): Setting AGC mode to 0
(audio_processing_impl.cc:699): Highpass filter activated: 0
(audio_processing_impl.cc:717): Gain Controller 2 activated: 0
(audio_processing_impl.cc:719): Pre-amplifier activated: 0
(webrtc_voice_engine.cc:309): WebRtcVoiceEngine::ApplyOptions: AudioOptions {aec: 1, agc: 1, ns: 1, hf: 1, swap: 0, audio_jitter_buffer_max_packets: 200, audio_jitter_buffer_fast_accelerate: 0, audio_jitter_buffer_min_delay_ms: 0, audio_jitter_buffer_enable_rtx_handling: 0, typing: 1, experimental_agc: 0, extended_filter_aec: 0, delay_agnostic_aec: 0, experimental_ns: 0, residual_echo_detector: 1, }
(render_delay_buffer.cc:341): Applying total delay of 5 blocks.
(matched_filter.cc:450): Filter 0: start: 0 ms, end: 128 ms.
(matched_filter.cc:450): Filter 1: start: 96 ms, end: 224 ms.
(matched_filter.cc:450): Filter 2: start: 192 ms, end: 320 ms.
(matched_filter.cc:450): Filter 3: start: 288 ms, end: 416 ms.
(matched_filter.cc:450): Filter 4: start: 384 ms, end: 512 ms.
(audio_processing_impl.cc:699): Highpass filter activated: 0
(audio_processing_impl.cc:717): Gain Controller 2 activated: 0
(audio_processing_impl.cc:719): Pre-amplifier activated: 0
(apm_helpers.cc:48): Echo control set to 1 with mode 0
(audio_processing_impl.cc:699): Highpass filter activated: 0
(audio_processing_impl.cc:717): Gain Controller 2 activated: 0
(audio_processing_impl.cc:719): Pre-amplifier activated: 0
(audio_processing_impl.cc:699): Highpass filter activated: 0
(audio_processing_impl.cc:717): Gain Controller 2 activated: 0
(audio_processing_impl.cc:719): Pre-amplifier activated: 0
(apm_helpers.cc:62): NS set to 1
(webrtc_voice_engine.cc:447): Stereo swapping enabled? 0
(webrtc_voice_engine.cc:452): NetEq capacity is 200
(webrtc_voice_engine.cc:458): NetEq fast mode? 0
(webrtc_voice_engine.cc:464): NetEq minimum delay is 0
(webrtc_voice_engine.cc:470): NetEq handle reordered packets? 0
(webrtc_voice_engine.cc:481): Delay agnostic aec is enabled? 0
(webrtc_voice_engine.cc:491): Extended filter aec is enabled? 0
(webrtc_voice_engine.cc:501): Experimental ns is enabled? 0
(webrtc_voice_engine.cc:511): Setting AGC to 1
(webrtc_voice_engine.cc:533): Typing detection is enabled? 1
(audio_processing_impl.cc:699): Highpass filter activated: 1
(audio_processing_impl.cc:717): Gain Controller 2 activated: 0
(audio_processing_impl.cc:719): Pre-amplifier activated: 0
(webrtc_sdp.cc:3255): Ignored line: a=sctpmap:5000 webrtc-datachannel 65535
(rtc_event_log_impl.cc:63): Creating legacy encoder for RTC event log.
(peer_connection_factory.cc:361): Using default network controller factory
(bitrate_prober.cc:69): Bandwidth probing enabled, set to inactive
(paced_sender.cc:421): ProcessThreadAttached 0xec072e20
(cpu_info.cc:53): Available number of cores: 8
(aimd_rate_control.cc:105): Using aimd rate control with back off factor 0.85
(remote_bitrate_estimator_single_stream.cc:71): RemoteBitrateEstimatorSingleStream: Instantiating.
(remote_estimator_proxy.cc:44): Maximum interval between transport feedback RTCP messages (ms): 250
(openssl_identity.cc:44): Making key pair
(peer_connection.cc:5531): Local and Remote descriptions must be applied to get the SSL Role of the SCTP transport.
(openssl_identity.cc:92): Returning key pair
(openssl_certificate.cc:58): Making certificate for WebRTC
(openssl_certificate.cc:108): Returning certificate
(p2p_transport_channel.cc:519): Set backup connection ping interval to 25000 milliseconds.
(p2p_transport_channel.cc:528): Set ICE receiving timeout to 2500 milliseconds
(p2p_transport_channel.cc:535): Set ping most likely connection to 0
(p2p_transport_channel.cc:542): Set stable_writable_connection_ping_interval to 2500
(p2p_transport_channel.cc:555): Set presume writable when fully relayed to 0
(p2p_transport_channel.cc:564): Set regather_on_failed_networks_interval to 300000
(p2p_transport_channel.cc:583): Set receiving_switching_delay to 1000
(jsep_transport_controller.cc:1214): Creating DtlsSrtpTransport.
(dtls_srtp_transport.cc:61): Setting RTCP Transport on 0 transport 0
(dtls_srtp_transport.cc:66): Setting RTP Transport on 0 transport dc004830
(p2p_transport_channel.cc:465): Received remote ICE parameters: ufrag=YAvY, renomination disabled
(peer_connection.cc:4185): Session: 7301418690559709073 Old state: kStable New state: kHaveRemoteOffer
(peer_connection.cc:5531): Local and Remote descriptions must be applied to get the SSL Role of the SCTP transport.
(peer_connection.cc:5559): Local and Remote descriptions must be applied to get the SSL Role of the session.
(paced_sender.cc:293): Elapsed time (12680 ms) longer than expected, limiting to 2000 ms
Signal: SIGSEGV (Segmentation fault)
I guess the problem is not in callback but in the connection initialization. But what am I doing wrong?

I've found the error in my code:
peer_connection_->SetRemoteDescription(&set_session_description_observer_, desc.get());
I passed the raw pointer then release the smart one with the memory.

Related

SCSIControllerDriverKit: Process gets stuck on UserCreateTargetForID

Context:
We are working on migration of the driver, which is currently represented as a kernel extension, to the DriverKit framework.
The driver works with Thunderbolt RAID storage devices.
When connected through the Thunderbolt interface to the host, the device itself presents in the OS as a PCI device. The main function of our driver (.kext) is to create a "virtual" SCSI device in the OS for each virtual RAID array. So that the OS can work with these SCSI drives as usual disk storage.
We use https://developer.apple.com/documentation/scsicontrollerdriverkit to migrate this functionality in the dext version of the driver.
Current issue:
When a device is connected - the dext driver cannot create a SCSI drive in the OS.
Technically our dext tries to create a SCSI drive using the UserCreateTargetForID() method.
On this step the OS sends the first SCSI command "Test Unit Ready" to the device to check if it is a SCSI device.
We process this command in an additional thread separated from the main process of the dext (as it is recommended in the DriverKit documentation).
We can see in the logs that the device receives this command and responses but when our dext sends this response to the OS the process is stuck in the waiting mode. How can we understand why it happens and fix it?
More details:
We are migrating functionality of an already existing “.kext” driver. We checked logs of the kext driver of this step:
15:06:17.902539+0700 Target device try to create for idx:0
15:06:17.902704+0700 Send command 0 for target 0 len 0
15:06:18.161777+0700 Complete command: 0 for target: 0 Len: 0 status: 0 flags: 0
15:06:18.161884+0700 Send command 18 for target 0 len 6
15:06:18.161956+0700 Complete command: 18 for target: 0 Len: 6 status: 0 flags: 0
15:06:18.162010+0700 Send command 18 for target 0 len 44
15:06:18.172972+0700 Complete command: 18 for target: 0 Len: 44 status: 0 flags: 0
15:06:18.275501+0700 Send command 18 for target 0 len 36
15:06:18.275584+0700 Complete command: 18 for target: 0 Len: 36 status: 0 flags: 0
15:06:18.276257+0700 Target device created for idx:0
We can see a successful message “Target device created for idx:0”
In the the dext logs of the same step:
We do not see the “Send command 18 for target 0 len 6” as we have in the kext logs
no log of the successful result “Target device created for idx:0”
I'll add a thread name to each line of the dext log (CustomThread,DefaultQueue,SendCommandCustomThread,InterruptQueue):
15:54:10.903466+0700 Try to create target for 0 UUID 432421434863538456 - CustomThread
15:54:10.903633+0700 UserDoesHBAPerformAutoSense - DefaultQueue
15:54:10.903763+0700 UserInitializeTargetForID - DefaultQueue
15:54:10.903876+0700 UserDoesHBASupportMultiPathing DefaultQueue
15:54:10.904200+0700 UserProcessParallelTask start - DefaultQueue
15:54:10.904298+0700 Sent command : 0 len 0 for target 0 - SendCommandCustomThread
15:54:11.163003+0700 Disable interrupts - InterruptQueue
15:54:11.163077+0700 Complete cmd : 0 for target: 0 len: 0 status: 0 flags: 0 - InterruptQueue
15:54:11.163085+0700 Enable interrupts - InterruptQueue
Code for complete task
SCSIUserParallelResponse osRsp = {0};
osRsp.fControllerTaskIdentifier = osTask->taskId;
osRsp.fTargetID = osTask->targetId;
osRsp.fServiceResponse = kSCSIServiceResponse_TASK_COMPLETE;
osRsp.fCompletionStatus = (SCSITaskStatus) response->status;
// Transfer length computation.
osRsp.fBytesTransferred = transferLength; // === 0 for this case.
ParallelTaskCompletion(osTask->action, osRsp);
osTask->action->release();
Will appreciate any help
This is effectively a deadlock, which you seem to have already worked out. It's not 100% clear from your your question, but as I initially had the same problem, I assume you're calling UserCreateTargetForID from the driver's default queue. This won't work, you must call it from a non-default queue because SCSIControllerDriverKit assumes that your default queue is idle and ready to handle requests from the kernel while you are calling this function. The header docs are very ambiguous on this, though they do mention it:
The dext class should call this method to create a new target for the
targetID. The framework ensures that the new target is created before the call returns.
Note that this call to the framework runs on the Auxiliary queue.
SCSIControllerDriverKit expects your driver to use 3 different dispatch queues (default, auxiliary, and interrupt), although I think it can be done with 2 as well. I recommend you (re-)watch the relevant part of the WWDC2020 session video about how Apple wants you to use the 3 dispatch queues, exactly. The framework does not seem to be very flexible on this point.
Good luck with the rest of the driver port, I found this DriverKit framework even more fussy than the other ones.
Thanks to pmdj for direction of think. For my case answer is just add initialization for version field for response.
osRsp.version = kScsiUserParallelTaskResponseCurrentVersion1;
It looks obvious. But there are no any information in docs or WWDC2020 video about initialization version field.
My project is hardware raid 'user space driver' . My driver has now completed the io stress test. Your problem should be in the SCSI command with data transfer. And you want to send data to the system by your software driver to complete the SCSI ' inquiry ' command. I think you also used 'UserGetDataBuffer'. It seems to be some distance from iokit's function.
kern_return_t IMPL ( XXXXUserSpaceDriver, UserProcessParallelTask )
{
/*
**********************************************************************
** UserGetDataBuffer
**********************************************************************
*/
if(parallelTask.fCommandDescriptorBlock[0] == SCSI_CMD_INQUIRY)
{
IOBufferMemoryDescriptor *data_buffer_memory_descriptor = nullptr;
/*
******************************************************************************************************************************************
** virtual kern_return_t UserGetDataBuffer(SCSIDeviceIdentifier fTargetID, uint64_t fControllerTaskIdentifier, IOBufferMemoryDescriptor **buffer);
******************************************************************************************************************************************
*/
if((UserGetDataBuffer(parallelTask.fTargetID, parallelTask.fControllerTaskIdentifier, &data_buffer_memory_descriptor) == kIOReturnSuccess) && (data_buffer_memory_descriptor != NULL))
{
IOAddressSegment data_buffer_virtual_address_segment = {0};
if(data_buffer_memory_descriptor->GetAddressRange(&data_buffer_virtual_address_segment) == kIOReturnSuccess)
{
IOAddressSegment data_buffer_physical_address_segment = {0};
IODMACommandSpecification dmaSpecification;
IODMACommand *data_buffer_iodmacommand = {0};
bzero(&dmaSpecification, sizeof(dmaSpecification));
dmaSpecification.options = kIODMACommandSpecificationNoOptions;
dmaSpecification.maxAddressBits = 64;
if(IODMACommand::Create(ivars->pciDevice, kIODMACommandCreateNoOptions, &dmaSpecification, &data_buffer_iodmacommand) == kIOReturnSuccess)
{
uint64_t dmaFlags = kIOMemoryDirectionInOut;
uint32_t dmaSegmentCount = 1;
pCCB->data_buffer_iodmacommand = data_buffer_iodmacommand;
if(data_buffer_iodmacommand->PrepareForDMA(kIODMACommandPrepareForDMANoOptions, data_buffer_memory_descriptor, 0/*offset*/, parallelTask.fRequestedTransferCount/*length*/, &dmaFlags, &dmaSegmentCount, &data_buffer_physical_address_segment) == kIOReturnSuccess)
{
parallelTask.fBufferIOVMAddr = (uint64_t)data_buffer_physical_address_segment.address; /* data_buffer_physical_address: overwrite original fBufferIOVMAddr */
pCCB->OSDataBuffer = reinterpret_cast <uint8_t *> (data_buffer_virtual_address_segment.address);/* data_buffer_virtual_address */
}
}
}
}
}
}
response.fBytesTransferred = dataxferlen;
response.version = kScsiUserParallelTaskResponseCurrentVersion1;
response.fTargetID = TARGETLUN2SCSITARGET(TargetID, 0);
response.fControllerTaskIdentifier = pCCB->fControllerTaskIdentifier;
response.fCompletionStatus = taskStatus;
response.fServiceResponse = serviceResponse;
response.fSenseLength = taskStatus;
IOUserSCSIParallelInterfaceController::ParallelTaskCompletion(pCCB->completion, response);
pCCB->completion->release();
pCCB->completion = NULL;
pCCB->ccb_flags.start = 0;/*reset startdone for outstanding ccb check*/
if(pCCB->data_buffer_iodmacommand != NULL)
{
pCCB->data_buffer_iodmacommand->CompleteDMA(kIODMACommandCompleteDMANoOptions);
OSSafeReleaseNULL(pCCB->data_buffer_iodmacommand); // pCCB->data_buffer_iodmacommand->free(); pCCB->data_buffer_iodmacommand = NULL;
pCCB->OSDataBuffer = NULL;
}

"queueproxy" : error reverse proxying request; sockstat: sockets TCP: inuse 27 orphan 2 tw 20 alloc 593 mem 52

I've issue with respect to above subject, using
knative v1.2.5
istio 1.12.7
Every 20mins we see below error in the queue proxy,
error: "context canceled"
knative.dev/key: "test-common-service/test-app-0-0-0"
knative.dev/pod: "test-app-0-0-0-deployment-xxxxxxx-xxxxx"
logger: "queueproxy"
message: "error reverse proxying request; sockstat: sockets: used 44
TCP: inuse 27 orphan 2 tw 20 alloc 593 mem 52
UDP: inuse 0 mem 3
UDPLITE: inuse 0
RAW: inuse 0
FRAG: inuse 0 memory 0
Can someone please let me know how can I fix this.
Thank you!
It looks like you may have requests which are hitting request timeouts in the queue proxy. What is your typical request latency, and what is the Revision timeoutSeconds set to?
It's also possible that istio is cancelling (resetting) some of the TCP connections to the queue-proxy, but that seems unlikely.

Condor master node and workers only see the master node

I am trying to set a HTCondor batch system, but when I do condor_status it only shows the master in both the master and worker nodes. They both show this:
Name OpSys Arch State Activity LoadAv Mem
[master ip] LINUX X86_64 Unclaimed Idle 0.000 973
Total Owner Claimed Unclaimed Matched Preempting Backfill Drain
X86_64/LINUX 1 0 0 1 0 0 0 0
Total 1 0 0 1 0 0 0 0
Condor_restart on the master node works fine, but on the worker nodes yields this error:
ERROR
SECMAN:2010:Received "DENIED" from server for user unauthenticated#unmapped using no authentication method, which may imply host-based security. Our address was '[ip address of master]', and server's address was '[ip address of worker]'. Check your ALLOW settings and IP protocols.
Here are the config files:
of the master node:
CONDOR_HOST = [private ip of master]
DAEMON_LIST = COLLECTOR, MASTER, NEGOTIATOR, SCHEDD, STARTD
# to avoid user authentication
HOSTALLOW_READ = *
HOSTALLOW_WRITE = *
HOSTALLOW_ADMINISTRATOR = *
of the worker node:
CONDOR_HOST = [private ip of master]
DAEMON_LIST = MASTER, STARTD
# to avoid user authentication
HOSTALLOW_READ = *
HOSTALLOW_WRITE = *
HOSTALLOW_ADMINISTRATOR = *
I am allowing on the same security group:
All TCP TCP 0 - 65535
All ICMP-IPv4 All
SSH on port 22
This is how it looks like (security group ending in '6')
Apparently the issue was running condor_reconfig -full. I just reinstalled it without doing that and using systemctl restart condor instead and it worked. If someone wants to bring some insight on why it was so please do so :)

VirtualBox idle Windows guest high CPU usage on host

VirtualBox windows 8.1 image is created on i7 system and performing perfectly (linux and windows host). When image is moved on Xeon system ~300% linux host CPU is used with the idle guest.
VBoxManage showvminfo vm
Name: vm
Groups: /
Guest OS: Windows 8.1 (64-bit)
UUID: fdb2debf-31ff-4867-8c53-a315d12d348b
Config file: /root/VirtualBox VMs/vm/vm.vbox
Snapshot folder: /root/VirtualBox VMs/vm/Snapshots
Log folder: /root/VirtualBox VMs/vm/Logs
Hardware UUID: fdb2debf-31ff-4867-8c53-a315d12d348b
Memory size 8000MB
Page Fusion: disabled
VRAM size: 128MB
CPU exec cap: 100%
HPET: disabled
CPUProfile: host
Chipset: piix3
Firmware: BIOS
Number of CPUs: 32
PAE: enabled
Long Mode: enabled
Triple Fault Reset: disabled
APIC: enabled
X2APIC: disabled
Nested VT-x/AMD-V: disabled
CPUID Portability Level: 0
CPUID overrides: None
Boot menu mode: disabled
Boot Device 1: HardDisk
Boot Device 2: Not Assigned
Boot Device 3: Not Assigned
Boot Device 4: Not Assigned
ACPI: enabled
IOAPIC: enabled
BIOS APIC mode: APIC
Time offset: 0ms
RTC: local time
Hardw. virt.ext: enabled
Nested Paging: enabled
Large Pages: enabled
VT-x VPID: enabled
VT-x unr. exec.: enabled
Paravirt. Provider: Default
Effective Paravirt. Prov.: HyperV
State: running (since 2019-05-03T04:26:59.439000000)
Monitor count: 1
3D Acceleration: disabled
2D Video Acceleration: disabled
Teleporter Enabled: disabled
Teleporter Port: 0
Teleporter Address:
Teleporter Password:
Tracing Enabled: disabled
Allow Tracing to Access VM: disabled
Tracing Configuration:
Autostart Enabled: disabled
Autostart Delay: 0
Default Frontend:
Storage Controller Name (0): cont
Storage Controller Type (0): IntelAhci
Storage Controller Instance Number (0): 0
Storage Controller Max Port Count (0): 30
Storage Controller Port Count (0): 30
Storage Controller Bootable (0): on
cont (0, 0): /root/VirtualBox VMs/vm/Snapshots/{eec50ba7-9d02-4c51-9676-4ed0a761c6ce}.vdi (UUID: eec50ba7-9d02-4c51-9676-4ed0a761c6ce)
NIC 1: MAC: 080027F44A7A, Attachment: NAT, Cable connected: on, Trace: off (file: none), Type: 82545EM, Reported speed: 0 Mbps, Boot priority: 0, Promisc Policy: deny, Bandwidth group: none
NIC 1 Settings: MTU: 0, Socket (send: 64, receive: 64), TCP Window (send:64, receive: 64)
NIC 1 Rule(0): name = http, protocol = tcp, host ip = , host port = 8000, guest ip = , guest port = 80
NIC 1 Rule(1): name = rdp, protocol = tcp, host ip = , host port = 3389, guest ip = , guest port = 3389
NIC 2: disabled
NIC 3: disabled
NIC 4: disabled
NIC 5: disabled
NIC 6: disabled
NIC 7: disabled
NIC 8: disabled
Pointing Device: PS/2 Mouse
Keyboard Device: PS/2 Keyboard
UART 1: disabled
UART 2: disabled
UART 3: disabled
UART 4: disabled
LPT 1: disabled
LPT 2: disabled
Audio: disabled
Audio playback: disabled
Audio capture: disabled
Clipboard Mode: disabled
Drag and drop Mode: disabled
Session name: headless
Video mode: 2217x1272x32 at 0,0 enabled
VRDE: enabled (Address 0.0.0.0, Ports 6666, MultiConn: off, ReuseSingleConn: on, Authentication type: external)
VRDE port: 6666
Video redirection: disabled
VRDE property : TCP/Ports = "6666"
VRDE property : TCP/Address = <not set>
VRDE property : VideoChannel/Enabled = <not set>
VRDE property : VideoChannel/Quality = <not set>
VRDE property : VideoChannel/DownscaleProtection = <not set>
VRDE property : Client/DisableDisplay = <not set>
VRDE property : Client/DisableInput = <not set>
VRDE property : Client/DisableAudio = <not set>
VRDE property : Client/DisableUSB = <not set>
VRDE property : Client/DisableClipboard = <not set>
VRDE property : Client/DisableUpstreamAudio = <not set>
VRDE property : Client/DisableRDPDR = <not set>
VRDE property : H3DRedirect/Enabled = <not set>
VRDE property : Security/Method = <not set>
VRDE property : Security/ServerCertificate = <not set>
VRDE property : Security/ServerPrivateKey = <not set>
VRDE property : Security/CACertificate = <not set>
VRDE property : Audio/RateCorrectionMode = <not set>
VRDE property : Audio/LogPath = <not set>
OHCI USB: disabled
EHCI USB: disabled
xHCI USB: disabled
USB Device Filters:
<none>
Available remote USB devices:
<none>
Currently Attached USB Devices:
<none>
Bandwidth groups: <none>
Shared folders:<none>
VRDE Connection: active
Clients so far: 3
Start time: 2019/05/03 04:27:05 UTC
Sent: 24421625Bytes
Average speed: 26504B/s
Sent total: 24421625Bytes
Received: 270108Bytes
Speed: 293B/s
Received total: 270108Bytes
User name: admin
Domain:
Client name: dev
Client IP: 192.168.0.59
Client version: 2600
Encryption: RDP4
Capturing: not active
Capture audio: not active
Capture screens:
Capture file: /root/VirtualBox VMs/vm/vm.webm
Capture dimensions: 1024x768
Capture rate: 512kbps
Capture FPS: 25kbps
Capture options:
Guest:
Configured memory balloon size: 0MB
OS type: Windows81_64
Additions run level: 3
Additions version 6.0.6 r130049
Guest Facilities:
Facility "VirtualBox Base Driver": active/running (last update: 2019/05/03 04:27:11 UTC)
Facility "VirtualBox System Service": active/running (last update: 2019/05/03 04:27:24 UTC)
Facility "VirtualBox Desktop Integration": active/running (last update: 2019/05/03 04:27:53 UTC)
Facility "Seamless Mode": active/running (last update: 2019/05/03 04:27:11 UTC)
Facility "Graphics Mode": active/running (last update: 2019/05/03 04:27:11 UTC)
Solution is counter-intuitive, limit guest OS run on one (1) core. It brings extremely fast OS boot times and in my case best processing performance. All this has something to do with CPU context switching and optimizations at OS or hardware levels.
Didn't figured out what exactly makes VBox perform this way but my guess is that something related to CPU and not limited to i7 / Xeon differences.
Found this usefull:
https://www.reddit.com/r/linux/comments/1tqlsz/adding_cpus_to_virtualbox_guests_makes_guests/
In my case it was needed to increase cpu count from 1 to 2 in the virtual machine settings (the host cpu load was constantly 25% before and became 1% after).

gi._glib.GError: no element "pocketsphinx" error on running livedemo

I am using ubuntu 14.04
I am trying to get a python program to get speech to text from microphone.
For this, I have installed sphinxbase and pocketsphinx. pocketsphinx_continuous works.
thekindlyone#deepthought:.../lib$ pocketsphinx_continuous -inmic yes
INFO: cmd_ln.c(691): Parsing command line:
pocketsphinx_continuous \
-inmic yes
Current configuration:
[NAME] [DEFLT] [VALUE]
-adcdev
-agc none none
-agcthresh 2.0 2.000000e+00
-alpha 0.97 9.700000e-01
-argfile
-ascale 20.0 2.000000e+01
-aw 1 1
-backtrace no no
-beam 1e-48 1.000000e-48
-bestpath yes yes
-bestpathlw 9.5 9.500000e+00
-bghist no no
-ceplen 13 13
-cmn current current
-cmninit 8.0 8.0
-compallsen no no
-debug 0
-dict
-dictcase no no
-dither no no
-doublebw no no
-ds 1 1
-fdict
-feat 1s_c_d_dd 1s_c_d_dd
-featparams
-fillprob 1e-8 1.000000e-08
-frate 100 100
-fsg
-fsgusealtpron yes yes
-fsgusefiller yes yes
-fwdflat yes yes
-fwdflatbeam 1e-64 1.000000e-64
-fwdflatefwid 4 4
-fwdflatlw 8.5 8.500000e+00
-fwdflatsfwin 25 25
-fwdflatwbeam 7e-29 7.000000e-29
-fwdtree yes yes
-hmm
-infile
-input_endian little little
-jsgf
-kdmaxbbi -1 -1
-kdmaxdepth 0 0
-kdtree
-latsize 5000 5000
-lda
-ldadim 0 0
-lextreedump 0 0
-lifter 0 0
-lm
-lmctl
-lmname default default
-logbase 1.0001 1.000100e+00
-logfn
-logspec no no
-lowerf 133.33334 1.333333e+02
-lpbeam 1e-40 1.000000e-40
-lponlybeam 7e-29 7.000000e-29
-lw 6.5 6.500000e+00
-maxhmmpf -1 -1
-maxnewoov 20 20
-maxwpf -1 -1
-mdef
-mean
-mfclogdir
-min_endfr 0 0
-mixw
-mixwfloor 0.0000001 1.000000e-07
-mllr
-mmap yes yes
-ncep 13 13
-nfft 512 512
-nfilt 40 40
-nwpen 1.0 1.000000e+00
-pbeam 1e-48 1.000000e-48
-pip 1.0 1.000000e+00
-pl_beam 1e-10 1.000000e-10
-pl_pbeam 1e-5 1.000000e-05
-pl_window 0 0
-rawlogdir
-remove_dc no no
-round_filters yes yes
-samprate 16000 1.600000e+04
-seed -1 -1
-sendump
-senlogdir
-senmgau
-silprob 0.005 5.000000e-03
-smoothspec no no
-svspec
-time no no
-tmat
-tmatfloor 0.0001 1.000000e-04
-topn 4 4
-topn_beam 0 0
-toprule
-transform legacy legacy
-unit_area yes yes
-upperf 6855.4976 6.855498e+03
-usewdphones no no
-uw 1.0 1.000000e+00
-var
-varfloor 0.0001 1.000000e-04
-varnorm no no
-verbose no no
-warp_params
-warp_type inverse_linear inverse_linear
-wbeam 7e-29 7.000000e-29
-wip 0.65 6.500000e-01
-wlen 0.025625 2.562500e-02
INFO: cmd_ln.c(691): Parsing command line:
\
-nfilt 20 \
-lowerf 1 \
-upperf 4000 \
-wlen 0.025 \
-transform dct \
-round_filters no \
-remove_dc yes \
-svspec 0-12/13-25/26-38 \
-feat 1s_c_d_dd \
-agc none \
-cmn current \
-cmninit 56,-3,1 \
-varnorm no
Current configuration:
[NAME] [DEFLT] [VALUE]
-agc none none
-agcthresh 2.0 2.000000e+00
-alpha 0.97 9.700000e-01
-ceplen 13 13
-cmn current current
-cmninit 8.0 56,-3,1
-dither no no
-doublebw no no
-feat 1s_c_d_dd 1s_c_d_dd
-frate 100 100
-input_endian little little
-lda
-ldadim 0 0
-lifter 0 0
-logspec no no
-lowerf 133.33334 1.000000e+00
-ncep 13 13
-nfft 512 512
-nfilt 40 20
-remove_dc no yes
-round_filters yes no
-samprate 16000 1.600000e+04
-seed -1 -1
-smoothspec no no
-svspec 0-12/13-25/26-38
-transform legacy dct
-unit_area yes yes
-upperf 6855.4976 4.000000e+03
-varnorm no no
-verbose no no
-warp_params
-warp_type inverse_linear inverse_linear
-wlen 0.025625 2.500000e-02
INFO: acmod.c(246): Parsed model-specific feature parameters from /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/feat.params
INFO: feat.c(713): Initializing feature stream to type: '1s_c_d_dd', ceplen=13, CMN='current', VARNORM='no', AGC='none'
INFO: cmn.c(142): mean[0]= 12.00, mean[1..12]= 0.0
INFO: acmod.c(167): Using subvector specification 0-12/13-25/26-38
INFO: mdef.c(517): Reading model definition: /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/mdef
INFO: mdef.c(528): Found byte-order mark BMDF, assuming this is a binary mdef file
INFO: bin_mdef.c(336): Reading binary model definition: /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/mdef
INFO: bin_mdef.c(513): 50 CI-phone, 143047 CD-phone, 3 emitstate/phone, 150 CI-sen, 5150 Sen, 27135 Sen-Seq
INFO: tmat.c(205): Reading HMM transition probability matrices: /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/transition_matrices
INFO: acmod.c(121): Attempting to use SCHMM computation module
INFO: ms_gauden.c(198): Reading mixture gaussian parameter: /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/means
INFO: ms_gauden.c(292): 1 codebook, 3 feature, size:
INFO: ms_gauden.c(294): 256x13
INFO: ms_gauden.c(294): 256x13
INFO: ms_gauden.c(294): 256x13
INFO: ms_gauden.c(198): Reading mixture gaussian parameter: /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/variances
INFO: ms_gauden.c(292): 1 codebook, 3 feature, size:
INFO: ms_gauden.c(294): 256x13
INFO: ms_gauden.c(294): 256x13
INFO: ms_gauden.c(294): 256x13
INFO: ms_gauden.c(354): 0 variance values floored
INFO: s2_semi_mgau.c(903): Loading senones from dump file /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/sendump
INFO: s2_semi_mgau.c(927): BEGIN FILE FORMAT DESCRIPTION
INFO: s2_semi_mgau.c(1022): Using memory-mapped I/O for senones
INFO: s2_semi_mgau.c(1296): Maximum top-N: 4 Top-N beams: 0 0 0
INFO: dict.c(317): Allocating 137543 * 32 bytes (4298 KiB) for word entries
INFO: dict.c(332): Reading main dictionary: /usr/share/pocketsphinx/model/lm/en_US/cmu07a.dic
INFO: dict.c(211): Allocated 1010 KiB for strings, 1664 KiB for phones
INFO: dict.c(335): 133436 words read
INFO: dict.c(341): Reading filler dictionary: /usr/share/pocketsphinx/model/hmm/en_US/hub4wsj_sc_8k/noisedict
INFO: dict.c(211): Allocated 0 KiB for strings, 0 KiB for phones
INFO: dict.c(344): 11 words read
INFO: dict2pid.c(396): Building PID tables for dictionary
INFO: dict2pid.c(404): Allocating 50^3 * 2 bytes (244 KiB) for word-initial triphones
INFO: dict2pid.c(131): Allocated 60400 bytes (58 KiB) for word-final triphones
INFO: dict2pid.c(195): Allocated 60400 bytes (58 KiB) for single-phone word triphones
INFO: ngram_model_arpa.c(77): No \data\ mark in LM file
INFO: ngram_model_dmp.c(142): Will use memory-mapped I/O for LM file
INFO: ngram_model_dmp.c(196): ngrams 1=5001, 2=436879, 3=418286
INFO: ngram_model_dmp.c(242): 5001 = LM.unigrams(+trailer) read
INFO: ngram_model_dmp.c(288): 436879 = LM.bigrams(+trailer) read
INFO: ngram_model_dmp.c(314): 418286 = LM.trigrams read
INFO: ngram_model_dmp.c(339): 37293 = LM.prob2 entries read
INFO: ngram_model_dmp.c(359): 14370 = LM.bo_wt2 entries read
INFO: ngram_model_dmp.c(379): 36094 = LM.prob3 entries read
INFO: ngram_model_dmp.c(407): 854 = LM.tseg_base entries read
INFO: ngram_model_dmp.c(463): 5001 = ascii word strings read
INFO: ngram_search_fwdtree.c(99): 788 unique initial diphones
INFO: ngram_search_fwdtree.c(147): 0 root, 0 non-root channels, 60 single-phone words
INFO: ngram_search_fwdtree.c(186): Creating search tree
INFO: ngram_search_fwdtree.c(191): before: 0 root, 0 non-root channels, 60 single-phone words
INFO: ngram_search_fwdtree.c(326): after: max nonroot chan increased to 13428
INFO: ngram_search_fwdtree.c(338): after: 457 root, 13300 non-root channels, 26 single-phone words
INFO: ngram_search_fwdflat.c(156): fwdflat: min_ef_width = 4, max_sf_win = 25
INFO: continuous.c(371): pocketsphinx_continuous COMPILED ON: Dec 22 2013, AT: 20:43:21
Then I ran livedemo.py from pocketsphinx/src/gst-plugin This is the error I get:
thekindlyone#deepthought:~/.../gst-plugin$ python livedemo.py
Using pygtkcompat and Gst from gi
Traceback (most recent call last):
File "livedemo.py", line 102, in <module>
app = DemoApp()
File "livedemo.py", line 31, in __init__
self.init_gst()
File "livedemo.py", line 53, in init_gst
+ '! pocketsphinx configured=true ! fakesink')
gi._glib.GError: no element "pocketsphinx"
thekindlyone#deepthought:~/.../gst-plugin$
I found that I have export a new path as per cmusphinx wiki. But /usr/local/lib/gstreamer-1.0 is not present. What should I do next?
output of gst-inspect-1.0 pocketsphinx
No such element or plugin 'pocketsphinx'
output of gst-inspect pocketsphinx
Factory Details:
Long name: PocketSphinx
Class: Filter/Audio
Description: Convert speech to text
Author(s): David Huggins-Daines <dhuggins#cs.cmu.edu>
Rank: none (0)
Plugin Details:
Name: pocketsphinx
Description: PocketSphinx plugin
Filename: /usr/lib/gstreamer-0.10/libgstpocketsphinx.so
Version: 0.8
License: BSD
Source module: pocketsphinx
Binary package: PocketSphinx
Origin URL: http://cmusphinx.sourceforge.net/
GObject
+----GstObject
+----GstElement
+----GstPocketSphinx
Pad Templates:
SINK template: 'sink'
Availability: Always
Capabilities:
audio/x-raw-int
width: 16
depth: 16
signed: true
endianness: 1234
channels: 1
rate: 8000
SRC template: 'src'
Availability: Always
Capabilities:
text/plain
Element Flags:
no flags set
Element Implementation:
Has change_state() function: gst_element_change_state_func
Has custom save_thyself() function: gst_element_save_thyself
Has custom restore_thyself() function: gst_element_restore_thyself
Element has no clocking capabilities.
Element has no indexing capabilities.
Element has no URI handling capabilities.
Pads:
SRC: 'src'
Implementation:
Has custom eventfunc(): gst_pad_event_default
Has custom queryfunc(): gst_pad_query_default
Has custom iterintlinkfunc(): gst_pad_iterate_internal_links_default
Has getcapsfunc(): gst_pad_get_fixed_caps_func
Has acceptcapsfunc(): gst_pad_acceptcaps_default
Pad Template: 'src'
SINK: 'sink'
Implementation:
Has chainfunc(): 0x7f4e0c00c4f0
Has custom eventfunc(): 0x7f4e0c00c1b0
Has custom queryfunc(): gst_pad_query_default
Has custom iterintlinkfunc(): gst_pad_iterate_internal_links_default
Has getcapsfunc(): gst_pad_get_fixed_caps_func
Has acceptcapsfunc(): gst_pad_acceptcaps_default
Pad Template: 'sink'
Element Properties:
name : The name of the object
flags: readable, writable
String. Default: "pocketsphinx0"
hmm : Directory containing acoustic model parameters
flags: readable, writable
String. Default: null
lm : Language model file
flags: readable, writable
String. Default: null
lmctl : Language model control file (for class LMs)
flags: readable, writable
String. Default: null
lmname : Language model name (to select LMs from lmctl)
flags: readable, writable
String. Default: "default"
dict : Dictionary File
flags: readable, writable
String. Default: null
mllr : MLLR file
flags: readable, writable
String. Default: null
fsg : Finite state grammar file
flags: readable, writable
String. Default: null
fsg-model : Finite state grammar object (fsg_model_t *)
flags: writable
Pointer. Write only
fwdflat : Enable Flat Lexicon Search
flags: readable, writable
Boolean. Default: false
bestpath : Enable Graph Search
flags: readable, writable
Boolean. Default: false
maxhmmpf : Maximum number of HMMs searched per frame
flags: readable, writable
Integer. Range: 1 - 100000 Default: 2000
maxwpf : Maximum number of words searched per frame
flags: readable, writable
Integer. Range: 1 - 100000 Default: 20
beam : Beam width applied to every frame in Viterbi search
flags: readable, writable
Float. Range: -1 - 1 Default: 0
wbeam : Beam width applied to phone transitions
flags: readable, writable
Float. Range: -1 - 1 Default: 0
pbeam : Beam width applied to phone transitions
flags: readable, writable
Float. Range: -1 - 1 Default: 0
dsratio : Evaluate acoustic model every N frames
flags: readable, writable
Integer. Range: 1 - 10 Default: 1
latdir : Output Directory for Lattices
flags: readable, writable
String. Default: null
lattice : Word lattice object for most recent result
flags: readable
Boxed pointer of type "PSLattice"
nbest : N-best results
flags: readable
Array of GValues of type "gchararray"
nbest-size : Number of hypothesis in the N-best list
flags: readable, writable
Integer. Range: 1 - 1000 Default: 10
decoder : The underlying decoder
flags: readable
Boxed pointer of type "PSDecoder"
configured : Set this to finalize configuration
flags: readable, writable
Boolean. Default: false
Element Signals:
"partial-result" : void user_function (GstElement* object,
gchararray arg0,
gchararray arg1,
gpointer user_data);
"result" : void user_function (GstElement* object,
gchararray arg0,
gchararray arg1,
gpointer user_data);
UPDATES:
I downloaded fresh copies from github and installed, no change.
sphinxbase build
sphinxbase install
pocketsphinx build
pocketsphinx install
5th attempt on clean install worked. /usr/local/lib/gstreamer1.0 created. Adding this to GST_PLUGIN_PATH worked.