.What to use for Improving speech-to-text recognition accuracy using pocketsphinx? - pocketsphinx-android

Question

I am totally new to using pocketsphinx, I have followed the integration of demo application as mentioned in
Android offline voice recognition using PocketSphinx
It's working fine after integration of pocketsphinx as library in my application, but the output is not as accurate as desired. It's taking words even which are not being uttered from the provided dictonary.
I want to understand, how to improve accuracy of detecting words : I had initally used a .lm file; then instead of using that, I simply created a .jsgf text file and used it, but still there was no improvement in the accuracy, So after using .jsgf file do I need to compile it or something or simply copy pasting the .jsgf text file in the assests file is enough
http://cmusphinx.sourceforge.net/wiki/tutorialandroid in this link it is given building pocketsphinx-android. I have not done this. Just integrated it as library project
The Code:
public class SphinxSpeechRecognizerActivity extends Activity implements RecognitionListener {
private static String TAG = SphinxSpeechRecognizerActivity.class.getSimpleName();
private SpeechRecognizer mRecognizer;
private HashMap<String, Integer> mCaptions;
// private static final String KWS_SEARCH = "wakeup";
// private static final String KEYPHRASE = "phone";
private static final String COMMANDS = "command";
private boolean mErrorFlag = false;
private static boolean isRecognizerInProgress = false;
#Override
public void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.fragment);
initViews();
}
#Override
public void onResume() {
super.onResume();
}
#Override
public void onPause() {
super.onPause();
}
public void onDestroy() {
super.onDestroy();
Log.d(TAG, "** onDestroy **");
stopRecgonizer(true);
}
#Override
public void onBackPressed() {
super.onBackPressed();
stopRecgonizer(true);
}
private void initViews() {
final ImageView img_close = (ImageView)findViewById(R.id.ttsClose);
final ImageView img_voice_view = (ImageView)findViewById(R.id.tts_voice_view);
final ImageView img_info = (ImageView)findViewById(R.id.ttsInfo);
img_close.setOnClickListener(mOnClickListener);
img_info.setOnClickListener(mOnClickListener);
img_voice_view.setOnClickListener(mOnClickListener);
}
// Set press indicator
private View.OnClickListener mOnClickListener = new View.OnClickListener() {
#Override
public void onClick(View v) {
switch (v.getId()){
case R.id.ttsInfo:
break;
case R.id.tts_voice_view:
if (!isRecognizerInProgress) {
isRecognizerInProgress = true;
setupRecognizerController();
} else {
Log.d(TAG, "Sphinx recognizer is already running");
}
break;
case R.id.ttsClose:
default:
// Call back event
onBackPressed();
break;
}
}
};
#Override
public void onBeginningOfSpeech() {
Log.d(TAG, "** onBeginningOfSpeech **" + mErrorFlag);
}
#Override
public void onEndOfSpeech() {
Log.d(TAG, "** onEndOfSpeech **");
mRecognizer.stop();
}
#Override
public void onPartialResult(Hypothesis hypothesis) {
Log.d(TAG, "** onPartialResult **");
if (hypothesis == null)
return;
mRecognizer.stop();
}
private void switchSearch(String languageModelSearch) {
mRecognizer.stop();
mRecognizer.startListening(languageModelSearch, 2000);
}
#Override
public void onResult(Hypothesis hypothesis) {
hideListeningBackground();
stopRecgonizer(true);
if(hypothesis != null){
final String recognizedCommand = hypothesis.getHypstr();
Log.d(TAG,"Recognized Text: = " + recognizedCommand + " Score: " + hypothesis.getBestScore());
runOnUiThread(new Runnable() {
#Override
public void run() {
if(!recognizedCommand.equals("")) {
if (recognizedCommand.equalsIgnoreCase(<given_command>)) {
Intent speech_converted_intent = new Intent(SphinxSpeechRecognizerActivity.this, Subclass.class);
startActivity(speech_converted_intent);
finish();
}
} else {
showErrorMsg(Constants.MODE_SUCCESS);
}
}
});
} else {
showErrorMsg(Constants.MODE_DEFAULT);
}
}
#Override
public void onError(Exception e) {
Log.e(TAG, "** onError **");
showErrorMsg(Constants.MODE_FAILED);
}
#Override
public void onTimeout() {
Log.i(TAG, "** onTimeout **");
mRecognizer.stop();
}
private void setupRecognizerController() {
new AsyncTask<Void, Void, Exception>() {
#Override
protected Exception doInBackground(Void... params) {
try {
Assets assets = new Assets(SphinxSpeechRecognizerActivity.this);
File assetDir = assets.syncAssets();
setupRecognizer(assetDir);
} catch (IOException e) {
return e;
}
return null;
}
#Override
protected void onPostExecute(Exception result) {
if(result == null){
Log.d(TAG, "Sphinx Recognizer: Start");
mRecognizer.startListening(COMMANDS, 3000);
}
displayListeningBackground();
}
}.execute();
}
private void setupRecognizer(File assetsDir) throws IOException {
mRecognizer = defaultSetup()
.setAcousticModel(new File(assetsDir, "en-us-ptm"))
.setDictionary(new File(assetsDir, "cmudict-en-us.dict"))
.setKeywordThreshold(1e-10f)
.setFloat("-beam", 1e-30f)
.setBoolean("-allphone_ci", true)
.getRecognizer();
mRecognizer.addListener(this);
File languageModel = new File(assetsDir, "command.gram");
mRecognizer.addGrammarSearch(COMMANDS, languageModel);
// reset();
}
private void reset(){
mRecognizer.stop();
// mRecognizer.startListening(COMMANDS);
}
private void stopRecgonizer(boolean flag){
if(flag && mRecognizer != null){
mRecognizer.cancel();
mRecognizer.shutdown();
isRecognizerInProgress = false;
}
hideListeningBackground();
}
String mShowText = "ERROR";
private void showErrorMsg(final int error_type) {
runOnUiThread(new Runnable() {
#Override
public void run() {
switch (error_type) {
case Constants.MODE_FAILED:
// ...
break;
case Constants.MODE_SUCCESS:
//...
break;
case Constants.MODE_DEFAULT:
default:
//../
break;
}
}
});
}
}
My grammar file
#JSGF V1.0;
grammar commands;
public <commands> = (<label> | <mainMenu> | <subMenu> | <track> )+;
<mainMenu> = ( music
| phone
| navigation
| vehicle
| homepage
| shortcut
);
<label> = ( back
| usb ( one | two )
| contact
| sms
| message
| dial
| ( homepage ( one | two | three ))
| ( shortcut ( one | two | three ))
);
<subMenu> = ( back
| ( next | previous ) station
| ( fm ( one | two ))
| ( dr ( one | two ))
| am
| listen
| play
| ( next | previous )
| search [ artists | playlists | songs | albums ]
| call
| received
| missed
| dial
| address
);
<track> = ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
| ten
| eleven
| twelve
| thirteen
| fourteen
| fifteen
| sixteen
| seventeen
| eighteen
| nineteen
| twenty
| (twenty ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
| thirty
| (thirty ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
| forty
| (forty ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
| fifty
| (fifty ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
| sixty
| (sixty ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
| seventy
| (seventy ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
| eighty
| (eighty ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
| ninety
| (ninety ( one
| two
| three
| four
| five
| six
| seven
| eight
| nine
)
)
);
My log shows:
I/cmusphinx: INFO: pocketsphinx.c(993): Writing raw audio log file: /storage/emulated/0/Android/data/com.techmahindra.rngo/files/sync/000000000.raw

Answer 1

Accuracy debugging is a complex process, there could be too many issues - noise in data, bad cpu speed causing delays in recording, bad estimation of channel.
In order to debug performance you first need to collect the data. Uncomment the call to setRawLogDir in demo and see in the logcat that raw data files are being stored on sdcard. Check those files to make sure audio is recorded correctly. Share the data together with the logs and your model to get help on accuracy. Make sure data is properly recorded, has no noise, has proper format, that you speak without accent.
In case you want to listen continuous and ignore words which are not of your interest, you need to use keyword spotting mode, not language model or grammar.