tools/test_g2g/test_g2g.c - platform/external/srec - Git at Google

 /*---------------------------------------------------------------------------*
  *  test_g2g.c  *
  *                                                                           *
  *  Copyright 2007, 2008 Nuance Communciations, Inc.                               *
  *                                                                           *
  *  Licensed under the Apache License, Version 2.0 (the 'License');          *
  *  you may not use this file except in compliance with the License.         *
  *                                                                           *
  *  You may obtain a copy of the License at                                  *
  *      http://www.apache.org/licenses/LICENSE-2.0                           *
  *                                                                           *
  *  Unless required by applicable law or agreed to in writing, software      *
  *  distributed under the License is distributed on an 'AS IS' BASIS,        *
  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
  *  See the License for the specific language governing permissions and      *
  *  limitations under the License.                                           *
  *                                                                           *
  *---------------------------------------------------------------------------*/


 #include "pstdio.h"
 #include "pmemory.h"
 #include "plog.h"
 #include "HashMap.h"
 #include "SR_Grammar.h"
 #include "SR_Vocabulary.h"
 #include "SR_SemanticResult.h"
 #include "ESR_Session.h"
 #include "ESR_Locale.h"
 #include "ESR_CommandLine.h"
 #include "LCHAR.h"

 #include "PFileSystem.h"
 #include "PANSIFileSystem.h"

 #include "SR_GrammarImpl.h"

 #include "simapi.h"
 #include "srec_context.h"
 #include "srec_arb.h"

 /**
  * @todo document
  */
 typedef struct
 {
   unsigned short nnodes;
   unsigned long  size;
   long    phoneme;
   unsigned short node_pos;
   unsigned long  node_off;
   short    low_genone_no;
   short    high_genone_no;
   short    low_pel_no;
   short    high_pel_no;
 }
 tree_head;


 int usage(LCHAR* exename)
 {
   pfprintf(PSTDOUT,"usage: %s -base <basefilename> \n",exename);
   pfprintf(PSTDOUT,"<basefilename> can be a file.g2g or @g2gfilelist\n");
   pfprintf(PSTDOUT,"[-checkword id] .. also checks word id in the file\n");
   pfprintf(PSTDOUT,"[-swiarb esr/config/lang/models/generic.swiarb] ... enables word check\n");
   return 1;
 }

 /* protos */
 ESR_ReturnCode find_phonemes_for_ihmms( CA_Arbdata* ca_arbdata, modelID* ihmms, int num_hmms);
 ESR_ReturnCode Parse(SR_Grammar* grammar, LCHAR* trans, PFile* fout);
 int CheckG2G(CA_Arbdata* arbdata, int* p4pTable, const char* base, int wordid, char* outbase);
 void load_filelist(char* filelist, char*** pfiles, int *pnum_files);
 int *phonemecode_for_pel_table(CA_Arbdata* arbdata);

 int debug = 0;
 #define MAX_LINE_LENGTH 256
 #define MAX_STR_LENGTH   80
 #define MAX_SEM_RESULTS   3
 #define MAX_KEYS         30

 /* main */

 int main (int argc, char **argv)
 {
   ESR_ReturnCode rc;
   LCHAR base[P_PATH_MAX] = L("");
   int i;
   CA_Arbdata* ca_arbdata;
   char*  arbfile = NULL;
   char** g2glist;
   int g2glist_len;
   char* outbase = NULL;
   int *p4pTable;
   int wordid = 0;
   int log_level = 0;

   /*
    * Initialize portable library.
    */
   CHKLOG(rc, PMemInit());
 /*  CHKLOG(rc, PFileSystemCreate());
   CHKLOG(rc, PANSIFileSystemCreate());
   CHKLOG(rc, PANSIFileSystemAddPath(L("/dev/ansi"), L("/")));*/

   /* Set ANSI file-system as default file-system */
 /*  CHKLOG(rc, PANSIFileSystemSetDefault(ESR_TRUE));*/
   /* Set virtual current working directory to native current working directory */
 /*  len = P_PATH_MAX;
   CHKLOG(rc, PANSIFileSystemGetcwd(cwd, &len));
   CHKLOG(rc, PFileSystemChdir(cwd));*/

   if( argc <= 1)
 	{
     usage(argv[0]);
     exit(EXIT_FAILURE);
   }

 	for (i = 1; i < argc; ++i)
 	{
 		if(!LSTRCMP(argv[i], L("-base")))
 		{
 			++i;
 			LSTRCPY(base, argv[i]);
 		}
 		else if(!LSTRCMP(argv[i],L("-out")))
 		{
 			outbase = argv[++i];
 		}
 		else if(!LSTRCMP(argv[i],L("-swiarb")))
 		{
 			arbfile = argv[++i];
 		}
 		else if(!LSTRCMP(argv[i],L("-checkword")))
 		{
 			wordid = atoi(argv[++i]);
 		}
 		else if(!LSTRCMP(argv[i],L("-log")))
 		{
 			log_level = 10;
 		}
 		else
 		{
 			printf("unrecog'd argument %s\n", argv[i]);
 			exit(1);
 		}
 	}

 	CHK(rc, PLogInit(NULL, log_level));

 	if(arbfile) {
 		ca_arbdata = CA_LoadArbdata(arbfile);
 		if(!ca_arbdata) {
       pfprintf(PSTDOUT, "Error: loading arbfile %s\n", arbfile);
       goto CLEANUP;
     }
     pfprintf(PSTDOUT, "arbdata done\n");
     p4pTable  = phonemecode_for_pel_table(ca_arbdata);
     pfprintf(PSTDOUT, "p4pTable done\n");
   } else {
     ca_arbdata = 0;
     p4pTable = 0;
   }

   if(base[0] == '@') {
     load_filelist(base+1, &g2glist, &g2glist_len);
     pfprintf(PSTDOUT, "g2glist %s .. %d entries\n", g2glist_len);
     for(i=0; i<g2glist_len; i++)
       CheckG2G( ca_arbdata, p4pTable, g2glist[i], wordid, outbase);
   }
   else {
     CheckG2G( ca_arbdata, p4pTable, base, wordid, outbase);
   }

 CLEANUP:
   PLogShutdown();
 /*  PANSIFileSystemDestroy();
   PFileSystemDestroy();*/
   PMemSetLogFile(PSTDOUT);
   PMemDumpLogFile();
   PMemShutdown();
   return rc;
 }

 int CheckG2G(CA_Arbdata* ca_arbdata, int* p4pTable, const char* base, int wordid, char* outbase)
 {
   ESR_ReturnCode rc;
   SR_GrammarImpl *grammarImpl;
   SR_Grammar* grammar = NULL;
   srec_context* fst;
   CA_Syntax* syntax;
   modelID ilabels_preceding[64], num_ilabels_preceding;
   modelID ilabels_following[64], num_ilabels_following;
   modelID ilabels[128], num_ilabels;
   int i,j;
   unsigned long g2gsize;

   if(1) {
     FILE* fp;
     fp = fopen(base, "rb");
     if(!fp) g2gsize = 0;
     else {
       fseek(fp, 0, SEEK_END);
       g2gsize = ftell(fp);
       fclose(fp);
     }
   }

   rc = SR_GrammarLoad(base, &grammar);
   if(rc != ESR_SUCCESS) {
     pfprintf(PSTDOUT, "%s failed at load\n", base);
     goto CLEANUP;
   }

   grammarImpl = (SR_GrammarImpl*)grammar;
   syntax = grammarImpl->syntax;
   if(outbase) {
     CA_DumpSyntax( syntax, outbase);
   }

   fst = syntax->synx;
   pfprintf(PSTDOUT, "%s %d arcs %d/%d/%d nodes %d/%d/%d words %d/%d chars %d/%d modelver %d\n",
 	   base, g2gsize,
 		 fst->num_arcs, fst->num_base_arcs, fst->FSMarc_list_len,
 		 fst->num_nodes, fst->num_base_nodes, fst->FSMnode_list_len,
 	   fst->olabels->num_words, fst->olabels->max_words,
 	   fst->olabels->next_chars-fst->olabels->chars,
 	   fst->olabels->max_chars,
 #ifdef IMAGE_FORMAT_V2
 	   fst->modelid
 #else
 	   -1
 #endif
 	   );

   if(wordid == 0 || ca_arbdata == 0)
     goto CLEANUP;

   if(wordid >= fst->olabels->num_words) {
     pfprintf(PSTDOUT, "%s failed 'cuz numwords(%d) < %d\n", base,
 	     fst->olabels->num_words, wordid);
     goto CLEANUP;
   }

   for(i=0; i<fst->num_arcs; i++) {
     if(fst->FSMarc_list[i].olabel == wordid) {
       FSMnode* node;
       FSMarc* arc = &fst->FSMarc_list[i];
       nodeID fr_node = arc->fr_node;
       arcID iarc;
       ilabels_following[0] = arc->ilabel;
       num_ilabels_following = 1;
       num_ilabels_preceding = 0;
       for( ; fr_node!=fst->start_node; fr_node=arc->fr_node) {
 	node = &fst->FSMnode_list[fr_node];
 	iarc = node->first_prev_arc;
 	for( ; iarc!=MAXarcID; iarc=arc->linkl_prev_arc) {
 	  arc = &fst->FSMarc_list[iarc];
 	  if(arc->fr_node != fr_node) break;
 	}
 	if(iarc == MAXarcID) {
 	  pfprintf(PSTDOUT, "%s failed at 11\n", base);
 	  goto CLEANUP;
 	}
 	if(arc->ilabel == WORD_BOUNDARY) break;
 	ilabels_preceding[num_ilabels_preceding++] = arc->ilabel;
       }
       arc = &fst->FSMarc_list[i];
       fr_node = arc->to_node;
       for( ; fr_node!=fst->end_node; fr_node=arc->to_node) {
 	node = &fst->FSMnode_list[fr_node];
 	iarc = node->un_ptr.first_next_arc;
 	for( ; iarc!=MAXarcID; iarc=arc->linkl_next_arc) {
 	  arc = &fst->FSMarc_list[iarc];
 	  if(arc->to_node != fr_node) break;
 	}
 	if(iarc == MAXarcID) {
 	  pfprintf(PSTDOUT, "%s failed at 12\n", base);
 	  goto CLEANUP;
 	}
 	ilabels_following[num_ilabels_following++] = arc->ilabel;
 	if(arc->ilabel == WORD_BOUNDARY) break;
       }
       num_ilabels = 0;
       for(j=0; j<num_ilabels_preceding; j++)
 	ilabels[num_ilabels++] = ilabels_preceding[num_ilabels_preceding-1-j];
       for(j=0; j<num_ilabels_following; j++)
 	ilabels[num_ilabels++] = ilabels_following[j];
       if(ilabels[num_ilabels-1] == WORD_BOUNDARY)
 	num_ilabels--;
       for(j=0; j<num_ilabels; j++) {
 	if(ilabels[j]<fst->hmm_ilabel_offset) {
 	  pfprintf(PSTDOUT, "%s failed at 15\n", base);
 	  goto CLEANUP;
 	} else
 	  ilabels[j] = ilabels[j] - (labelID)fst->hmm_ilabel_offset;
       }
       pfprintf(PSTDOUT, "%s (W%d) ihmms ", fst->olabels->words[wordid], wordid);
       for(j=0;j<num_ilabels;j++)
 	pfprintf(PSTDOUT, " %d", ilabels[j]);
       pfprintf(PSTDOUT, "\n");
       if(num_ilabels < 2) {
 	pfprintf(PSTDOUT, "%s failed at 1\n", base);
 	goto CLEANUP;
       }
       if(p4pTable)
 	rc = find_phonemes_for_ihmms( ca_arbdata, ilabels, num_ilabels);
       else {
 	rc = ESR_SUCCESS;
 	for(j=0; j<num_ilabels; j++) {
 	  if(p4pTable[ ilabels[j]]<0) {
 	    rc = ESR_NO_MATCH_ERROR;
 	    ilabels[j] = MAXmodelID;
 	  } else {
 	    ilabels[j] = (modelID)p4pTable[ ilabels[j]];
 	  }
 	}
       }

       if(rc) {
 	pfprintf(PSTDOUT, "%s failed at 2\n", base);
 	goto CLEANUP;
       }
       pfprintf(PSTDOUT, "%s ", fst->olabels->words[wordid]);
       for(j=0;j<num_ilabels;j++) pfprintf(PSTDOUT, "%c", ilabels[j]);
       pfprintf(PSTDOUT, "\n");
       rc = Parse( grammar, fst->olabels->words[wordid], PSTDOUT);
       if(rc) {
 	pfprintf(PSTDOUT, "%s failed at 3\n", base);
 	goto CLEANUP;
       }
       pfprintf(PSTDOUT, "%s PASSED (on %s)\n", base, fst->olabels->words[wordid]);
       break;
     }
   }

   return 0;
  CLEANUP:
   if(grammar) SR_GrammarDestroy(grammar);
   return 1;

 }


 int traverse_tree(tree_node* node, tree_head *tree_topo, int *num_terminal_nodes)
 {
   if(node)
     tree_topo->nnodes++;

   if(node->node.quest_index < 0) {
     if(num_terminal_nodes)
       (*num_terminal_nodes)++;
     if( node->term.pelid < tree_topo->low_pel_no)
       tree_topo->low_pel_no = tree_topo->low_genone_no = node->term.pelid;
     if( node->term.pelid > tree_topo->high_pel_no)
       tree_topo->high_pel_no = tree_topo->high_genone_no = node->term.pelid;
   } else {
     traverse_tree( (tree_node*)node->node.fail, tree_topo, num_terminal_nodes);
     traverse_tree( (tree_node*)node->node.pass, tree_topo, num_terminal_nodes);
   }
   return 0;

 }

 int num_nodes_in_tree(tree_node* node, int *num_terminal_nodes)
 {
   tree_head topo;
   *num_terminal_nodes = 0;
   topo.nnodes = 0;
   traverse_tree(node, &topo, num_terminal_nodes);
   return topo.nnodes;
 }

 ESR_ReturnCode find_phonemes_for_ihmms( CA_Arbdata* ca_arbdata, modelID* ihmms, int num_ihmms)
 {
   int ii, i;
   int num_hmms_in_phoneme;
   tree_head topo;
   srec_arbdata* a = (srec_arbdata*)ca_arbdata;
   int num_phonemes_for_ihmms = 0;

   for(ii=0; ii<num_ihmms; ii++) {
     for(i=0; i<a->num_phonemes; i++) {
       num_hmms_in_phoneme = 0;
       topo.low_pel_no  = 32567;
       topo.high_pel_no = 0;
       traverse_tree(a->pdata[i].model_nodes, &topo, &num_hmms_in_phoneme);
       if(debug)printf("phoneme %d num_hmms %d (%d-%d)\n", i, num_hmms_in_phoneme,
 		      topo.low_pel_no, topo.high_pel_no);
       if(ihmms[ii] >= topo.low_pel_no && ihmms[ii]<= topo.high_pel_no) {
 	ihmms[ii] = (modelID)i;
 	num_phonemes_for_ihmms++;
 	break;
       }
     }
     if( i==a->num_phonemes) {
       if(ihmms[ii]<=5) {
 	ihmms[ii] = 0;
 	num_phonemes_for_ihmms++;
       } else {
 	PLogError("error: could not find hmm%d under any phoneme! ",ihmms[ii]);
       }
     }

   }
   if(num_phonemes_for_ihmms != num_ihmms)
     return ESR_INVALID_ARGUMENT;
   else {
     for(ii=0; ii<num_ihmms; ii++) ihmms[ii] =  a->pdata[ ihmms[ii]].code;
     return ESR_SUCCESS;
   }
 }

 void display_results(SR_SemanticResult *result, PFile* fout)
 {
   size_t i, size, len;
   LCHAR* keys[MAX_KEYS]; /* array of pointers to strings */
   LCHAR  value[MAX_STR_LENGTH];
   ESR_ReturnCode rc;

   size = MAX_KEYS;
   rc = result->getKeyList(result, (LCHAR**) &keys, &size); /* get the key list */
   if(rc == ESR_SUCCESS)
   {
     for(i=0; i<size; i++)
     {
       len = MAX_STR_LENGTH;
       if ((rc = result->getValue(result,keys[i],value,&len)) == ESR_SUCCESS)
         pfprintf(fout,"{%s : %s}\n",keys[i],value);
       else
         pfprintf(fout,"Error: %s\n",ESR_rc2str(rc));
     }
   }
   else
     pfprintf(fout,"Error: %s\n",ESR_rc2str(rc));
 }

 ESR_ReturnCode Parse(SR_Grammar* grammar, LCHAR* trans, PFile* fout)
 {
   ESR_ReturnCode rc;
   int i, result_count;
   SR_SemanticResult* semanticResults[MAX_SEM_RESULTS];

   result_count = MAX_SEM_RESULTS; /* initially not greater than MAX */
   for(i =0; i<result_count; i++)
     SR_SemanticResultCreate(&semanticResults[i]); /* create the result holders */

   lstrtrim(trans);

   rc = grammar->checkParse(grammar, trans, semanticResults, (size_t*) &result_count);
   if(rc != ESR_SUCCESS)
     return rc;

   if(result_count < 1)
   {
     pfprintf(fout,"no parse\n\n");
     return ESR_NO_MATCH_ERROR;
   }
   else
   {
     pfprintf(fout,"parse ok (%d results)\n", result_count);
     for(i=0; i < result_count; i++)
       display_results(semanticResults[i],fout);

     for(i=0; i < MAX_SEM_RESULTS; i++)
     {
       rc = semanticResults[i]->destroy(semanticResults[i]);
       if(rc != ESR_SUCCESS)
         return rc;
     }
     return ESR_SUCCESS;
   }
 }

 void load_filelist(char* filelist, char*** pfiles, int *pnum_files)
 {
   int i = 0;
   FILE* fp;
   char line[512];
   char **files = 0, *file;
   int num_files = 0;

   fp = fopen(filelist, "r");
   if(!fp) {
     pfprintf(PSTDOUT, "failed to open %s\n", filelist);
     goto DONE;
   }

   while( fgets(line, sizeof(line), fp)) {
     if(line[0] == '#') continue;
     i++;
   }
   fclose(fp);

   num_files = i;
   *files = CALLOC( num_files, sizeof(char*), __FILE__);
   fp = fopen(filelist, "r");
   for(i=0; fgets(line,sizeof(line),fp) && i<num_files; i++) {
     if(line[0] == '#') continue;
     strtok(line,"\n\r\t");
     file = files[i++] = CALLOC(strlen(line)+1,sizeof(char),__FILE__);
     strcpy( file, line);
   }
   fclose(fp);
   num_files = i;

  DONE:
   *pfiles = files;
   *pnum_files = num_files;
 }

 int* phonemecode_for_pel_table(CA_Arbdata* ca_arbdata)
 {
   static int table[2048];
   int i,j;
   tree_head topo;
   srec_arbdata* a = (srec_arbdata*)ca_arbdata;
   int num_hmms_in_phoneme;

   for(j=0; j< (int)(sizeof(table)/sizeof(int)); j++)
     table[j] = 0;

   for(i=0; i<a->num_phonemes; i++) {
     num_hmms_in_phoneme = 0;
     topo.low_pel_no  = 32567;
     topo.high_pel_no = 0;
     traverse_tree(a->pdata[i].model_nodes, &topo, &num_hmms_in_phoneme);
     if(debug)printf("phoneme %d num_hmms %d (%d-%d)\n", i, num_hmms_in_phoneme,
 		    topo.low_pel_no, topo.high_pel_no);

     for(j=topo.low_pel_no; j<=topo.high_pel_no; j++)
       table[j] = a->pdata[i].code;
   }
   return &table[0];
 }
	/---------------------------------------------------------------------------
	* test_g2g.c *
	* *
	* Copyright 2007, 2008 Nuance Communciations, Inc. *
	* *
	* Licensed under the Apache License, Version 2.0 (the 'License'); *
	* you may not use this file except in compliance with the License. *
	* *
	* You may obtain a copy of the License at *
	* http://www.apache.org/licenses/LICENSE-2.0 *
	* *
	* Unless required by applicable law or agreed to in writing, software *
	* distributed under the License is distributed on an 'AS IS' BASIS, *
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
	* See the License for the specific language governing permissions and *
	* limitations under the License. *
	* *
	---------------------------------------------------------------------------/



	#include "pstdio.h"
	#include "pmemory.h"
	#include "plog.h"
	#include "HashMap.h"
	#include "SR_Grammar.h"
	#include "SR_Vocabulary.h"
	#include "SR_SemanticResult.h"
	#include "ESR_Session.h"
	#include "ESR_Locale.h"
	#include "ESR_CommandLine.h"
	#include "LCHAR.h"

	#include "PFileSystem.h"
	#include "PANSIFileSystem.h"

	#include "SR_GrammarImpl.h"

	#include "simapi.h"
	#include "srec_context.h"
	#include "srec_arb.h"

	/**
	* @todo document
	*/
	typedef struct
	{
	unsigned short nnodes;
	unsigned long size;
	long phoneme;
	unsigned short node_pos;
	unsigned long node_off;
	short low_genone_no;
	short high_genone_no;
	short low_pel_no;
	short high_pel_no;
	}
	tree_head;


	int usage(LCHAR* exename)
	{
	pfprintf(PSTDOUT,"usage: %s -base <basefilename> \n",exename);
	pfprintf(PSTDOUT,"<basefilename> can be a file.g2g or @g2gfilelist\n");
	pfprintf(PSTDOUT,"[-checkword id] .. also checks word id in the file\n");
	pfprintf(PSTDOUT,"[-swiarb esr/config/lang/models/generic.swiarb] ... enables word check\n");
	return 1;
	}

	/* protos */
	ESR_ReturnCode find_phonemes_for_ihmms( CA_Arbdata* ca_arbdata, modelID* ihmms, int num_hmms);
	ESR_ReturnCode Parse(SR_Grammar* grammar, LCHAR* trans, PFile* fout);
	int CheckG2G(CA_Arbdata* arbdata, int* p4pTable, const char* base, int wordid, char* outbase);
	void load_filelist(char* filelist, char*** pfiles, int *pnum_files);
	int phonemecode_for_pel_table(CA_Arbdata arbdata);

	int debug = 0;
	#define MAX_LINE_LENGTH 256
	#define MAX_STR_LENGTH 80
	#define MAX_SEM_RESULTS 3
	#define MAX_KEYS 30

	/* main */

	int main (int argc, char **argv)
	{
	ESR_ReturnCode rc;
	LCHAR base[P_PATH_MAX] = L("");
	int i;
	CA_Arbdata* ca_arbdata;
	char* arbfile = NULL;
	char** g2glist;
	int g2glist_len;
	char* outbase = NULL;
	int *p4pTable;
	int wordid = 0;
	int log_level = 0;

	/*
	* Initialize portable library.
	*/
	CHKLOG(rc, PMemInit());
	/* CHKLOG(rc, PFileSystemCreate());
	CHKLOG(rc, PANSIFileSystemCreate());
	CHKLOG(rc, PANSIFileSystemAddPath(L("/dev/ansi"), L("/")));*/

	/* Set ANSI file-system as default file-system */
	/* CHKLOG(rc, PANSIFileSystemSetDefault(ESR_TRUE));*/
	/* Set virtual current working directory to native current working directory */
	/* len = P_PATH_MAX;
	CHKLOG(rc, PANSIFileSystemGetcwd(cwd, &len));
	CHKLOG(rc, PFileSystemChdir(cwd));*/

	if( argc <= 1)
	{
	usage(argv[0]);
	exit(EXIT_FAILURE);
	}

	for (i = 1; i < argc; ++i)
	{
	if(!LSTRCMP(argv[i], L("-base")))
	{
	++i;
	LSTRCPY(base, argv[i]);
	}
	else if(!LSTRCMP(argv[i],L("-out")))
	{
	outbase = argv[++i];
	}
	else if(!LSTRCMP(argv[i],L("-swiarb")))
	{
	arbfile = argv[++i];
	}
	else if(!LSTRCMP(argv[i],L("-checkword")))
	{
	wordid = atoi(argv[++i]);
	}
	else if(!LSTRCMP(argv[i],L("-log")))
	{
	log_level = 10;
	}
	else
	{
	printf("unrecog'd argument %s\n", argv[i]);
	exit(1);
	}
	}

	CHK(rc, PLogInit(NULL, log_level));

	if(arbfile) {
	ca_arbdata = CA_LoadArbdata(arbfile);
	if(!ca_arbdata) {
	pfprintf(PSTDOUT, "Error: loading arbfile %s\n", arbfile);
	goto CLEANUP;
	}
	pfprintf(PSTDOUT, "arbdata done\n");
	p4pTable = phonemecode_for_pel_table(ca_arbdata);
	pfprintf(PSTDOUT, "p4pTable done\n");
	} else {
	ca_arbdata = 0;
	p4pTable = 0;
	}

	if(base[0] == '@') {
	load_filelist(base+1, &g2glist, &g2glist_len);
	pfprintf(PSTDOUT, "g2glist %s .. %d entries\n", g2glist_len);
	for(i=0; i<g2glist_len; i++)
	CheckG2G( ca_arbdata, p4pTable, g2glist[i], wordid, outbase);
	}
	else {
	CheckG2G( ca_arbdata, p4pTable, base, wordid, outbase);
	}

	CLEANUP:
	PLogShutdown();
	/* PANSIFileSystemDestroy();
	PFileSystemDestroy();*/
	PMemSetLogFile(PSTDOUT);
	PMemDumpLogFile();
	PMemShutdown();
	return rc;
	}

	int CheckG2G(CA_Arbdata* ca_arbdata, int* p4pTable, const char* base, int wordid, char* outbase)
	{
	ESR_ReturnCode rc;
	SR_GrammarImpl *grammarImpl;
	SR_Grammar* grammar = NULL;
	srec_context* fst;
	CA_Syntax* syntax;
	modelID ilabels_preceding[64], num_ilabels_preceding;
	modelID ilabels_following[64], num_ilabels_following;
	modelID ilabels[128], num_ilabels;
	int i,j;
	unsigned long g2gsize;

	if(1) {
	FILE* fp;
	fp = fopen(base, "rb");
	if(!fp) g2gsize = 0;
	else {
	fseek(fp, 0, SEEK_END);
	g2gsize = ftell(fp);
	fclose(fp);
	}
	}

	rc = SR_GrammarLoad(base, &grammar);
	if(rc != ESR_SUCCESS) {
	pfprintf(PSTDOUT, "%s failed at load\n", base);
	goto CLEANUP;
	}

	grammarImpl = (SR_GrammarImpl*)grammar;
	syntax = grammarImpl->syntax;
	if(outbase) {
	CA_DumpSyntax( syntax, outbase);
	}

	fst = syntax->synx;
	pfprintf(PSTDOUT, "%s %d arcs %d/%d/%d nodes %d/%d/%d words %d/%d chars %d/%d modelver %d\n",
	base, g2gsize,
	fst->num_arcs, fst->num_base_arcs, fst->FSMarc_list_len,
	fst->num_nodes, fst->num_base_nodes, fst->FSMnode_list_len,
	fst->olabels->num_words, fst->olabels->max_words,
	fst->olabels->next_chars-fst->olabels->chars,
	fst->olabels->max_chars,
	#ifdef IMAGE_FORMAT_V2
	fst->modelid
	#else
	-1
	#endif
	);

	if(wordid == 0 \|\| ca_arbdata == 0)
	goto CLEANUP;

	if(wordid >= fst->olabels->num_words) {
	pfprintf(PSTDOUT, "%s failed 'cuz numwords(%d) < %d\n", base,
	fst->olabels->num_words, wordid);
	goto CLEANUP;
	}

	for(i=0; i<fst->num_arcs; i++) {
	if(fst->FSMarc_list[i].olabel == wordid) {
	FSMnode* node;
	FSMarc* arc = &fst->FSMarc_list[i];
	nodeID fr_node = arc->fr_node;
	arcID iarc;
	ilabels_following[0] = arc->ilabel;
	num_ilabels_following = 1;
	num_ilabels_preceding = 0;
	for( ; fr_node!=fst->start_node; fr_node=arc->fr_node) {
	node = &fst->FSMnode_list[fr_node];
	iarc = node->first_prev_arc;
	for( ; iarc!=MAXarcID; iarc=arc->linkl_prev_arc) {
	arc = &fst->FSMarc_list[iarc];
	if(arc->fr_node != fr_node) break;
	}
	if(iarc == MAXarcID) {
	pfprintf(PSTDOUT, "%s failed at 11\n", base);
	goto CLEANUP;
	}
	if(arc->ilabel == WORD_BOUNDARY) break;
	ilabels_preceding[num_ilabels_preceding++] = arc->ilabel;
	}
	arc = &fst->FSMarc_list[i];
	fr_node = arc->to_node;
	for( ; fr_node!=fst->end_node; fr_node=arc->to_node) {
	node = &fst->FSMnode_list[fr_node];
	iarc = node->un_ptr.first_next_arc;
	for( ; iarc!=MAXarcID; iarc=arc->linkl_next_arc) {
	arc = &fst->FSMarc_list[iarc];
	if(arc->to_node != fr_node) break;
	}
	if(iarc == MAXarcID) {
	pfprintf(PSTDOUT, "%s failed at 12\n", base);
	goto CLEANUP;
	}
	ilabels_following[num_ilabels_following++] = arc->ilabel;
	if(arc->ilabel == WORD_BOUNDARY) break;
	}
	num_ilabels = 0;
	for(j=0; j<num_ilabels_preceding; j++)
	ilabels[num_ilabels++] = ilabels_preceding[num_ilabels_preceding-1-j];
	for(j=0; j<num_ilabels_following; j++)
	ilabels[num_ilabels++] = ilabels_following[j];
	if(ilabels[num_ilabels-1] == WORD_BOUNDARY)
	num_ilabels--;
	for(j=0; j<num_ilabels; j++) {
	if(ilabels[j]<fst->hmm_ilabel_offset) {
	pfprintf(PSTDOUT, "%s failed at 15\n", base);
	goto CLEANUP;
	} else
	ilabels[j] = ilabels[j] - (labelID)fst->hmm_ilabel_offset;
	}
	pfprintf(PSTDOUT, "%s (W%d) ihmms ", fst->olabels->words[wordid], wordid);
	for(j=0;j<num_ilabels;j++)
	pfprintf(PSTDOUT, " %d", ilabels[j]);
	pfprintf(PSTDOUT, "\n");
	if(num_ilabels < 2) {
	pfprintf(PSTDOUT, "%s failed at 1\n", base);
	goto CLEANUP;
	}
	if(p4pTable)
	rc = find_phonemes_for_ihmms( ca_arbdata, ilabels, num_ilabels);
	else {
	rc = ESR_SUCCESS;
	for(j=0; j<num_ilabels; j++) {
	if(p4pTable[ ilabels[j]]<0) {
	rc = ESR_NO_MATCH_ERROR;
	ilabels[j] = MAXmodelID;
	} else {
	ilabels[j] = (modelID)p4pTable[ ilabels[j]];
	}
	}
	}

	if(rc) {
	pfprintf(PSTDOUT, "%s failed at 2\n", base);
	goto CLEANUP;
	}
	pfprintf(PSTDOUT, "%s ", fst->olabels->words[wordid]);
	for(j=0;j<num_ilabels;j++) pfprintf(PSTDOUT, "%c", ilabels[j]);
	pfprintf(PSTDOUT, "\n");
	rc = Parse( grammar, fst->olabels->words[wordid], PSTDOUT);
	if(rc) {
	pfprintf(PSTDOUT, "%s failed at 3\n", base);
	goto CLEANUP;
	}
	pfprintf(PSTDOUT, "%s PASSED (on %s)\n", base, fst->olabels->words[wordid]);
	break;
	}
	}

	return 0;
	CLEANUP:
	if(grammar) SR_GrammarDestroy(grammar);
	return 1;

	}


	int traverse_tree(tree_node* node, tree_head tree_topo, int num_terminal_nodes)
	{
	if(node)
	tree_topo->nnodes++;

	if(node->node.quest_index < 0) {
	if(num_terminal_nodes)
	(*num_terminal_nodes)++;
	if( node->term.pelid < tree_topo->low_pel_no)
	tree_topo->low_pel_no = tree_topo->low_genone_no = node->term.pelid;
	if( node->term.pelid > tree_topo->high_pel_no)
	tree_topo->high_pel_no = tree_topo->high_genone_no = node->term.pelid;
	} else {
	traverse_tree( (tree_node*)node->node.fail, tree_topo, num_terminal_nodes);
	traverse_tree( (tree_node*)node->node.pass, tree_topo, num_terminal_nodes);
	}
	return 0;

	}

	int num_nodes_in_tree(tree_node* node, int *num_terminal_nodes)
	{
	tree_head topo;
	*num_terminal_nodes = 0;
	topo.nnodes = 0;
	traverse_tree(node, &topo, num_terminal_nodes);
	return topo.nnodes;
	}

	ESR_ReturnCode find_phonemes_for_ihmms( CA_Arbdata* ca_arbdata, modelID* ihmms, int num_ihmms)
	{
	int ii, i;
	int num_hmms_in_phoneme;
	tree_head topo;
	srec_arbdata* a = (srec_arbdata*)ca_arbdata;
	int num_phonemes_for_ihmms = 0;

	for(ii=0; ii<num_ihmms; ii++) {
	for(i=0; i<a->num_phonemes; i++) {
	num_hmms_in_phoneme = 0;
	topo.low_pel_no = 32567;
	topo.high_pel_no = 0;
	traverse_tree(a->pdata[i].model_nodes, &topo, &num_hmms_in_phoneme);
	if(debug)printf("phoneme %d num_hmms %d (%d-%d)\n", i, num_hmms_in_phoneme,
	topo.low_pel_no, topo.high_pel_no);
	if(ihmms[ii] >= topo.low_pel_no && ihmms[ii]<= topo.high_pel_no) {
	ihmms[ii] = (modelID)i;
	num_phonemes_for_ihmms++;
	break;
	}
	}
	if( i==a->num_phonemes) {
	if(ihmms[ii]<=5) {
	ihmms[ii] = 0;
	num_phonemes_for_ihmms++;
	} else {
	PLogError("error: could not find hmm%d under any phoneme! ",ihmms[ii]);
	}
	}

	}
	if(num_phonemes_for_ihmms != num_ihmms)
	return ESR_INVALID_ARGUMENT;
	else {
	for(ii=0; ii<num_ihmms; ii++) ihmms[ii] = a->pdata[ ihmms[ii]].code;
	return ESR_SUCCESS;
	}
	}

	void display_results(SR_SemanticResult result, PFile fout)
	{
	size_t i, size, len;
	LCHAR* keys[MAX_KEYS]; /* array of pointers to strings */
	LCHAR value[MAX_STR_LENGTH];
	ESR_ReturnCode rc;

	size = MAX_KEYS;
	rc = result->getKeyList(result, (LCHAR*) &keys, &size); / get the key list */
	if(rc == ESR_SUCCESS)
	{
	for(i=0; i<size; i++)
	{
	len = MAX_STR_LENGTH;
	if ((rc = result->getValue(result,keys[i],value,&len)) == ESR_SUCCESS)
	pfprintf(fout,"{%s : %s}\n",keys[i],value);
	else
	pfprintf(fout,"Error: %s\n",ESR_rc2str(rc));
	}
	}
	else
	pfprintf(fout,"Error: %s\n",ESR_rc2str(rc));
	}

	ESR_ReturnCode Parse(SR_Grammar* grammar, LCHAR* trans, PFile* fout)
	{
	ESR_ReturnCode rc;
	int i, result_count;
	SR_SemanticResult* semanticResults[MAX_SEM_RESULTS];

	result_count = MAX_SEM_RESULTS; /* initially not greater than MAX */
	for(i =0; i<result_count; i++)
	SR_SemanticResultCreate(&semanticResults[i]); /* create the result holders */

	lstrtrim(trans);

	rc = grammar->checkParse(grammar, trans, semanticResults, (size_t*) &result_count);
	if(rc != ESR_SUCCESS)
	return rc;

	if(result_count < 1)
	{
	pfprintf(fout,"no parse\n\n");
	return ESR_NO_MATCH_ERROR;
	}
	else
	{
	pfprintf(fout,"parse ok (%d results)\n", result_count);
	for(i=0; i < result_count; i++)
	display_results(semanticResults[i],fout);

	for(i=0; i < MAX_SEM_RESULTS; i++)
	{
	rc = semanticResults[i]->destroy(semanticResults[i]);
	if(rc != ESR_SUCCESS)
	return rc;
	}
	return ESR_SUCCESS;
	}
	}

	void load_filelist(char* filelist, char*** pfiles, int *pnum_files)
	{
	int i = 0;
	FILE* fp;
	char line[512];
	char *files = 0, file;
	int num_files = 0;

	fp = fopen(filelist, "r");
	if(!fp) {
	pfprintf(PSTDOUT, "failed to open %s\n", filelist);
	goto DONE;
	}

	while( fgets(line, sizeof(line), fp)) {
	if(line[0] == '#') continue;
	i++;
	}
	fclose(fp);

	num_files = i;
	files = CALLOC( num_files, sizeof(char), __FILE__);
	fp = fopen(filelist, "r");
	for(i=0; fgets(line,sizeof(line),fp) && i<num_files; i++) {
	if(line[0] == '#') continue;
	strtok(line,"\n\r\t");
	file = files[i++] = CALLOC(strlen(line)+1,sizeof(char),__FILE__);
	strcpy( file, line);
	}
	fclose(fp);
	num_files = i;

	DONE:
	*pfiles = files;
	*pnum_files = num_files;
	}

	int* phonemecode_for_pel_table(CA_Arbdata* ca_arbdata)
	{
	static int table[2048];
	int i,j;
	tree_head topo;
	srec_arbdata* a = (srec_arbdata*)ca_arbdata;
	int num_hmms_in_phoneme;

	for(j=0; j< (int)(sizeof(table)/sizeof(int)); j++)
	table[j] = 0;

	for(i=0; i<a->num_phonemes; i++) {
	num_hmms_in_phoneme = 0;
	topo.low_pel_no = 32567;
	topo.high_pel_no = 0;
	traverse_tree(a->pdata[i].model_nodes, &topo, &num_hmms_in_phoneme);
	if(debug)printf("phoneme %d num_hmms %d (%d-%d)\n", i, num_hmms_in_phoneme,
	topo.low_pel_no, topo.high_pel_no);

	for(j=topo.low_pel_no; j<=topo.high_pel_no; j++)
	table[j] = a->pdata[i].code;
	}
	return &table[0];
	}