| /* |
| * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| /** |
| * @file picoktab.c |
| * |
| * symbol tables needed at runtime |
| * |
| * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland |
| * All rights reserved. |
| * |
| * History: |
| * - 2009-04-20 -- initial version |
| * |
| */ |
| |
| #include "picoos.h" |
| #include "picodbg.h" |
| #include "picoknow.h" |
| #include "picobase.h" |
| #include "picoktab.h" |
| #include "picodata.h" |
| |
| #ifdef __cplusplus |
| extern "C" { |
| #endif |
| #if 0 |
| } |
| #endif |
| |
| |
| /** @todo : the following would be better part of a knowledge base. |
| * Make sure it is consistent with the phoneme symbol table used in the lingware */ |
| |
| /* PLANE_PHONEMES */ |
| |
| /* PLANE_POS */ |
| |
| /* PLANE_PB_STRENGTHS */ |
| |
| /* PLANE_ACCENTS */ |
| |
| /* PLANE_INTERN */ |
| #define PICOKTAB_TMPID_PHONSTART '\x26' /* 38 '&' */ |
| #define PICOKTAB_TMPID_PHONTERM '\x23' /* 35 '#' */ |
| |
| |
| /* ************************************************************/ |
| /* fixed ids */ |
| /* ************************************************************/ |
| |
| |
| static pico_status_t ktabIdsInitialize(register picoknow_KnowledgeBase this, |
| picoos_Common common) |
| { |
| picoktab_FixedIds ids; |
| |
| PICODBG_DEBUG(("start")); |
| |
| if (NULL == this || NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| ids = (picoktab_FixedIds) this->subObj; |
| |
| ids->phonStartId = PICOKTAB_TMPID_PHONSTART; |
| ids->phonTermId = PICOKTAB_TMPID_PHONTERM; |
| return PICO_OK; |
| } |
| |
| |
| static pico_status_t ktabIdsSubObjDeallocate(register picoknow_KnowledgeBase this, |
| picoos_MemoryManager mm) |
| { |
| if (NULL != this) { |
| picoos_deallocate(mm, (void *) &this->subObj); |
| } |
| return PICO_OK; |
| } |
| |
| pico_status_t picoktab_specializeIdsKnowledgeBase(picoknow_KnowledgeBase this, |
| picoos_Common common) |
| { |
| if (NULL == this) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| this->subDeallocate = ktabIdsSubObjDeallocate; |
| this->subObj = picoos_allocate(common->mm, sizeof(picoktab_fixed_ids_t)); |
| if (NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, |
| NULL, NULL); |
| } |
| return ktabIdsInitialize(this, common); |
| } |
| |
| picoktab_FixedIds picoktab_getFixedIds(picoknow_KnowledgeBase this) |
| { |
| return ((NULL == this) ? NULL : ((picoktab_FixedIds) this->subObj)); |
| } |
| |
| |
| picoktab_FixedIds picoktab_newFixedIds(picoos_MemoryManager mm) |
| { |
| picoktab_FixedIds this = (picoktab_FixedIds) picoos_allocate(mm,sizeof(*this)); |
| if (NULL != this) { |
| /* initialize */ |
| } |
| return this; |
| } |
| |
| |
| void picoktab_disposeFixedIds(picoos_MemoryManager mm, picoktab_FixedIds * this) |
| { |
| if (NULL != (*this)) { |
| /* terminate */ |
| picoos_deallocate(mm,(void *)this); |
| } |
| } |
| |
| |
| |
| /* ************************************************************/ |
| /* Graphs */ |
| /* ************************************************************/ |
| |
| /* overview binary file format for graphs kb: |
| |
| graphs-kb = NROFSENTRIES SIZEOFSENTRY ofstable graphs |
| |
| NROFSENTRIES : 2 bytes, number of entries in offset table |
| SIZEOFSENTRY : 1 byte, size of one entry in offset table |
| |
| ofstable = {OFFSET}=NROFSENTRIES (contains NROFSENTRIES entries of OFFSET) |
| |
| OFFSET: SIZEOFSENTRY bytes, offset to baseaddress of graphs-kb to entry in graphs |
| |
| graphs = {graph}=NROFSENTRIES (contains NROFSENTRIES entries of graph) |
| |
| graph = PROPSET FROM TO [TOKENTYPE] [TOKENSUBTYPE] [VALUE] [LOWERCASE] [GRAPHSUBS1] [GRAPHSUBS2] |
| |
| FROM : 1..4 unsigned bytes, UTF8 character without terminating 0 |
| TO : 1..4 unsigned bytes, UTF8 character without terminating 0 |
| PROPSET : 1 unsigned byte, least significant bit : has TO field |
| next bit : has TOKENTYPE |
| next bit : has TOKENSUBTYPE |
| next bit : has VALUE |
| next bit : has LOWERCASE |
| next bit : has GRAPHSUBS1 |
| next bit : has GRAPHSUBS2 |
| next bit : has PUNC |
| |
| TOKENTYPE : 1 unsigned byte |
| TOKENSUBTYPE : 1 unsigned byte |
| VALUE : 1 unsigned byte |
| LOWERCASE : 1..4 unsigned bytes, UTF8 character without terminating 0 |
| GRAPHSUBS1 : 1..4 unsigned bytes, UTF8 character without terminating 0 |
| GRAPHSUBS2 : 1..4 unsigned bytes, UTF8 character without terminating 0 |
| PUNC : 1 unsigned byte |
| */ |
| |
| static picoos_uint32 ktab_propOffset (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint32 prop); |
| |
| #define KTAB_START_GRAPHS_NR_OFFSET 0 |
| #define KTAB_START_GRAPHS_SIZE_OFFSET 2 |
| #define KTAB_START_GRAPHS_OFFSET_TABLE 3 |
| #define KTAB_START_GRAPHS_GRAPH_TABLE 0 |
| |
| /* bitmasks to extract the grapheme properties info from the property set */ |
| #define KTAB_GRAPH_PROPSET_TO ((picoos_uint8)'\x01') |
| #define KTAB_GRAPH_PROPSET_TOKENTYPE ((picoos_uint8)'\x02') |
| #define KTAB_GRAPH_PROPSET_TOKENSUBTYPE ((picoos_uint8)'\x04') |
| #define KTAB_GRAPH_PROPSET_VALUE ((picoos_uint8)'\x08') |
| #define KTAB_GRAPH_PROPSET_LOWERCASE ((picoos_uint8)'\x010') |
| #define KTAB_GRAPH_PROPSET_GRAPHSUBS1 ((picoos_uint8)'\x020') |
| #define KTAB_GRAPH_PROPSET_GRAPHSUBS2 ((picoos_uint8)'\x040') |
| #define KTAB_GRAPH_PROPSET_PUNCT ((picoos_uint8)'\x080') |
| |
| |
| typedef struct ktabgraphs_subobj *ktabgraphs_SubObj; |
| |
| typedef struct ktabgraphs_subobj { |
| picoos_uint16 nrOffset; |
| picoos_uint16 sizeOffset; |
| |
| picoos_uint8 * offsetTable; |
| picoos_uint8 * graphTable; |
| } ktabgraphs_subobj_t; |
| |
| |
| |
| static pico_status_t ktabGraphsInitialize(register picoknow_KnowledgeBase this, |
| picoos_Common common) { |
| ktabgraphs_subobj_t * ktabgraphs; |
| |
| PICODBG_DEBUG(("start")); |
| |
| if (NULL == this || NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| ktabgraphs = (ktabgraphs_subobj_t *) this->subObj; |
| ktabgraphs->nrOffset = ((int)(this->base[KTAB_START_GRAPHS_NR_OFFSET])) + 256*(int)(this->base[KTAB_START_GRAPHS_NR_OFFSET+1]); |
| ktabgraphs->sizeOffset = (int)(this->base[KTAB_START_GRAPHS_SIZE_OFFSET]); |
| ktabgraphs->offsetTable = &(this->base[KTAB_START_GRAPHS_OFFSET_TABLE]); |
| ktabgraphs->graphTable = &(this->base[KTAB_START_GRAPHS_GRAPH_TABLE]); |
| return PICO_OK; |
| } |
| |
| static pico_status_t ktabGraphsSubObjDeallocate(register picoknow_KnowledgeBase this, |
| picoos_MemoryManager mm) { |
| if (NULL != this) { |
| picoos_deallocate(mm, (void *) &this->subObj); |
| } |
| return PICO_OK; |
| } |
| |
| |
| pico_status_t picoktab_specializeGraphsKnowledgeBase(picoknow_KnowledgeBase this, |
| picoos_Common common) { |
| if (NULL == this) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| this->subDeallocate = ktabGraphsSubObjDeallocate; |
| this->subObj = picoos_allocate(common->mm, sizeof(ktabgraphs_subobj_t)); |
| if (NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, |
| NULL, NULL); |
| } |
| return ktabGraphsInitialize(this, common); |
| } |
| |
| |
| picoktab_Graphs picoktab_getGraphs(picoknow_KnowledgeBase this) { |
| if (NULL == this) { |
| return NULL; |
| } else { |
| return (picoktab_Graphs) this->subObj; |
| } |
| } |
| |
| |
| /* Graphs methods */ |
| |
| picoos_uint8 picoktab_hasVowellikeProp(const picoktab_Graphs this, |
| const picoos_uint8 *graph, |
| const picoos_uint8 graphlenmax) { |
| |
| picoos_uint8 ui8App; |
| picoos_uint32 graphsOffset; |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this; |
| |
| ui8App = graphlenmax; /* avoid warning "var not used in this function"*/ |
| |
| graphsOffset = picoktab_graphOffset (this, (picoos_uchar *)graph); |
| return g->graphTable[graphsOffset + ktab_propOffset (this, graphsOffset, KTAB_GRAPH_PROPSET_TOKENTYPE)] == PICODATA_ITEMINFO1_TOKTYPE_LETTERV; |
| } |
| |
| |
| static void ktab_getStrProp (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint32 propOffset, picoos_uchar * str) |
| { |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this; |
| picoos_uint32 i, l; |
| |
| i = 0; |
| l = picobase_det_utf8_length(g->graphTable[graphsOffset+propOffset]); |
| while (i<l) { |
| str[i] = g->graphTable[graphsOffset+propOffset+i]; |
| i++; |
| } |
| str[l] = 0; |
| } |
| |
| |
| static picoos_uint32 ktab_propOffset(const picoktab_Graphs this, |
| picoos_uint32 graphsOffset, picoos_uint32 prop) |
| /* Returns offset of property 'prop' inside the graph with offset 'graphsOffset' in graphs table; |
| If the property is found, a value > 0 is returned otherwise 0 */ |
| { |
| picoos_uint32 n = 0; |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj) this; |
| |
| if ((g->graphTable[graphsOffset] & prop) == prop) { |
| n = n + 1; /* overread PROPSET field */ |
| n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread FROM field */ |
| if (prop > KTAB_GRAPH_PROPSET_TO) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_TO) |
| == KTAB_GRAPH_PROPSET_TO) { |
| n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread TO field */ |
| } |
| } else { |
| return n; |
| } |
| if (prop > KTAB_GRAPH_PROPSET_TOKENTYPE) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_TOKENTYPE) |
| == KTAB_GRAPH_PROPSET_TOKENTYPE) { |
| n = n + 1; /* overread TOKENTYPE field */ |
| } |
| } else { |
| return n; |
| } |
| if (prop > KTAB_GRAPH_PROPSET_TOKENSUBTYPE) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_TOKENSUBTYPE) |
| == KTAB_GRAPH_PROPSET_TOKENSUBTYPE) { |
| n = n + 1; /* overread stokentype field */ |
| } |
| } else { |
| return n; |
| } |
| if (prop > KTAB_GRAPH_PROPSET_VALUE) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_VALUE) |
| == KTAB_GRAPH_PROPSET_VALUE) { |
| n = n + 1; /* overread value field */ |
| } |
| } else { |
| return n; |
| } |
| if (prop > KTAB_GRAPH_PROPSET_LOWERCASE) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_LOWERCASE) |
| == KTAB_GRAPH_PROPSET_LOWERCASE) { |
| n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread lowercase field */ |
| } |
| } else { |
| return n; |
| } |
| if (prop > KTAB_GRAPH_PROPSET_GRAPHSUBS1) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_GRAPHSUBS1) |
| == KTAB_GRAPH_PROPSET_GRAPHSUBS1) { |
| n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread graphsubs1 field */ |
| } |
| } else { |
| return n; |
| } |
| if (prop > KTAB_GRAPH_PROPSET_GRAPHSUBS2) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_GRAPHSUBS2) |
| == KTAB_GRAPH_PROPSET_GRAPHSUBS2) { |
| n = n + picobase_det_utf8_length(g->graphTable[graphsOffset+n]); /* overread graphsubs2 field */ |
| } |
| } else { |
| return n; |
| } |
| if (prop > KTAB_GRAPH_PROPSET_PUNCT) { |
| if ((g->graphTable[graphsOffset] & KTAB_GRAPH_PROPSET_PUNCT) |
| == KTAB_GRAPH_PROPSET_PUNCT) { |
| n = n + 1; /* overread value field */ |
| } |
| } else { |
| return n; |
| } |
| } |
| |
| return n; |
| } |
| |
| |
| picoos_uint32 picoktab_graphOffset (const picoktab_Graphs this, picoos_uchar * utf8graph) |
| { ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this; |
| picoos_int32 a, b, m; |
| picoos_uint32 graphsOffset; |
| picoos_uint32 propOffset; |
| picobase_utf8char from; |
| picobase_utf8char to; |
| picoos_bool utfGEfrom; |
| picoos_bool utfLEto; |
| |
| if (g->nrOffset > 0) { |
| a = 0; |
| b = g->nrOffset-1; |
| do { |
| m = (a+b) / 2; |
| |
| /* get offset to graph[m] */ |
| if (g->sizeOffset == 1) { |
| graphsOffset = g->offsetTable[g->sizeOffset*m]; |
| } |
| else { |
| graphsOffset = g->offsetTable[g->sizeOffset*m ] + |
| 256*g->offsetTable[g->sizeOffset*m + 1]; |
| /* PICODBG_DEBUG(("picoktab_graphOffset: %i %i %i %i", m, g->offsetTable[g->sizeOffset*m], g->offsetTable[g->sizeOffset*m + 1], graphsOffset)); |
| */ |
| } |
| |
| /* get FROM and TO field of graph[m] */ |
| ktab_getStrProp(this, graphsOffset, 1, from); |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_TO); |
| if (propOffset > 0) { |
| ktab_getStrProp(this, graphsOffset, propOffset, to); |
| } |
| else { |
| picoos_strcpy((picoos_char *)to, (picoos_char *)from); |
| } |
| |
| /* PICODBG_DEBUG(("picoktab_graphOffset: %i %i %i '%s' '%s' '%s'", a, m, b, from, utf8graph, to)); |
| */ |
| utfGEfrom = picoos_strcmp((picoos_char *)utf8graph, (picoos_char *)from) >= 0; |
| utfLEto = picoos_strcmp((picoos_char *)utf8graph, (picoos_char *)to) <= 0; |
| |
| if (utfGEfrom && utfLEto) { |
| /* PICODBG_DEBUG(("picoktab_graphOffset: utf char '%s' found", utf8graph)); |
| */ |
| return graphsOffset; |
| } |
| if (!utfGEfrom) { |
| b = m-1; |
| } |
| else if (!utfLEto) { |
| a = m+1; |
| } |
| } while (a<=b); |
| } |
| PICODBG_DEBUG(("picoktab_graphOffset: utf char '%s' not found", utf8graph)); |
| return 0; |
| } |
| |
| |
| |
| |
| picoos_bool picoktab_getIntPropTokenType (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint8 * stokenType) |
| { |
| picoos_uint32 propOffset; |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this; |
| |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_TOKENTYPE); |
| if (propOffset > 0) { |
| *stokenType = (picoos_uint8)(g->graphTable[graphsOffset+propOffset]); |
| return TRUE; |
| } |
| else { |
| return FALSE; |
| } |
| } |
| |
| |
| picoos_bool picoktab_getIntPropTokenSubType (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_int8 * stokenSubType) |
| { |
| picoos_uint32 propOffset; |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this; |
| |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_TOKENSUBTYPE); |
| if (propOffset > 0) { |
| *stokenSubType = (picoos_int8)(g->graphTable[graphsOffset+propOffset]); |
| return TRUE; |
| } |
| else { |
| return FALSE; |
| } |
| } |
| |
| picoos_bool picoktab_getIntPropValue (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint32 * value) |
| { |
| picoos_uint32 propOffset; |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this; |
| |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_VALUE); |
| if (propOffset > 0) { |
| *value = (picoos_uint32)(g->graphTable[graphsOffset+propOffset]); |
| return TRUE; |
| } |
| else { |
| return FALSE; |
| } |
| } |
| |
| |
| picoos_bool picoktab_getIntPropPunct (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uint8 * info1, picoos_uint8 * info2) |
| { |
| picoos_uint32 propOffset; |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj)this; |
| |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_PUNCT); |
| if (propOffset > 0) { |
| if (g->graphTable[graphsOffset+propOffset] == 2) { |
| *info1 = PICODATA_ITEMINFO1_PUNC_SENTEND; |
| } |
| else { |
| *info1 = PICODATA_ITEMINFO1_PUNC_PHRASEEND; |
| } |
| if (g->graphTable[graphsOffset+1] == '.') { |
| *info2 = PICODATA_ITEMINFO2_PUNC_SENT_T; |
| } |
| else if (g->graphTable[graphsOffset+1] == '?') { |
| *info2 = PICODATA_ITEMINFO2_PUNC_SENT_Q; |
| } |
| else if (g->graphTable[graphsOffset+1] == '!') { |
| *info2 = PICODATA_ITEMINFO2_PUNC_SENT_E; |
| } |
| else { |
| *info2 = PICODATA_ITEMINFO2_PUNC_PHRASE; |
| } |
| return TRUE; |
| } |
| else { |
| return FALSE; |
| } |
| } |
| |
| |
| picoos_bool picoktab_getStrPropLowercase (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uchar * lowercase) |
| { |
| picoos_uint32 propOffset; |
| |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_LOWERCASE); |
| if (propOffset > 0) { |
| ktab_getStrProp(this, graphsOffset, propOffset, lowercase); |
| return TRUE; |
| } |
| else { |
| return FALSE; |
| } |
| } |
| |
| |
| picoos_bool picoktab_getStrPropGraphsubs1 (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uchar * graphsubs1) |
| { |
| picoos_uint32 propOffset; |
| |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_GRAPHSUBS1); |
| if (propOffset > 0) { |
| ktab_getStrProp(this, graphsOffset, propOffset, graphsubs1); |
| return TRUE; |
| } |
| else { |
| return FALSE; |
| } |
| } |
| |
| |
| picoos_bool picoktab_getStrPropGraphsubs2 (const picoktab_Graphs this, picoos_uint32 graphsOffset, picoos_uchar * graphsubs2) |
| { |
| picoos_uint32 propOffset; |
| |
| propOffset = ktab_propOffset(this, graphsOffset, KTAB_GRAPH_PROPSET_GRAPHSUBS2); |
| if (propOffset > 0) { |
| ktab_getStrProp(this, graphsOffset, propOffset, graphsubs2); |
| return TRUE; |
| } |
| else { |
| return FALSE; |
| } |
| } |
| /* *****************************************************************/ |
| /* used for tools */ |
| |
| static void ktab_getUtf8 (picoos_uchar ** pos, picoos_uchar * to) |
| { |
| picoos_uint32 l; |
| l = picobase_det_utf8_length(**pos); |
| while (l>0) { |
| *(to++) = *((*pos)++); |
| l--; |
| } |
| *to = 0; |
| } |
| |
| picoos_uint16 picoktab_graphsGetNumEntries(const picoktab_Graphs this) |
| { |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj) this; |
| return g->nrOffset; |
| } |
| |
| void picoktab_graphsGetGraphInfo(const picoktab_Graphs this, |
| picoos_uint16 graphIndex, picoos_uchar * from, picoos_uchar * to, |
| picoos_uint8 * propset, |
| picoos_uint8 * stokenType, picoos_uint8 * stokenSubType, |
| picoos_uint8 * value, picoos_uchar * lowercase, |
| picoos_uchar * graphsubs1, picoos_uchar * graphsubs2, |
| picoos_uint8 * punct) { |
| ktabgraphs_subobj_t * g = (ktabgraphs_SubObj) this; |
| picoos_uint32 graphsOffset; |
| picoos_uint8 * pos; |
| |
| /* calculate offset of graph[graphIndex] */ |
| if (g->sizeOffset == 1) { |
| graphsOffset = g->offsetTable[graphIndex]; |
| } else { |
| graphsOffset = g->offsetTable[2 * graphIndex] |
| + (g->offsetTable[2 * graphIndex + 1] << 8); |
| } |
| pos = &(g->graphTable[graphsOffset]); |
| *propset = *pos; |
| |
| pos++; /* advance to FROM */ |
| ktab_getUtf8(&pos, from); /* get FROM and advance */ |
| if ((*propset) & KTAB_GRAPH_PROPSET_TO) { |
| ktab_getUtf8(&pos, to); /* get TO and advance */ |
| } else { |
| picoos_strcpy((picoos_char *)to, (picoos_char *)from); |
| } |
| if ((*propset) & KTAB_GRAPH_PROPSET_TOKENTYPE) { |
| (*stokenType) = *(pos++); /* get TOKENTYPE and advance */ |
| } else { |
| (*stokenType) = -1; |
| } |
| if ((*propset) & KTAB_GRAPH_PROPSET_TOKENSUBTYPE) { |
| (*stokenSubType) = *(pos++); /* get TOKENSUBTYPE and advance */ |
| } else { |
| (*stokenSubType) = -1; |
| } |
| if ((*propset) & KTAB_GRAPH_PROPSET_VALUE) { |
| (*value) = *(pos++); /* get VALUE and advance */ |
| } else { |
| (*value) = -1; |
| } |
| if ((*propset) & KTAB_GRAPH_PROPSET_LOWERCASE) { |
| ktab_getUtf8(&pos, lowercase); /* get LOWERCASE and advance */ |
| } else { |
| lowercase[0] = NULLC; |
| } |
| if ((*propset) & KTAB_GRAPH_PROPSET_GRAPHSUBS1) { |
| ktab_getUtf8(&pos, graphsubs1); /* get GRAPHSUBS1 and advance */ |
| } else { |
| graphsubs1[0] = NULLC; |
| } |
| if ((*propset) & KTAB_GRAPH_PROPSET_GRAPHSUBS2) { |
| ktab_getUtf8(&pos, graphsubs2); /* get GRAPHSUBS2 and advance */ |
| } else { |
| graphsubs2[0] = NULLC; |
| } |
| if ((*propset) & KTAB_GRAPH_PROPSET_PUNCT) { |
| (*punct) = *(pos++); /* get PUNCT and advance */ |
| } else { |
| (*punct) = -1; |
| } |
| } |
| |
| /* ************************************************************/ |
| /* Phones */ |
| /* ************************************************************/ |
| |
| /* overview binary file format for phones kb: |
| |
| phones-kb = specids propertytable |
| |
| specids = PRIMSTRESSID1 SECSTRESSID1 SYLLBOUNDID1 PAUSEID1 WORDBOUNDID1 |
| RESERVE1 RESERVE1 RESERVE1 |
| |
| propertytable = {PHONEPROP2}=256 |
| |
| PRIMSTRESSID1: one byte, ID of primary stress |
| SECSTRESSID1: one byte, ID of secondary stress |
| SYLLBOUNDID1: one byte, ID of syllable boundary |
| PAUSEID1: one byte, ID of pause |
| RESERVE1: reserved for future use |
| |
| PHONEPROP2: one byte, max. of 256 phones directly access this table |
| to check a property for a phone; binary properties |
| encoded (1 bit per prop) |
| least significant bit: vowel |
| next bit: diphth |
| next bit: glott |
| next bit: nonsyllvowel |
| next bit: syllcons |
| 3 bits spare |
| */ |
| |
| #define KTAB_START_SPECIDS 0 |
| #define KTAB_IND_PRIMSTRESS 0 |
| #define KTAB_IND_SECSTRESS 1 |
| #define KTAB_IND_SYLLBOUND 2 |
| #define KTAB_IND_PAUSE 3 |
| #define KTAB_IND_WORDBOUND 4 |
| |
| #define KTAB_START_PROPS 8 |
| |
| |
| typedef struct ktabphones_subobj *ktabphones_SubObj; |
| |
| typedef struct ktabphones_subobj { |
| picoos_uint8 *specids; |
| picoos_uint8 *props; |
| } ktabphones_subobj_t; |
| |
| |
| /* bitmasks to extract the property info from props */ |
| #define KTAB_PPROP_VOWEL '\x01' |
| #define KTAB_PPROP_DIPHTH '\x02' |
| #define KTAB_PPROP_GLOTT '\x04' |
| #define KTAB_PPROP_NONSYLLVOWEL '\x08' |
| #define KTAB_PPROP_SYLLCONS '\x10' |
| |
| |
| static pico_status_t ktabPhonesInitialize(register picoknow_KnowledgeBase this, |
| picoos_Common common) { |
| ktabphones_subobj_t * ktabphones; |
| |
| PICODBG_DEBUG(("start")); |
| |
| if (NULL == this || NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| ktabphones = (ktabphones_subobj_t *) this->subObj; |
| ktabphones->specids = &(this->base[KTAB_START_SPECIDS]); |
| ktabphones->props = &(this->base[KTAB_START_PROPS]); |
| return PICO_OK; |
| } |
| |
| static pico_status_t ktabPhonesSubObjDeallocate(register picoknow_KnowledgeBase this, |
| picoos_MemoryManager mm) { |
| if (NULL != this) { |
| picoos_deallocate(mm, (void *) &this->subObj); |
| } |
| return PICO_OK; |
| } |
| |
| pico_status_t picoktab_specializePhonesKnowledgeBase(picoknow_KnowledgeBase this, |
| picoos_Common common) { |
| if (NULL == this) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| this->subDeallocate = ktabPhonesSubObjDeallocate; |
| this->subObj = picoos_allocate(common->mm, sizeof(ktabphones_subobj_t)); |
| if (NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, |
| NULL, NULL); |
| } |
| return ktabPhonesInitialize(this, common); |
| } |
| |
| picoktab_Phones picoktab_getPhones(picoknow_KnowledgeBase this) { |
| if (NULL == this) { |
| return NULL; |
| } else { |
| return (picoktab_Phones) this->subObj; |
| } |
| } |
| |
| |
| /* Phones methods */ |
| |
| picoos_uint8 picoktab_hasVowelProp(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (KTAB_PPROP_VOWEL & ((ktabphones_SubObj)this)->props[ch]); |
| } |
| picoos_uint8 picoktab_hasDiphthProp(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (KTAB_PPROP_DIPHTH & ((ktabphones_SubObj)this)->props[ch]); |
| } |
| picoos_uint8 picoktab_hasGlottProp(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (KTAB_PPROP_GLOTT & ((ktabphones_SubObj)this)->props[ch]); |
| } |
| picoos_uint8 picoktab_hasNonsyllvowelProp(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (KTAB_PPROP_NONSYLLVOWEL & ((ktabphones_SubObj)this)->props[ch]); |
| } |
| picoos_uint8 picoktab_hasSyllconsProp(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (KTAB_PPROP_SYLLCONS & ((ktabphones_SubObj)this)->props[ch]); |
| } |
| |
| picoos_bool picoktab_isSyllCarrier(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| picoos_uint8 props; |
| props = ((ktabphones_SubObj)this)->props[ch]; |
| return (((KTAB_PPROP_VOWEL & props) && |
| !(KTAB_PPROP_NONSYLLVOWEL & props)) |
| || (KTAB_PPROP_SYLLCONS & props)); |
| } |
| |
| picoos_bool picoktab_isPrimstress(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_PRIMSTRESS]); |
| } |
| picoos_bool picoktab_isSecstress(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_SECSTRESS]); |
| } |
| picoos_bool picoktab_isSyllbound(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_SYLLBOUND]); |
| } |
| picoos_bool picoktab_isWordbound(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_WORDBOUND]); |
| } |
| picoos_bool picoktab_isPause(const picoktab_Phones this, |
| const picoos_uint8 ch) { |
| return (ch == ((ktabphones_SubObj)this)->specids[KTAB_IND_PAUSE]); |
| } |
| |
| picoos_uint8 picoktab_getPrimstressID(const picoktab_Phones this) { |
| return ((ktabphones_SubObj)this)->specids[KTAB_IND_PRIMSTRESS]; |
| } |
| picoos_uint8 picoktab_getSecstressID(const picoktab_Phones this) { |
| return ((ktabphones_SubObj)this)->specids[KTAB_IND_SECSTRESS]; |
| } |
| picoos_uint8 picoktab_getSyllboundID(const picoktab_Phones this) { |
| return ((ktabphones_SubObj)this)->specids[KTAB_IND_SYLLBOUND]; |
| } |
| picoos_uint8 picoktab_getWordboundID(const picoktab_Phones this) { |
| return ((ktabphones_SubObj)this)->specids[KTAB_IND_WORDBOUND]; |
| } |
| picoos_uint8 picoktab_getPauseID(const picoktab_Phones this) { |
| return ((ktabphones_SubObj)this)->specids[KTAB_IND_PAUSE]; |
| } |
| |
| /* ************************************************************/ |
| /* Pos */ |
| /* ************************************************************/ |
| |
| /* overview binary file format for pos kb: |
| |
| pos-kb = header posids |
| header = {COUNT2 OFFS2}=8 |
| posids = {POSID1 {PARTID1}0:8}1: |
| |
| where POSID1 is the value of the (combined) part-of-speech symbol, |
| and {PARTID1} are the symbol values of its components (empty if it |
| is not a combined symbol). The {PARTID1} list is sorted. |
| Part-of-speech symbols with equal number of components are grouped |
| together. |
| |
| The header contains information about these groups: |
| |
| COUNT2 specifies the number of elements in the group, and OFFS2 |
| specifies the offset (relative to the beginning of the kb) where |
| the group data starts, i.e.: |
| |
| 25 32 -> 25 not-combined elements, starting at offset 32 |
| 44 57 -> 44 elements composed of 2 symbols, starting at offset 57 |
| 23 189 -> 23 elements composed of 3 symbols, starting at offset 189 |
| ... |
| |
| Currently, each symbol may be composed of up to 8 other symbols. |
| Therefore, the header has 8 entries, too. The header starts with |
| the unique POS list, and then in increasing order, 2 symbols, 3 |
| symbols,... |
| |
| Zur Anschauung die ge-printf-te Version: |
| |
| 25 32 |
| 44 57 |
| 23 189 |
| 12 281 |
| 4 341 |
| 1 365 |
| 0 0 |
| 0 0 |
| 33 | |
| 34 | |
| 35 | |
| 60 | |
| etc. |
| 36 | 35 60 |
| 50 | 35 95 |
| 51 | 35 97 |
| 58 | 35 120 |
| 59 | 35 131 |
| 61 | 60 75 |
| 63 | 60 95 |
| 64 | 60 97 |
| etc. |
| 42 | 35 60 117 |
| 44 | 35 60 131 |
| 45 | 35 73 97 |
| 48 | 35 84 97 |
| 54 | 35 97 131 |
| 56 | 35 113 120 |
| 57 | 35 117 120 |
| 62 | 60 84 122 |
| etc. |
| */ |
| |
| typedef struct ktabpos_subobj *ktabpos_SubObj; |
| |
| typedef struct ktabpos_subobj { |
| picoos_uint16 nrcomb[PICOKTAB_MAXNRPOS_IN_COMB]; |
| picoos_uint8 *nrcombstart[PICOKTAB_MAXNRPOS_IN_COMB]; |
| } ktabpos_subobj_t; |
| |
| |
| static pico_status_t ktabPosInitialize(register picoknow_KnowledgeBase this, |
| picoos_Common common) { |
| ktabpos_subobj_t *ktabpos; |
| picoos_uint16 osprev; |
| picoos_uint16 os, pos; |
| picoos_uint8 i; |
| |
| PICODBG_DEBUG(("start")); |
| |
| if (NULL == this || NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| ktabpos = (ktabpos_subobj_t *)this->subObj; |
| |
| os = 0; |
| for (i = 0, pos = 0; i < PICOKTAB_MAXNRPOS_IN_COMB; i++, pos += 4) { |
| ktabpos->nrcomb[i] = ((picoos_uint16)(this->base[pos+1])) << 8 | |
| this->base[pos]; |
| if (ktabpos->nrcomb[i] > 0) { |
| osprev = os; |
| os = ((picoos_uint16)(this->base[pos+3])) << 8 | this->base[pos+2]; |
| ktabpos->nrcombstart[i] = &(this->base[os]); |
| PICODBG_TRACE(("i %d, pos %d, nr %d, osprev %d, os %d", i, pos, |
| ktabpos->nrcomb[i], osprev, os)); |
| if (osprev >= os) { |
| /* cannot be, in a valid kb */ |
| return picoos_emRaiseException(common->em, |
| PICO_EXC_FILE_CORRUPT, |
| NULL, NULL); |
| } |
| } else { |
| if (i == 0) { |
| /* cannot be, in a valid kb */ |
| return picoos_emRaiseException(common->em, |
| PICO_EXC_FILE_CORRUPT, |
| NULL, NULL); |
| } |
| ktabpos->nrcombstart[i] = NULL; |
| } |
| } |
| return PICO_OK; |
| } |
| |
| static pico_status_t ktabPosSubObjDeallocate(register picoknow_KnowledgeBase this, |
| picoos_MemoryManager mm) { |
| if (NULL != this) { |
| picoos_deallocate(mm, (void *) &this->subObj); |
| } |
| return PICO_OK; |
| } |
| |
| pico_status_t picoktab_specializePosKnowledgeBase(picoknow_KnowledgeBase this, |
| picoos_Common common) { |
| if (NULL == this) { |
| return picoos_emRaiseException(common->em, PICO_EXC_KB_MISSING, |
| NULL, NULL); |
| } |
| this->subDeallocate = ktabPosSubObjDeallocate; |
| this->subObj = picoos_allocate(common->mm, sizeof(ktabpos_subobj_t)); |
| if (NULL == this->subObj) { |
| return picoos_emRaiseException(common->em, PICO_EXC_OUT_OF_MEM, |
| NULL, NULL); |
| } |
| return ktabPosInitialize(this, common); |
| } |
| |
| picoktab_Pos picoktab_getPos(picoknow_KnowledgeBase this) { |
| if (NULL == this) { |
| return NULL; |
| } else { |
| return (picoktab_Pos) this->subObj; |
| } |
| } |
| |
| |
| /* Pos methods */ |
| |
| static picoos_int16 ktab_isEqualPosGroup(const picoos_uint8 *grp1, |
| const picoos_uint8 *grp2, |
| picoos_uint8 len) |
| { |
| /* if both, grp1 and grp2 would be sorted in ascending order |
| we could implement a function picoktab_comparePosGroup in |
| a similar manner as strcmp */ |
| |
| picoos_uint16 i, j, equal; |
| |
| equal = 1; |
| |
| i = 0; |
| while (equal && (i < len)) { |
| /* search grp1[i] in grp2 */ |
| j = 0; |
| while ((j < len) && (grp1[i] != grp2[j])) { |
| j++; |
| } |
| equal = (j < len); |
| i++; |
| } |
| |
| return equal; |
| } |
| |
| |
| picoos_bool picoktab_isUniquePos(const picoktab_Pos this, |
| const picoos_uint8 pos) { |
| ktabpos_subobj_t *ktabpos; |
| picoos_uint16 i; |
| |
| /* speed-up possible with e.g. binary search */ |
| |
| ktabpos = (ktabpos_subobj_t *)this; |
| PICODBG_TRACE(("pos %d, nrcombinations %d", pos, ktabpos->nrcomb[0])); |
| i = 0; |
| while ((i < ktabpos->nrcomb[0]) && (pos > ktabpos->nrcombstart[0][i])) { |
| PICODBG_TRACE(("compare with pos %d at position %d", |
| ktabpos->nrcombstart[0][i], pos, i)); |
| i++; |
| } |
| return ((i < ktabpos->nrcomb[0]) && (pos == ktabpos->nrcombstart[0][i])); |
| } |
| |
| |
| picoos_bool picoktab_isPartOfPosGroup(const picoktab_Pos this, |
| const picoos_uint8 pos, |
| const picoos_uint8 posgroup) |
| { |
| ktabpos_subobj_t *ktabpos; |
| picoos_uint8 *grp; |
| picoos_uint16 i, j, n, s, grplen; |
| picoos_uint8 *e; |
| picoos_uint8 found; |
| |
| ktabpos = (ktabpos_subobj_t *) this; |
| |
| grp = NULL; |
| found = FALSE; |
| grplen = 0; |
| |
| /* currently, a linear search is required to find 'posgroup'; the |
| knowledge base should be extended to allow for a faster search */ |
| |
| /* treat case i==0, grplen==0, ie. pos == posgroup */ |
| if (pos == posgroup) { |
| found = TRUE; |
| } |
| |
| i = 1; |
| while ((grp == NULL) && (i < PICOKTAB_MAXNRPOS_IN_COMB)) { |
| n = ktabpos->nrcomb[i]; /* number of entries */ |
| e = ktabpos->nrcombstart[i]; /* ptr to first entry */ |
| s = i + 2; /* size of an entry in bytes */ |
| /* was with while starting at 0: |
| s = i > 0 ? i + 2 : 1; |
| */ |
| j = 0; |
| while ((grp == NULL) && (j < n)) { |
| if (posgroup == e[0]) { |
| grp = e + 1; |
| grplen = s - 1; |
| } |
| e += s; |
| j++; |
| } |
| i++; |
| } |
| |
| /* test if 'pos' is contained in the components of 'posgroup' */ |
| if (grp != NULL) { |
| for (i = 0; !found && (i < grplen); i++) { |
| if (pos == grp[i]) { |
| found = TRUE; |
| } |
| } |
| |
| /* just a way to test picoktab_getPosGroup */ |
| /* |
| PICODBG_ASSERT(picoktab_getPosGroup(this, grp, grplen) == posgroup); |
| */ |
| } |
| |
| return found; |
| } |
| |
| |
| picoos_uint8 picoktab_getPosGroup(const picoktab_Pos this, |
| const picoos_uint8 *poslist, |
| const picoos_uint8 poslistlen) |
| { |
| picoos_uint8 poscomb; |
| ktabpos_subobj_t *ktabpos; |
| picoos_uint16 i, j, n, s; |
| picoos_uint8 *e; |
| |
| ktabpos = (ktabpos_subobj_t *) this; |
| poscomb = 0; |
| |
| if ((poslistlen > 0) && (poslistlen <= PICOKTAB_MAXNRPOS_IN_COMB)) { |
| i = poslistlen - 1; |
| if (i > 0) { |
| n = ktabpos->nrcomb[i]; /* number of entries */ |
| e = ktabpos->nrcombstart[i]; /* ptr to first entry */ |
| s = i + 2; /* size of an entry in bytes */ |
| j = 0; |
| while (!poscomb && (j < n)) { |
| if (ktab_isEqualPosGroup(poslist, e + 1, poslistlen)) { |
| poscomb = *e; |
| } |
| e += s; |
| j++; |
| } |
| if (!poscomb) { |
| /* combination not found; shouldn't occur if lingware OK! */ |
| /* contingency solution: take first */ |
| PICODBG_WARN(("dynamically created POS combination not found in table; taking first (%i)",poslist[0])); |
| poscomb = poslist[0]; |
| } |
| } else { /* not a composed POS */ |
| poscomb = poslist[0]; |
| } |
| } |
| |
| return poscomb; |
| } |
| |
| #ifdef __cplusplus |
| } |
| #endif |
| |
| |
| /* end */ |
