/*
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * @file picotok.c
 *
 * tokenizer
 *
 * Copyright (C) 2008-2009 SVOX AG, Baslerstr. 30, 8048 Zuerich, Switzerland
 * All rights reserved.
 *
 * History:
 * - 2009-04-20 -- initial version
 *
 */


/* ************************************************************/
/* tokenisation and markup handling */
/* ************************************************************/

/** @addtogroup picotok
  @b tokenisation_overview

  markup handling overview:

  The following markups are recognized
     - ignore
     - speed
     - pitch
     - volume
     - voice
     - preproccontext
     - mark
     - play
     - usesig
     - genfile
     - sentence
     - s
     - paragraph
     - p
     - break
     - spell            (pauses between letter)
     - phoneme

  All markups which are recognized but are not yet implemented in pico
  system have the mark.
*/


#include "picodefs.h"
#include "picoos.h"
#include "picobase.h"
#include "picodbg.h"
#include "picodata.h"
#include "picotok.h"
#include "picoktab.h"

#ifdef __cplusplus
extern "C" {
#endif
#if 0
}
#endif

/* *****************************************************************************/

#define IN_BUF_SIZE   255
#define OUT_BUF_SIZE  IN_BUF_SIZE + 3 * PICODATA_ITEM_HEADSIZE + 3

#define MARKUP_STRING_BUF_SIZE (IN_BUF_SIZE*5)
#define MAX_NR_MARKUP_PARAMS 6
#define MARKUP_HANDLING_DISABLED  0
#define MARKUP_HANDLING_ENABLED 1
#define EOL '\n'


typedef picoos_int8 pico_tokenSubType;
typedef picoos_uint8 pico_tokenType;

/** @todo : consider adding these specialized exception codes: */

#define PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE PICO_ERR_OTHER
#define PICO_ERR_INVALID_MARKUP_TAG        PICO_ERR_OTHER
#define PICO_ERR_INTERNAL_LIMIT            PICO_ERR_OTHER

typedef enum {MIDummyStart, MIIgnore,
              MIPitch, MISpeed, MIVolume,
              MIVoice, MIPreprocContext, MIMarker,
              MIPlay, MIUseSig, MIGenFile, MIParagraph,
              MISentence, MIBreak, MISpell, MIPhoneme, MIItem, MISpeaker, MIDummyEnd
             }  MarkupId;
typedef enum {MSNotInMarkup, MSGotStart, MSExpectingmarkupTagName, MSInmarkupTagName,
              MSGotmarkupTagName, MSInAttrName, MSGotAttrName, MSGotEqual, MSInAttrValue,
              MSInAttrValueEscaped, MSGotAttrValue, MSGotEndSlash, MSGotEnd,
              MSError, MSErrorTooLong, MSErrorSyntax
             }  MarkupState;
typedef enum {MENone, MEMissingStart, MEUnknownTag, MEIdent, MEMissingEqual,
              MEMissingQuote, MEMissingEnd, MEUnexpectedChar, MEInterprete
             }  MarkupParseError;

typedef enum {MTNone, MTStart, MTEnd, MTEmpty} MarkupTagType;

#define UTF_CHAR_COMPLETE   2
#define UTF_CHAR_INCOMPLETE 1
#define UTF_CHAR_MALFORMED  0

#define TOK_MARKUP_KW_IGNORE     (picoos_uchar*)"ignore"
#define TOK_MARKUP_KW_SPEED      (picoos_uchar*)"speed"
#define TOK_MARKUP_KW_PITCH      (picoos_uchar*)"pitch"
#define TOK_MARKUP_KW_VOLUME     (picoos_uchar*)"volume"
#define TOK_MARKUP_KW_VOICE      (picoos_uchar*)"voice"
#define TOK_MARKUP_KW_CONTEXT    (picoos_uchar*)"preproccontext"
#define TOK_MARKUP_KW_MARK       (picoos_uchar*)"mark"
#define TOK_MARKUP_KW_PLAY       (picoos_uchar*)"play"
#define TOK_MARKUP_KW_USESIG     (picoos_uchar*)"usesig"
#define TOK_MARKUP_KW_GENFILE    (picoos_uchar*)"genfile"
#define TOK_MARKUP_KW_SENTENCE   (picoos_uchar*)"sentence"
#define TOK_MARKUP_KW_S          (picoos_uchar*)"s"
#define TOK_MARKUP_KW_PARAGRAPH  (picoos_uchar*)"paragraph"
#define TOK_MARKUP_KW_P          (picoos_uchar*)"p"
#define TOK_MARKUP_KW_BREAK      (picoos_uchar*)"break"
#define TOK_MARKUP_KW_SPELL      (picoos_uchar*)"spell"
#define TOK_MARKUP_KW_PHONEME    (picoos_uchar*)"phoneme"
#define TOK_MARKUP_KW_ITEM       (picoos_uchar*)"item"
#define TOK_MARKUP_KW_SPEAKER    (picoos_uchar*)"speaker"

#define KWLevel (picoos_uchar *)"level"
#define KWName (picoos_uchar *)"name"
#define KWProsDomain (picoos_uchar *)"prosodydomain"
#define KWTime (picoos_uchar *)"time"
#define KWMode (picoos_uchar *)"mode"
#define KWSB (picoos_uchar *)"sb"
#define KWPB (picoos_uchar *)"pb"
#define KWFile (picoos_uchar *)"file"
#define KWType (picoos_uchar *)"type"
#define KWF0Beg (picoos_uchar *)"f0beg"
#define KWF0End (picoos_uchar *)"f0end"
#define KWXFadeBeg (picoos_uchar *)"xfadebeg"
#define KWXFadeEnd (picoos_uchar *)"xfadeend"
#define KWAlphabet (picoos_uchar *)"alphabet"
#define KWPH (picoos_uchar *)"ph"
#define KWOrthMode (picoos_uchar *)"orthmode"
#define KWIgnorePunct (picoos_uchar *)"ignorepunct"
#define KWInfo1 (picoos_uchar *)"info1"
#define KWInfo2 (picoos_uchar *)"info2"
#define KWDATA (picoos_uchar *)"data"

#define PICO_SPEED_MIN           20
#define PICO_SPEED_MAX          500
#define PICO_SPEED_DEFAULT      100
#define PICO_SPEED_FACTOR_MIN   500
#define PICO_SPEED_FACTOR_MAX  2000

#define PICO_PITCH_MIN           50
#define PICO_PITCH_MAX          200
#define PICO_PITCH_DEFAULT      100
#define PICO_PITCH_FACTOR_MIN   500
#define PICO_PITCH_FACTOR_MAX  2000
#define PICO_PITCH_ADD_MIN     -100
#define PICO_PITCH_ADD_MAX      100
#define PICO_PITCH_ADD_DEFAULT    0

#define PICO_VOLUME_MIN           0
#define PICO_VOLUME_MAX         500
#define PICO_VOLUME_DEFAULT     100
#define PICO_VOLUME_FACTOR_MIN  500
#define PICO_VOLUME_FACTOR_MAX 2000

#define PICO_SPEAKER_MIN          20
#define PICO_SPEAKER_MAX         180
#define PICO_SPEAKER_DEFAULT     100
#define PICO_SPEAKER_FACTOR_MIN  500
#define PICO_SPEAKER_FACTOR_MAX 2000

#define PICO_CONTEXT_DEFAULT   (picoos_uchar*)"DEFAULT"

#define PARAGRAPH_PAUSE_DUR 500
#define SPELL_WITH_PHRASE_BREAK  1
#define SPELL_WITH_SENTENCE_BREAK  2

/* *****************************************************************************/

#define TOK_PUNC_FLUSH  (picoos_char) '\0'

typedef picoos_uchar Word[MARKUP_STRING_BUF_SIZE];


struct MarkupParam {
    Word paramId;
    Word paramVal;
};

typedef struct MarkupParam MarkupParams[MAX_NR_MARKUP_PARAMS];


/** subobject : TokenizeUnit
 *  shortcut  : tok
 */
typedef struct tok_subobj
{
    picoos_int32 ignLevel;

    picoos_uchar utf[5];
    picoos_int32 utfpos;
    picoos_int32 utflen;

    MarkupParams markupParams;
    picoos_int32 nrMarkupParams;
    MarkupState markupState;
    picoos_uchar markupStr[MARKUP_STRING_BUF_SIZE];
    picoos_int32 markupPos;
    picoos_int32 markupLevel[MIDummyEnd+1];
    picoos_uchar markupTagName[IN_BUF_SIZE];
    MarkupTagType markupTagType;
    MarkupParseError markupTagErr;

    picoos_int32 strPos;
    picoos_uchar strDelim;
    picoos_bool isFileAttr;

    pico_tokenType tokenType;
    pico_tokenSubType tokenSubType;

    picoos_int32 tokenPos;
    picoos_uchar tokenStr[IN_BUF_SIZE];

    picoos_int32 nrEOL;

    picoos_bool markupHandlingMode;       /* to be moved ??? */
    picoos_bool aborted;                  /* to be moved ??? */

    picoos_bool start;

    picoos_uint8 outBuf[OUT_BUF_SIZE]; /* internal output buffer */
    picoos_uint16 outReadPos; /* next pos to read from outBuf */
    picoos_uint16 outWritePos; /* next pos to write to outBuf */

    picoos_uchar saveFile[IN_BUF_SIZE];
    Word phonemes;

    picotrns_SimpleTransducer transducer;

    /* kbs */

    picoktab_Graphs graphTab;
    picokfst_FST xsampa_parser;
    picokfst_FST svoxpa_parser;
    picokfst_FST xsampa2svoxpa_mapper;


} tok_subobj_t;

/* *****************************************************************************/

static void tok_treatMarkupAsSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok);
static void tok_treatChar (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar ch, picoos_bool markupHandling);
static void tok_treatMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok);
static void tok_putToMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar str[]);
static void tok_treatSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok);
static MarkupId tok_markupTagId (picoos_uchar tagId[]);

/* *****************************************************************************/

static picoos_bool tok_strEqual(picoos_uchar * str1, picoos_uchar * str2)
{
   return (picoos_strcmp((picoos_char*)str1, (picoos_char*)str2) == 0);
}

static void tok_reduceBlanks(picoos_uchar * str)
            /* Remove leading and trailing blanks of 'str' and reduce
               groups of blanks within string to exactly one blank. */

{
    int i = 0;
    int j = 0;

     while (str[j] != 0) {
        if (str[j] == (picoos_uchar)' ') {
            /* note one blank except at the beginning of string */
            if (i > 0) {
                str[i] = (picoos_uchar)' ';
                i++;
            }
            j++;
            while (str[j] == (picoos_uchar)' ') {
                j++;
            }
        } else {
            str[i] = str[j];
            j++;
            i++;
        }
    }

    /* remove blanks at end of string */
    if ((i > 0) && (str[i - 1] == ' ')) {
        i--;
    }
    str[i] = 0;
}


static void tok_startIgnore (tok_subobj_t * tok)
{
    tok->ignLevel++;
}


static void tok_endIgnore (tok_subobj_t * tok)
{
    if (tok->ignLevel > 0) {
        tok->ignLevel--;
    }
}


static void tok_getParamIntVal (MarkupParams params, picoos_uchar paramId[], picoos_int32 * paramVal, picoos_bool * paramFound)
{
    int i=0;

    while ((i < MAX_NR_MARKUP_PARAMS) && !tok_strEqual(paramId,params[i].paramId)) {
        i++;
    }
    if ((i < MAX_NR_MARKUP_PARAMS)) {
        (*paramVal) = picoos_atoi((picoos_char*)params[i].paramVal);
        (*paramFound) = TRUE;
    } else {
        (*paramVal) =  -1;
        (*paramFound) = FALSE;
    }
}


static void tok_getParamStrVal (MarkupParams params, picoos_uchar paramId[], picoos_uchar paramStrVal[], picoos_bool * paramFound)
{
    int i=0;

    while ((i < MAX_NR_MARKUP_PARAMS) &&  !tok_strEqual(paramId,params[i].paramId)) {
        i++;
    }
    if (i < MAX_NR_MARKUP_PARAMS) {
        picoos_strcpy((picoos_char*)paramStrVal, (picoos_char*)params[i].paramVal);
        (*paramFound) = TRUE;
    } else {
        paramStrVal[0] = 0;
        (*paramFound) = FALSE;
    }
}


static void tok_getParamPhonesStr (MarkupParams params, picoos_uchar paramId[], picoos_uchar alphabet[], picoos_uchar phones[], picoos_int32 phoneslen, picoos_bool * paramFound)
{

    int i;
    picoos_bool done;

    i = 0;
    while ((i < MAX_NR_MARKUP_PARAMS) &&  !tok_strEqual(paramId, params[i].paramId)) {
        i++;
    }
    if (i < MAX_NR_MARKUP_PARAMS) {
        if (tok_strEqual(alphabet, PICODATA_XSAMPA) || tok_strEqual(alphabet, (picoos_uchar*)"")) {
            picoos_strlcpy((picoos_char*)phones, (picoos_char*)params[i].paramVal, phoneslen);
            done = TRUE;
        } else {
            done = FALSE;
        }
        (*paramFound) = TRUE;
    } else {
        done = FALSE;
        (*paramFound) = FALSE;
    }
    if (!done) {
        phones[0] = 0;
    }
}


static void tok_clearMarkupParams (MarkupParams params)
{
    int i;

    for (i = 0; i<MAX_NR_MARKUP_PARAMS; i++) {
        params[i].paramId[0] = 0;
        params[i].paramVal[0] = 0;
    }
}


static void tok_getDur (picoos_uchar durStr[], picoos_uint32 * dur, picoos_bool * done)
{

    int num=0;
    int i=0;
    picoos_uchar tmpWord[IN_BUF_SIZE];

    picoos_strlcpy((picoos_char*)tmpWord, (picoos_char*)durStr, sizeof(tmpWord));
    tok_reduceBlanks(tmpWord);
    while ((durStr[i] >= '0') && (durStr[i] <= '9')) {
        num = 10 * num + (int)durStr[i] - (int)'0';
        tmpWord[i] = ' ';
        i++;
    }
    tok_reduceBlanks(tmpWord);
    if (tok_strEqual(tmpWord, (picoos_uchar*)"s")) {
        (*dur) = (1000 * num);
        (*done) = TRUE;
    } else if (tok_strEqual(tmpWord,(picoos_uchar*)"ms")) {
        (*dur) = num;
        (*done) = TRUE;
    } else {
        (*dur) = 0;
        (*done) = FALSE;
    }
}


static picoos_int32 tok_putToUtf (tok_subobj_t * tok, picoos_uchar ch)
{
    if (tok->utfpos < PICOBASE_UTF8_MAXLEN) {
        tok->utf[tok->utfpos] = ch;
        if (tok->utfpos == 0) {
            tok->utflen = picobase_det_utf8_length(ch);
        } else if (((ch < (picoos_uchar)'\200') || (ch >= (picoos_uchar)'\300'))) {
            tok->utflen = 0;
        }
        (tok->utfpos)++;
        if ((tok->utfpos == tok->utflen)) {
            if ((tok->utfpos < PICOBASE_UTF8_MAXLEN)) {
                tok->utf[tok->utfpos] = 0;
            }
            return UTF_CHAR_COMPLETE;
        } else if (tok->utfpos < tok->utflen) {
            return UTF_CHAR_INCOMPLETE;
        } else {
            return UTF_CHAR_MALFORMED;
        }
    } else {
        return UTF_CHAR_MALFORMED;
    }
}


static picoos_bool tok_isRelative (picoos_uchar strval[], picoos_uint32 * val)
{
    picoos_int32 len;
    picoos_bool rel;

    rel = FALSE;
    len = picoos_strlen((picoos_char*)strval);
    if (len > 0) {
        if (strval[len - 1] == '%') {
            strval[len - 1] = 0;
            if ((strval[0] == '+') || (strval[0] == '-')) {
                (*val) = 1000 + (picoos_atoi((picoos_char*)strval) * 10);
            } else {
                (*val) = picoos_atoi((picoos_char*)strval) * 10;
            }
            rel = TRUE;
        }
    }
    return rel;
}


static void tok_putItem (picodata_ProcessingUnit this,  tok_subobj_t * tok,
                         picoos_uint8 itemType, picoos_uint8 info1, picoos_uint8 info2,
                         picoos_uint16 val,
                         picoos_uchar str[])
{
    picoos_int32 len, i;

    if ((itemType == PICODATA_ITEM_CMD) && (info1 == PICODATA_ITEMINFO1_CMD_FLUSH)) {
        tok->outBuf[tok->outWritePos++] = itemType;
        tok->outBuf[tok->outWritePos++] = info1;
        tok->outBuf[tok->outWritePos++] = info2;
        tok->outBuf[tok->outWritePos++] = 0;
    }
    else if (tok->ignLevel <= 0) {
        switch (itemType) {
        case PICODATA_ITEM_CMD:
            switch (info1) {
            case PICODATA_ITEMINFO1_CMD_CONTEXT:
            case PICODATA_ITEMINFO1_CMD_VOICE:
            case PICODATA_ITEMINFO1_CMD_MARKER:
            case PICODATA_ITEMINFO1_CMD_PLAY:
            case PICODATA_ITEMINFO1_CMD_SAVE:
            case PICODATA_ITEMINFO1_CMD_UNSAVE:
            case PICODATA_ITEMINFO1_CMD_PROSDOMAIN:
            case PICODATA_ITEMINFO1_CMD_PHONEME:
                len = picoos_strlen((picoos_char*)str);
                if (tok->outWritePos + 4 + len < OUT_BUF_SIZE) {
                    tok->outBuf[tok->outWritePos++] = itemType;
                    tok->outBuf[tok->outWritePos++] = info1;
                    tok->outBuf[tok->outWritePos++] = info2;
                    tok->outBuf[tok->outWritePos++] = len;
                    for (i=0; i<len; i++) {
                        tok->outBuf[tok->outWritePos++] = str[i];
                    }
                }
                else {
                    PICODBG_WARN(("tok_putItem: output buffer too small"));
                }
                break;
            case PICODATA_ITEMINFO1_CMD_IGNSIG:
            case PICODATA_ITEMINFO1_CMD_IGNORE:
                if (tok->outWritePos + 4 < OUT_BUF_SIZE) {
                    tok->outBuf[tok->outWritePos++] = itemType;
                    tok->outBuf[tok->outWritePos++] = info1;
                    tok->outBuf[tok->outWritePos++] = info2;
                    tok->outBuf[tok->outWritePos++] = 0;
                }
                else {
                    PICODBG_WARN(("tok_putItem: output buffer too small"));
                }
                break;
            case PICODATA_ITEMINFO1_CMD_SPEED:
            case PICODATA_ITEMINFO1_CMD_PITCH:
            case PICODATA_ITEMINFO1_CMD_VOLUME:
            case PICODATA_ITEMINFO1_CMD_SPELL:
            case PICODATA_ITEMINFO1_CMD_SIL:
            case PICODATA_ITEMINFO1_CMD_SPEAKER:
                if (tok->outWritePos + 4 + 2 < OUT_BUF_SIZE) {
                    tok->outBuf[tok->outWritePos++] = itemType;
                    tok->outBuf[tok->outWritePos++] = info1;
                    tok->outBuf[tok->outWritePos++] = info2;
                    tok->outBuf[tok->outWritePos++] = 2;
                    tok->outBuf[tok->outWritePos++] = val % 256;
                    tok->outBuf[tok->outWritePos++] = val / 256;
                }
                else {
                    PICODBG_WARN(("tok_putItem: output buffer too small"));
                }
                break;
            default:
                PICODBG_WARN(("tok_putItem: unknown command type"));
            }
            break;
        case PICODATA_ITEM_TOKEN:
            len = picoos_strlen((picoos_char*)str);
            if (tok->outWritePos + 4 + len < OUT_BUF_SIZE) {
                tok->outBuf[tok->outWritePos++] = itemType;
                tok->outBuf[tok->outWritePos++] = info1;
                tok->outBuf[tok->outWritePos++] = info2;
                tok->outBuf[tok->outWritePos++] = len;
                for (i=0; i<len; i++) {
                    tok->outBuf[tok->outWritePos++] = str[i];
                }
            }
            else {
                PICODBG_WARN(("tok_putItem: output buffer too small"));
            }
            break;
        default:
            PICODBG_WARN(("tok_putItem: unknown item type"));
        }
    }
}


static void tok_putItem2 (picodata_ProcessingUnit this,  tok_subobj_t * tok,
                          picoos_uint8 type,
                          picoos_uint8 info1, picoos_uint8 info2,
                          picoos_uint8 len,
                          picoos_uint8 data[])
{
    picoos_int32 i;

    if (is_valid_itemtype(type)) {
        tok->outBuf[tok->outWritePos++] = type;
        tok->outBuf[tok->outWritePos++] = info1;
        tok->outBuf[tok->outWritePos++] = info2;
        tok->outBuf[tok->outWritePos++] = len;
        for (i=0; i<len; i++) {
            tok->outBuf[tok->outWritePos++] = data[i];
        }
    }
}


static MarkupId tok_markupTagId (picoos_uchar tagId[])
{
    if (picoos_strstr(tagId,(picoos_char *)"svox:") == (picoos_char *)tagId) {
        tagId+=5;
    }
    if (tok_strEqual(tagId, TOK_MARKUP_KW_IGNORE)) {
        return MIIgnore;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SPEED)) {
        return MISpeed;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PITCH)) {
        return MIPitch;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_VOLUME)) {
        return MIVolume;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SPEAKER)) {
        return MISpeaker;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_VOICE)) {
        return MIVoice;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_CONTEXT)) {
        return MIPreprocContext;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_MARK)) {
        return MIMarker;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PLAY)) {
        return MIPlay;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_USESIG)) {
        return MIUseSig;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_GENFILE)) {
        return MIGenFile;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SENTENCE) || tok_strEqual(tagId, TOK_MARKUP_KW_S)) {
        return MISentence;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PARAGRAPH) || tok_strEqual(tagId, TOK_MARKUP_KW_P)) {
        return MIParagraph;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_BREAK)) {
        return MIBreak;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_SPELL)) {
        return MISpell;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_PHONEME)) {
        return MIPhoneme;
    } else if (tok_strEqual(tagId, TOK_MARKUP_KW_ITEM)) {
        return MIItem;
    } else {
        return MIDummyEnd;
    }
}


extern void tok_checkLimits (picodata_ProcessingUnit this, picoos_uint32 * value, picoos_uint32 min, picoos_uint32 max, picoos_uchar valueType[])
{
    if ((((*value) < min) || ((*value) > max))) {
        picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE, (picoos_char*)"", (picoos_char*)"attempt to set illegal value %i for %s", *value, valueType);
        if (((*value) < min)) {
            (*value) = min;
        } else if (((*value) > max)) {
            (*value) = max;
        }
    }
}


extern void tok_checkRealLimits (picodata_ProcessingUnit this, picoos_single * value, picoos_single min, picoos_single max, picoos_uchar valueType[])
{
    if ((((*value) < min) || ((*value) > max))) {
          picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE, (picoos_char*)"", (picoos_char*)"attempt to set illegal value %f for %s", *value, valueType);
        if (((*value) < min)) {
            (*value) = min;
        } else if (((*value) > max)) {
            (*value) = max;
        }
    }
}


#define VAL_STR_LEN 21

static void tok_interpretMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_bool isStartTag, MarkupId mId)
{
    picoos_bool done;
    picoos_int32 ival;
    picoos_uint32 uval;
    picoos_int32 ival2;
    picoos_uchar valStr[VAL_STR_LEN];
    picoos_uchar valStr2[VAL_STR_LEN];
    picoos_uchar valStr3[VAL_STR_LEN];
    picoos_int32 i2;
    picoos_uint32 dur;
    picoos_bool done1;
    picoos_bool paramFound;
    picoos_uint8 type, info1, info2;
    picoos_uint8 data[256];
    picoos_int32 pos, n, len;
    picoos_uchar part[10];

    done = FALSE;
    switch (mId) {
        case MIIgnore:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) {
                tok_startIgnore(tok);
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_endIgnore(tok);
                done = TRUE;
            }
            break;
        case MISpeed:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) {
                if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) {
                    tok_checkLimits(this, & uval, PICO_SPEED_FACTOR_MIN, PICO_SPEED_FACTOR_MAX,(picoos_uchar*)"relative speed factor");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEED, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)"");
                } else {
                    uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal);
                    tok_checkLimits(this, & uval, PICO_SPEED_MIN, PICO_SPEED_MAX,(picoos_uchar*)"speed");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEED, PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)"");
                }
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEED, PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_SPEED_DEFAULT, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIPitch:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) {
                if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) {
                    tok_checkLimits(this, & uval,PICO_PITCH_FACTOR_MIN,PICO_PITCH_FACTOR_MAX, (picoos_uchar*)"relative pitch factor");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PITCH, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)"");
                } else {
                    uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal);
                    tok_checkLimits(this, & uval,PICO_PITCH_MIN,PICO_PITCH_MAX, (picoos_uchar*)"pitch");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PITCH,PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)"");
                }
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PITCH,PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_PITCH_DEFAULT, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIVolume:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) {
                if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) {
                    tok_checkLimits(this, & uval, PICO_VOLUME_FACTOR_MIN, PICO_VOLUME_FACTOR_MAX, (picoos_uchar*)"relative volume factor");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOLUME, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)"");
                } else {
                    uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal);
                    tok_checkLimits(this, & uval, PICO_VOLUME_MIN, PICO_VOLUME_MAX, (picoos_uchar*)"volume");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOLUME, PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)"");
                }
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOLUME, PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_VOLUME_DEFAULT, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MISpeaker:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWLevel)) {
                if (tok_isRelative(tok->markupParams[0].paramVal, & uval)) {
                    tok_checkLimits(this, & uval, PICO_SPEAKER_FACTOR_MIN, PICO_SPEAKER_FACTOR_MAX, (picoos_uchar*)"relative speaker factor");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEAKER, PICODATA_ITEMINFO2_CMD_RELATIVE, uval, (picoos_uchar*)"");
                } else {
                    uval = picoos_atoi((picoos_char*)tok->markupParams[0].paramVal);
                    tok_checkLimits(this, & uval, PICO_SPEAKER_MIN, PICO_SPEAKER_MAX, (picoos_uchar*)"volume");
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEAKER, PICODATA_ITEMINFO2_CMD_ABSOLUTE, uval, (picoos_uchar*)"");
                }
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPEAKER, PICODATA_ITEMINFO2_CMD_ABSOLUTE, PICO_SPEAKER_DEFAULT, (picoos_uchar*)"");
                done = TRUE;
            }
            break;

        case MIVoice:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWName)) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOICE, PICODATA_ITEMINFO2_NA, 0, tok->markupParams[0].paramVal);
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 0, 0, (picoos_uchar*)"");
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_VOICE, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 0, 0, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIPreprocContext:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWName)) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_CONTEXT, PICODATA_ITEMINFO2_NA, 0, tok->markupParams[0].paramVal);
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_CONTEXT, PICODATA_ITEMINFO2_NA, 0, PICO_CONTEXT_DEFAULT);
                done = TRUE;
            }
            break;
        case MIMarker:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWName)) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_MARKER, PICODATA_ITEMINFO2_NA, 0, tok->markupParams[0].paramVal);
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) {
                done = TRUE;
            }
            break;
        case MISentence:
            if (isStartTag) {
                tok_getParamStrVal(tok->markupParams, KWProsDomain, (picoos_uchar*)valStr, & paramFound);
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 2, 0, valStr);
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 2, 0, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIParagraph:
            if (isStartTag) {
                tok_getParamStrVal(tok->markupParams, KWProsDomain, (picoos_uchar*)valStr, & paramFound);
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 1, 0, valStr);
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, PARAGRAPH_PAUSE_DUR, (picoos_uchar*)"");
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PROSDOMAIN, 1, 0, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIBreak:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWTime)) {
                tok_getDur(tok->markupParams[0].paramVal, & dur, & done1);
                tok_checkLimits (this, &dur, 0, 65535, (picoos_uchar*)"time");
                if (done1) {
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SIL, PICODATA_ITEMINFO2_NA, dur, (picoos_uchar*)"");
                    done = TRUE;
                }
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                done = TRUE;
            }
            break;
        case MISpell:
            if (isStartTag) {
                if (tok_strEqual(tok->markupParams[0].paramId, KWMode)) {
                    if (tok_strEqual(tok->markupParams[0].paramVal, KWPB)) {
                        uval = SPELL_WITH_PHRASE_BREAK;
                    } else if (tok_strEqual(tok->markupParams[0].paramVal, KWSB)) {
                        uval = SPELL_WITH_SENTENCE_BREAK;
                    } else {
                        tok_getDur(tok->markupParams[0].paramVal, & uval, & done1);
                        tok_checkLimits (this, & uval, 0, 65535, (picoos_uchar*)"time");
                        if (done1) {
                            done = TRUE;
                        }
                    }
                } else {
                    uval = SPELL_WITH_PHRASE_BREAK;
                }
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPELL, PICODATA_ITEMINFO2_CMD_START, uval, (picoos_uchar*)"");
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SPELL, PICODATA_ITEMINFO2_CMD_END, 0, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIGenFile:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWFile)) {
                if (tok->saveFile[0] != 0) {
                   tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
                               picodata_getPuTypeFromExtension(tok->saveFile, /*input*/FALSE), 0, tok->saveFile);
                   tok->saveFile[0] = 0;
                }
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_SAVE,
                            picodata_getPuTypeFromExtension(tok->markupParams[0].paramVal,  /*input*/FALSE), 0, tok->markupParams[0].paramVal);
                picoos_strcpy((picoos_char*)tok->saveFile, (picoos_char*)tok->markupParams[0].paramVal);
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                if (tok->saveFile[0] != 0) {
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_UNSAVE,
                                picodata_getPuTypeFromExtension(tok->saveFile, /*input*/FALSE), 0, (picoos_uchar*)"");
                    tok->saveFile[0] = 0;
                }
                done = TRUE;
            }
            break;
        case MIPlay:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWFile)) {
                if (picoos_FileExists(this->common, (picoos_char*)tok->markupParams[0].paramVal)) {
                    tok_getParamIntVal(tok->markupParams,KWF0Beg,& ival,& paramFound);
                    tok_getParamIntVal(tok->markupParams,KWF0End,& ival2,& paramFound);
                    tok_getParamStrVal(tok->markupParams,KWAlphabet,valStr3,& paramFound);
                    tok_getParamPhonesStr(tok->markupParams,KWXFadeBeg,valStr3,valStr,VAL_STR_LEN,& paramFound);
                    tok_getParamPhonesStr(tok->markupParams,KWXFadeEnd,valStr3,valStr2,VAL_STR_LEN,& paramFound);
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PLAY,
                                picodata_getPuTypeFromExtension(tok->markupParams[0].paramVal, /*input*/TRUE), 0, tok->markupParams[0].paramVal);
                    tok_startIgnore(tok);
                } else {
                    if (tok->ignLevel > 0) {
                        tok_startIgnore(tok);
                    } else {
                       picoos_emRaiseWarning(this->common->em, PICO_EXC_CANT_OPEN_FILE, (picoos_char*)"", (picoos_char*)"file '%s' not found; synthesizing enclosed text instead\n", tok->markupParams[0].paramVal);
                    }
                }
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_endIgnore(tok);
                done = TRUE;
            }
            break;
        case MIUseSig:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWFile)) {
                if (picoos_FileExists(this->common, (picoos_char*)tok->markupParams[0].paramVal)) {
                    tok_getParamIntVal(tok->markupParams,KWF0Beg,& ival,& paramFound);
                    tok_getParamIntVal(tok->markupParams,KWF0End,& ival2,& paramFound);
                    tok_getParamStrVal(tok->markupParams,KWAlphabet,valStr3, & paramFound);
                    tok_getParamPhonesStr(tok->markupParams,KWXFadeBeg,valStr3,valStr,VAL_STR_LEN,& paramFound);
                    tok_getParamPhonesStr(tok->markupParams,KWXFadeEnd,valStr3,valStr2,VAL_STR_LEN,& paramFound);
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PLAY,
                                picodata_getPuTypeFromExtension(tok->markupParams[0].paramVal, /*input*/TRUE), 0, tok->markupParams[0].paramVal);
                    tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNSIG, PICODATA_ITEMINFO2_CMD_START, 0, (picoos_uchar*)"");
                } else {
                    if (tok->ignLevel <= 0) {
                        picoos_emRaiseWarning(this->common->em, PICO_EXC_CANT_OPEN_FILE, (picoos_char*)"", (picoos_char*)"file '%s' not found; synthesizing enclosed text instead", tok->markupParams[0].paramVal);
                    }
                }
                done = TRUE;
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_IGNSIG, PICODATA_ITEMINFO2_CMD_END, 0, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIPhoneme:
            i2 = 0;
            if (isStartTag) {
                if (tok_strEqual(tok->markupParams[0].paramId, KWAlphabet) && tok_strEqual(tok->markupParams[1].paramId, KWPH)) {
                    if (tok_strEqual(tok->markupParams[2].paramId, KWOrthMode)
                        && tok_strEqual(tok->markupParams[2].paramVal, KWIgnorePunct)) {
                        i2 = 1;
                    }
                    if (picodata_mapPAStrToPAIds(tok->transducer, this->common, tok->xsampa_parser, tok->svoxpa_parser, tok->xsampa2svoxpa_mapper, tok->markupParams[1].paramVal, tok->markupParams[0].paramVal, tok->phonemes, sizeof(tok->phonemes)-1) == PICO_OK) {
                        tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME,
                            PICODATA_ITEMINFO2_CMD_START, i2, tok->phonemes);
                        done = TRUE;
                    } else {
                        PICODBG_WARN(("cannot map phonetic string '%s'; synthesizeing text instead", tok->markupParams[1].paramVal));
                        picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE,(picoos_char*)"", (picoos_char*)"cannot map phonetic string '%s'; synthesizeing text instead", tok->markupParams[1].paramVal);
                        done = TRUE;
                    }
                } else if (tok_strEqual(tok->markupParams[0].paramId, KWPH)) {
                    if (tok_strEqual(tok->markupParams[1].paramId, KWOrthMode)
                        && tok_strEqual(tok->markupParams[1].paramVal, KWIgnorePunct)) {
                        i2 = 1;
                    }
                    if (picodata_mapPAStrToPAIds(tok->transducer, this->common, tok->xsampa_parser, tok->svoxpa_parser, tok->xsampa2svoxpa_mapper, tok->markupParams[0].paramVal, PICODATA_XSAMPA, tok->phonemes, sizeof(tok->phonemes)) == PICO_OK) {
                        tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME,
                            PICODATA_ITEMINFO2_CMD_START, i2, tok->phonemes);
                        done = TRUE;
                    }
                    else {
                        PICODBG_WARN(("cannot map phonetic string '%s'; synthesizeing text instead", tok->markupParams[1].paramVal));
                        picoos_emRaiseWarning(this->common->em, PICO_ERR_MARKUP_VALUE_OUT_OF_RANGE,(picoos_char*)"", (picoos_char*)"cannot map phonetic string '%s'; synthesizing text instead", tok->markupParams[0].paramVal);
                        done = TRUE;
                    }
                }
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId, (picoos_uchar*)"")) {
                tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_PHONEME,
                    PICODATA_ITEMINFO2_CMD_END, i2, (picoos_uchar*)"");
                done = TRUE;
            }
            break;
        case MIItem:
            if (isStartTag && tok_strEqual(tok->markupParams[0].paramId, KWType) &&
                              tok_strEqual(tok->markupParams[1].paramId, KWInfo1)&&
                              tok_strEqual(tok->markupParams[2].paramId, KWInfo2)&&
                              tok_strEqual(tok->markupParams[3].paramId, KWDATA)) {
                  picoos_int32 len2, n2;
                  type = picoos_atoi(tok->markupParams[0].paramVal);
                  info1 = picoos_atoi(tok->markupParams[1].paramVal);
                  info2 = picoos_atoi(tok->markupParams[2].paramVal);
                  n = 0; n2 = 0;
                  len2 = (picoos_int32)picoos_strlen(tok->markupParams[3].paramVal);
                  while (n<len2) {
                      while ((tok->markupParams[3].paramVal[n] != 0) && (tok->markupParams[3].paramVal[n] <= 32)) {
                          n++;
                      }
                      tok->markupParams[3].paramVal[n2] = tok->markupParams[3].paramVal[n];
                      n++;
                      n2++;
                  }
                  if (is_valid_itemtype(type)) {
                      done = TRUE;
                      len = 0;
                      pos = 0;
                      picoos_get_sep_part_str(tok->markupParams[3].paramVal, picoos_strlen(tok->markupParams[3].paramVal),
                                          &pos, ',', part, 10, &done1);
                      while (done && done1) {
                          n = picoos_atoi(part);
                          if ((n>=0) && (n<256) && (len<256)) {
                              data[len++] = n;
                          }
                          else {
                              done = FALSE;
                          }
                          picoos_get_sep_part_str(tok->markupParams[3].paramVal, picoos_strlen(tok->markupParams[3].paramVal),
                                          &pos, ',', part, 10, &done1);
                      }
                      if (done) {
                          tok_putItem2(this, tok, type, info1, info2, len, data);
                      }
                  }
                  else {
                      done = FALSE;
                  }
            } else if (!isStartTag && tok_strEqual(tok->markupParams[0].paramId,(picoos_uchar*)"")) {
                done = TRUE;
            }
            break;
    default:
        break;
    }
    if (!done) {
        tok->markupTagErr = MEInterprete;
    }
    if (isStartTag) {
        tok->markupLevel[mId]++;
    } else if ((tok->markupLevel[mId] > 0)) {
        tok->markupLevel[mId]--;
    }
}


static picoos_bool tok_attrChar (picoos_uchar ch, picoos_bool first)
{
    return ((((ch >= (picoos_uchar)'A') && (ch <= (picoos_uchar)'Z')) ||
             ((ch >= (picoos_uchar)'a') && (ch <= (picoos_uchar)'z'))) ||
             ( !(first) && ((ch >= (picoos_uchar)'0') && (ch <= (picoos_uchar)'9'))));
}


static picoos_bool tok_idChar (picoos_uchar ch, picoos_bool first)
{
    return tok_attrChar(ch, first) || ( !(first) && (ch == (picoos_uchar)':'));
}


static void tok_setIsFileAttr (picoos_uchar name[], picoos_bool * isFile)
{
    (*isFile) = tok_strEqual(name, KWFile);
}

/* *****************************************************************************/

static void tok_putToSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar str[], pico_tokenType type, pico_tokenSubType subtype)
{
    int i, len;

    if (str[0] != 0) {
        len = picoos_strlen((picoos_char*)str);
        for (i = 0; i < len; i++) {
            if (tok->tokenPos >= IN_BUF_SIZE) {
                picoos_emRaiseWarning(this->common->em, PICO_ERR_INTERNAL_LIMIT, (picoos_char*)"", (picoos_char*)"simple token too long; forced treatment");
                tok_treatSimpleToken(this, tok);
            }
            tok->tokenStr[tok->tokenPos] = str[i];
            tok->tokenPos++;
        }
    }
    tok->tokenType = type;
    tok->tokenSubType = subtype;
}


static void tok_putToMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar str[])
{
    picoos_int32 i, len;
    picoos_uint8 ok;

    tok->markupTagErr = MENone;
    len = picoos_strlen((picoos_char*)str);
    for (i = 0; i< len; i++) {
        if (tok->markupPos >= (MARKUP_STRING_BUF_SIZE - 1)) {
            if ((tok->markupPos == (MARKUP_STRING_BUF_SIZE - 1)) && (tok_markupTagId(tok->markupTagName) != MIDummyEnd)) {
                picoos_emRaiseWarning(this->common->em, PICO_ERR_INTERNAL_LIMIT ,(picoos_char*)"", (picoos_char*)"markup tag too long");
            }
            tok->markupState = MSErrorTooLong;
        } else if ((str[i] == (picoos_uchar)' ') && ((tok->markupState == MSExpectingmarkupTagName) || (tok->markupState == MSGotmarkupTagName) || (tok->markupState == MSGotAttrName) || (tok->markupState == MSGotEqual) || (tok->markupState == MSGotAttrValue))) {
        } else if ((str[i] == (picoos_uchar)'>') && ((tok->markupState == MSGotmarkupTagName) || (tok->markupState == MSInmarkupTagName) || (tok->markupState == MSGotAttrValue))) {
            tok->markupState = MSGotEnd;
        } else if ((str[i] == (picoos_uchar)'/') && ((tok->markupState == MSGotmarkupTagName) || (tok->markupState == MSInmarkupTagName) || (tok->markupState == MSGotAttrValue))) {
            if (tok->markupTagType == MTEnd) {
                tok->markupTagErr = MEUnexpectedChar;
                tok->markupState = MSError;
            } else {
                tok->markupTagType = MTEmpty;
                tok->markupState = MSGotEndSlash;
            }
        } else {
            switch (tok->markupState) {
                case MSNotInMarkup:
                    if (str[i] == (picoos_uchar)'<') {
                        tok_clearMarkupParams(tok->markupParams);
                        tok->nrMarkupParams = 0;
                        tok->strPos = 0;
                        tok->markupTagType = MTStart;
                        tok->markupState = MSGotStart;
                    } else {
                        tok->markupTagErr = MEMissingStart;
                        tok->markupState = MSError;
                    }
                    break;
                case MSGotStart:
                    if (str[i] == (picoos_uchar)'/') {
                        tok->markupTagType = MTEnd;
                        tok->markupState = MSExpectingmarkupTagName;
                    } else if (str[i] == (picoos_uchar)' ') {
                        tok->markupState = MSExpectingmarkupTagName;
                    } else if (tok_idChar(str[i],TRUE)) {
                        tok->markupTagType = MTStart;
                        tok->markupTagName[tok->strPos] = str[i];
                        tok->strPos++;
                        tok->markupTagName[tok->strPos] = 0;
                        tok->markupState = MSInmarkupTagName;
                    } else {
                        tok->markupTagErr = MEUnexpectedChar;
                        tok->markupState = MSError;
                    }
                    break;
                case MSInmarkupTagName:   case MSExpectingmarkupTagName:
                    if (tok_idChar(str[i],tok->markupState == MSExpectingmarkupTagName)) {
                        tok->markupTagName[tok->strPos] = str[i];
                        tok->strPos++;
                        tok->markupTagName[(tok->strPos)] = 0;
                        tok->markupState = MSInmarkupTagName;
                    } else if ((tok->markupState == MSInmarkupTagName) && (str[i] == (picoos_uchar)' ')) {
                        tok->markupState = MSGotmarkupTagName;
                        picobase_lowercase_utf8_str(tok->markupTagName, (picoos_char*)tok->markupTagName, IN_BUF_SIZE, &ok);
                        tok->strPos = 0;
                    } else {
                        tok->markupTagErr = MEIdent;
                        tok->markupState = MSError;
                    }
                    break;
                case MSGotmarkupTagName:   case MSGotAttrValue:
                    if (tok_attrChar(str[i], TRUE)) {
                        if (tok->markupTagType == MTEnd) {
                            tok->markupTagErr = MEUnexpectedChar;
                            tok->markupState = MSError;
                        } else {
                            if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) {
                                tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = str[i];
                                tok->strPos++;
                                tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = 0;
                            } else {
                                picoos_emRaiseWarning(this->common->em, PICO_ERR_INTERNAL_LIMIT ,(picoos_char*)"", (picoos_char*)"too many attributes in markup; ignoring");
                            }
                            tok->markupState = MSInAttrName;
                        }
                    } else {
                        tok->markupTagErr = MEUnexpectedChar;
                        tok->markupState = MSError;
                    }
                    break;
                case MSInAttrName:
                    if (tok_attrChar(str[i], FALSE)) {
                        if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) {
                            tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = str[i];
                            tok->strPos++;
                            tok->markupParams[tok->nrMarkupParams].paramId[tok->strPos] = 0;
                        }
                        tok->markupState = MSInAttrName;
                    } else if (str[i] == (picoos_uchar)' ') {
                        picobase_lowercase_utf8_str(tok->markupParams[tok->nrMarkupParams].paramId, (picoos_char*)tok->markupParams[tok->nrMarkupParams].paramId, IN_BUF_SIZE, &ok);
                        tok_setIsFileAttr(tok->markupParams[tok->nrMarkupParams].paramId, & tok->isFileAttr);
                        tok->markupState = MSGotAttrName;
                    } else if (str[i] == (picoos_uchar)'=') {
                        picobase_lowercase_utf8_str(tok->markupParams[tok->nrMarkupParams].paramId, (picoos_char*)tok->markupParams[tok->nrMarkupParams].paramId, IN_BUF_SIZE, &ok);
                        tok_setIsFileAttr(tok->markupParams[tok->nrMarkupParams].paramId, & tok->isFileAttr);
                        tok->markupState = MSGotEqual;
                    } else {
                        tok->markupTagErr = MEMissingEqual;
                        tok->markupState = MSError;
                    }
                    break;
                case MSGotAttrName:
                    if (str[i] == (picoos_uchar)'=') {
                        tok->markupState = MSGotEqual;
                    } else {
                        tok->markupTagErr = MEMissingEqual;
                        tok->markupState = MSError;
                    }
                    break;
                case MSGotEqual:
                    if ((str[i] == (picoos_uchar)'"') || (str[i] == (picoos_uchar)'\'')) {
                        tok->strDelim = str[i];
                        tok->strPos = 0;
                        tok->markupState = MSInAttrValue;
                    } else {
                        tok->markupTagErr = MEMissingQuote;
                        tok->markupState = MSError;
                    }
                    break;
                case MSInAttrValue:
                    if (!(tok->isFileAttr) && (str[i] == (picoos_uchar)'\\')) {
                        tok->markupState = MSInAttrValueEscaped;
                    } else if (str[i] == tok->strDelim) {
                        if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) {
                            tok->nrMarkupParams++;
                        }
                        tok->strPos = 0;
                        tok->markupState = MSGotAttrValue;
                    } else {
                        if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) {
                            tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = str[i];
                            tok->strPos++;
                            tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = 0;
                        }
                        tok->markupState = MSInAttrValue;
                    }
                    break;
                case MSInAttrValueEscaped:
                    if (tok->nrMarkupParams < MAX_NR_MARKUP_PARAMS) {
                        tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = str[i];
                        tok->strPos++;
                        tok->markupParams[tok->nrMarkupParams].paramVal[tok->strPos] = 0;
                    }
                    tok->markupState = MSInAttrValue;
                    break;
                case MSGotEndSlash:
                    if (str[i] == (picoos_uchar)'>') {
                        tok->markupState = MSGotEnd;
                    } else {
                        tok->markupTagErr = MEUnexpectedChar;
                        tok->markupState = MSError;
                    }
                    break;
            default:
                tok->markupTagErr = MEUnexpectedChar;
                tok->markupState = MSError;
                break;
            }
        }
        tok->markupStr[tok->markupPos] = str[i];
        tok->markupPos++;
        tok->markupStr[tok->markupPos] = 0;
    }
    /*
    PICODBG_DEBUG(("putToMarkup %s", tok->markupStr));
    */
}

/* *****************************************************************************/

static void tok_treatMarkupAsSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok)
{
    picoos_int32 i;

    tok->utfpos = 0;
    tok->utflen = 0;
    tok->markupState = MSNotInMarkup;
    for (i = 0; i < tok->markupPos; i++) {
        tok_treatChar(this, tok, tok->markupStr[i], FALSE);
    }
    tok->markupPos = 0;
    tok->strPos = 0;
}


static void tok_treatMarkup (picodata_ProcessingUnit this, tok_subobj_t * tok)
{
    MarkupId mId;

    if (tok_markupTagId(tok->markupTagName) != MIDummyEnd) {
        if (tok->markupTagErr == MENone) {
            tok->markupState = MSNotInMarkup;
            if ((tok->tokenType != PICODATA_ITEMINFO1_TOKTYPE_SPACE) && (tok->tokenType != PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED)) {
                tok_treatSimpleToken(this, tok);
            }
            tok_putToSimpleToken(this, tok, (picoos_uchar*)" ", PICODATA_ITEMINFO1_TOKTYPE_SPACE, -1);
            mId = tok_markupTagId(tok->markupTagName);
            if ((tok->markupTagType == MTStart) || (tok->markupTagType == MTEmpty)) {
                tok_interpretMarkup(this, tok, TRUE, mId);
            }
            if (((tok->markupTagType == MTEnd) || (tok->markupTagType == MTEmpty))) {
                tok_clearMarkupParams(tok->markupParams);
                tok->nrMarkupParams = 0;
                tok_interpretMarkup(this, tok, FALSE,mId);
            }
        }
        if (tok->markupTagErr != MENone) {
            if (!tok->aborted) {
              picoos_emRaiseWarning(this->common->em, PICO_ERR_INVALID_MARKUP_TAG, (picoos_char*)"", (picoos_char*)"syntax error in markup token '%s'",tok->markupStr);
            }
            tok_treatMarkupAsSimpleToken(this, tok);
        }
    } else {
        tok_treatMarkupAsSimpleToken(this, tok);
    }
    tok->markupState = MSNotInMarkup;
    tok->markupPos = 0;
    tok->strPos = 0;
}


static void tok_treatChar (picodata_ProcessingUnit this, tok_subobj_t * tok, picoos_uchar ch, picoos_bool markupHandling)
{
    picoos_int32 id;
    picoos_uint8 uval8;
    pico_tokenType type = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
    pico_tokenSubType subtype = -1;
    picoos_bool dummy;

    if (ch == NULLC) {
      tok_treatSimpleToken(this, tok);
      tok_putItem(this, tok, PICODATA_ITEM_CMD, PICODATA_ITEMINFO1_CMD_FLUSH, PICODATA_ITEMINFO2_NA, 0, (picoos_uchar*)"");
    }
    else {
      switch (tok_putToUtf(tok, ch)) {
        case UTF_CHAR_MALFORMED:
            tok->utfpos = 0;
            tok->utflen = 0;
            break;
        case UTF_CHAR_INCOMPLETE:
            break;
        case UTF_CHAR_COMPLETE:
            markupHandling = (markupHandling && (tok->markupHandlingMode == MARKUP_HANDLING_ENABLED));
            id = picoktab_graphOffset(tok->graphTab, tok->utf);
            if (id > 0) {
                if (picoktab_getIntPropTokenType(tok->graphTab, id, &uval8)) {
                    type = (pico_tokenType)uval8;
                    if (type == PICODATA_ITEMINFO1_TOKTYPE_LETTERV) {
                        type = PICODATA_ITEMINFO1_TOKTYPE_LETTER;
                    }
                }
                dummy = picoktab_getIntPropTokenSubType(tok->graphTab, id, &subtype);
            } else if (ch <= (picoos_uchar)' ') {
                type = PICODATA_ITEMINFO1_TOKTYPE_SPACE;
                subtype =  -1;
            } else {
                type = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
                subtype =  -1;
            }
            if ((ch > (picoos_uchar)' ')) {
                tok->nrEOL = 0;
            } else if ((ch == EOL)) {
                tok->nrEOL++;
            }
            if (markupHandling && (tok->markupState != MSNotInMarkup)) {
                tok_putToMarkup(this, tok, tok->utf);
                if (tok->markupState >= MSError) {
                    tok_treatMarkupAsSimpleToken(this, tok);
                } else if (tok->markupState == MSGotEnd) {
                    tok_treatMarkup(this, tok);
                }
            } else if ((markupHandling && (ch == (picoos_uchar)'<'))) {
                tok_putToMarkup(this, tok, tok->utf);
            } else if (type != PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED) {
                if ((type != tok->tokenType) || (type == PICODATA_ITEMINFO1_TOKTYPE_CHAR) || (subtype != tok->tokenSubType)) {
                    tok_treatSimpleToken(this, tok);
                } else if ((ch == EOL) && (tok->nrEOL == 2)) {
                    tok_treatSimpleToken(this, tok);
                    tok_putToSimpleToken(this, tok, (picoos_uchar*)".", PICODATA_ITEMINFO1_TOKTYPE_CHAR, -1);
                    tok_treatSimpleToken(this, tok);
                }
                tok_putToSimpleToken(this, tok, tok->utf, type, subtype);
            } else {
                tok_treatSimpleToken(this, tok);
            }
            tok->utfpos = 0;
            tok->utflen = 0;
            break;
      }
    }
}


static void tok_treatSimpleToken (picodata_ProcessingUnit this, tok_subobj_t * tok)
{
    if (tok->tokenPos < IN_BUF_SIZE) {
        tok->tokenStr[tok->tokenPos] = 0;
    }
    if (tok->markupState != MSNotInMarkup) {
        if (!(tok->aborted) && (tok->markupState >= MSGotmarkupTagName) && (tok_markupTagId(tok->markupTagName) != MIDummyEnd)) {
            picoos_emRaiseWarning(this->common->em, PICO_ERR_INVALID_MARKUP_TAG, (picoos_char*)"", (picoos_char*)"unfinished markup tag '%s'",tok->markupStr);
        }
        tok_treatMarkupAsSimpleToken(this, tok);
        tok_treatSimpleToken(this, tok);
    } else if ((tok->tokenPos > 0) && ((tok->ignLevel <= 0) || (tok->tokenType == PICODATA_ITEMINFO1_TOKTYPE_SPACE))) {
        tok_putItem(this, tok, PICODATA_ITEM_TOKEN, tok->tokenType, (picoos_uint8)tok->tokenSubType, 0, tok->tokenStr);
    }
    tok->tokenPos = 0;
    tok->tokenType = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
    tok->tokenSubType =  -1;
}

/* *****************************************************************************/

static pico_status_t tokReset(register picodata_ProcessingUnit this)
{
    tok_subobj_t * tok;
    MarkupId mId;

    if (NULL == this || NULL == this->subObj) {
        return PICO_ERR_OTHER;
    }
    tok = (tok_subobj_t *) this->subObj;

    tok->ignLevel = 0;

    tok->utfpos = 0;
    tok->utflen = 0;

    tok_clearMarkupParams(tok->markupParams);
    tok->nrMarkupParams = 0;
    tok->markupState = MSNotInMarkup;
    tok->markupPos = 0;
    for (mId = MIDummyStart; mId <= MIDummyEnd; mId++) {
        tok->markupLevel[mId] = 0;
    }
    tok->markupTagName[0] = 0;
    tok->markupTagType = MTNone;
    tok->markupTagErr = MENone;

    tok->strPos = 0;
    tok->strDelim = 0;
    tok->isFileAttr = FALSE;

    tok->tokenType = PICODATA_ITEMINFO1_TOKTYPE_UNDEFINED;
    tok->tokenSubType =  -1;
    tok->tokenPos = 0;

    tok->nrEOL = 0;


    tok->markupHandlingMode = TRUE;
    tok->aborted = FALSE;

    tok->start = TRUE;

    tok->outReadPos = 0;
    tok->outWritePos = 0;

    tok->saveFile[0] = 0;


    tok->graphTab = picoktab_getGraphs(this->voice->kbArray[PICOKNOW_KBID_TAB_GRAPHS]);

    tok->xsampa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA_PARSE]);
    PICODBG_TRACE(("got xsampa_parser @ %i",tok->xsampa_parser));

    tok->svoxpa_parser = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_SVOXPA_PARSE]);
    PICODBG_TRACE(("got svoxpa_parser @ %i",tok->svoxpa_parser));

    tok->xsampa2svoxpa_mapper = picokfst_getFST(this->voice->kbArray[PICOKNOW_KBID_FST_XSAMPA2SVOXPA]);
    PICODBG_TRACE(("got xsampa2svoxpa_mapper @ %i",tok->xsampa2svoxpa_mapper));


    return PICO_OK;
}

static pico_status_t tokInitialize(register picodata_ProcessingUnit this)
{
/*

    tok_subobj_t * tok;

    if (NULL == this || NULL == this->subObj) {
        return PICO_ERR_OTHER;
    }
    tok = (tok_subobj_t *) this->subObj;
*/
    return tokReset(this);
}


static pico_status_t tokTerminate(register picodata_ProcessingUnit this)
{
    return PICO_OK;
}

static picodata_step_result_t tokStep(register picodata_ProcessingUnit this, picoos_int16 mode, picoos_uint16 * numBytesOutput);

static pico_status_t tokSubObjDeallocate(register picodata_ProcessingUnit this,
        picoos_MemoryManager mm)
{

    if (NULL != this) {
        picoos_deallocate(this->common->mm, (void *) &this->subObj);
    }
    mm = mm;        /* avoid warning "var not used in this function"*/
    return PICO_OK;
}

picodata_ProcessingUnit picotok_newTokenizeUnit(picoos_MemoryManager mm, picoos_Common common,
        picodata_CharBuffer cbIn, picodata_CharBuffer cbOut,
        picorsrc_Voice voice)
{
    tok_subobj_t * tok;
    picodata_ProcessingUnit this = picodata_newProcessingUnit(mm, common, cbIn, cbOut, voice);
    if (this == NULL) {
        return NULL;
    }
    this->initialize = tokInitialize;
    PICODBG_DEBUG(("set this->step to tokStep"));
    this->step = tokStep;
    this->terminate = tokTerminate;
    this->subDeallocate = tokSubObjDeallocate;
    this->subObj = picoos_allocate(mm, sizeof(tok_subobj_t));
    if (this->subObj == NULL) {
        picoos_deallocate(mm, (void *)&this);
        return NULL;
    }
    tok = (tok_subobj_t *) this->subObj;
    tok->transducer = picotrns_newSimpleTransducer(mm, common, 10*(PICOTRNS_MAX_NUM_POSSYM+2));
    if (NULL == tok->transducer) {
        tokSubObjDeallocate(this,mm);
        picoos_deallocate(mm, (void *)&this);
        return NULL;
    }
    tokInitialize(this);
    return this;
}

/**
 * fill up internal buffer, try to locate token, write token to output
 */
picodata_step_result_t tokStep(register picodata_ProcessingUnit this,
        picoos_int16 mode, picoos_uint16 * numBytesOutput)
{
    register tok_subobj_t * tok;

    if (NULL == this || NULL == this->subObj) {
        return PICODATA_PU_ERROR;
    }
    tok = (tok_subobj_t *) this->subObj;

    mode = mode;        /* avoid warning "var not used in this function"*/

    *numBytesOutput = 0;
    while (1) { /* exit via return */
        picoos_int16 ch;

        if ((tok->outWritePos - tok->outReadPos) > 0) {
            if (picodata_cbPutItem(this->cbOut, &tok->outBuf[tok->outReadPos], tok->outWritePos - tok->outReadPos, numBytesOutput) == PICO_OK) {
                PICODATA_INFO_ITEM(this->voice->kbArray[PICOKNOW_KBID_DBG],
                    (picoos_uint8 *)"tok:", &tok->outBuf[tok->outReadPos], tok->outWritePos - tok->outReadPos);
                tok->outReadPos += *numBytesOutput;
                if (tok->outWritePos == tok->outReadPos) {
                    tok->outWritePos = 0;
                    tok->outReadPos = 0;
                }
            }
            else {
                return PICODATA_PU_OUT_FULL;
            }

        }
        else if (PICO_EOF != (ch = picodata_cbGetCh(this->cbIn))) {
            PICODBG_DEBUG(("read in %c", (picoos_char) ch));
            tok_treatChar(this, tok, (picoos_uchar) ch, /*markupHandling*/TRUE);
        }
        else {
            return PICODATA_PU_IDLE;
        }
    }
}

#ifdef __cplusplus
}
#endif

/* end */