Updated OpenFST version to openfst-1.3.2-CL32004048 from Greco3.
Change-Id: I19b0db718256b35c0e3e5a7315f1ed6335e6dcac
diff --git a/AUTHORS b/AUTHORS
index ac92ec6..2a8b3e4 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -8,6 +8,7 @@
Boulos Harb <harb@google.com>
Dan Bikel <dbikel@google.com
Jacob Ratkiewicz <jpr@cs.indiana.edu>
+Jeff Sorensen <sorenj@google.com>
Johan Schalkwyk <johans@google.com>
Kasturi Raghavan <kasturir@ucla.edu>
Masha Shugrina <shumash@google.com>
diff --git a/Android.mk b/Android.mk
index 0748c91..51989dc 100644
--- a/Android.mk
+++ b/Android.mk
@@ -1,9 +1,7 @@
-#
# Copyright 2012 Google Inc. All Rights Reserved.
# Author: npereira@google.com (Nicole Pereira)
#
# Android makefile for openfst library.
-#
OPENFST_DIR := $(call my-dir)
diff --git a/LICENCE b/LICENSE
similarity index 100%
rename from LICENCE
rename to LICENSE
diff --git a/NEWS b/NEWS
index 0152c55..530d7b4 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,24 @@
-OpenFst - Release 1.2
+OpenFst - Release 1.3
+ * Support for non-fatal exits on errors: (1.3.1)
+ * Added FLAGS_fst_error_fatal: FST errors are
+ fatal if true (default); o.w. return objects flagged as bad:
+ e.g., FSTs - kError
+ prop. true, FST weights - not a Member().
+ * Added kError property bit signifying bad FST
+ * Added NoWeight() method to FST weight requirements that returns
+ weight that is not a Member().
+ * Various improvements to the FAR extensions (1.3.1)
+ * a single FST is now a FAR type
+ * FLAGS_initial_symbols: Uses the symbol table from the
+ first FST in the archive for all entries"
+ * Input/output to standard input/output for some FAR and arc types
+ * --with-icu configuration option no longer needed (1.3.1)
+ * Improved flags usage esp. if use SET_FLAGS not SetFlags/InitFst (1.3.2)
+ * Added 'fst' as possible far writer type (1.3.2)
+ * phi matcher can now accept 0 as the phi label (1.3.2)
+ * Added ngram-fst extension (1.3.2)
+OpenFst - Release 1.2
* Added lookahead matching and filtering for faster composition
* Added EditFst for mutation of o.w. immutable FSTs
* Added script sub-namespace defining type FstClass - a non-templated
diff --git a/README b/README
index a4643d7..aff9e2f 100644
--- a/README
+++ b/README
@@ -1,4 +1,4 @@
-OpenFst - Release 1.2
+OpenFst - Release 1.3
OpenFst is a library for constructing, combining, optimizing, and
searching weighted finite-state transducers (FSTs).
@@ -11,7 +11,7 @@
INSTALLATION:
Follow the generic GNU build system instructions in ./INSTALL. We
- recommend configuring with =--enable-static=no= for faster
+ recommend configuring with --enable-static=no for faster
compiles.
Optional features:
@@ -20,8 +20,8 @@
--enable-const-fsts Enable all ConstFst classes (def: no)
--enable-far Enable FAR (FST Archive) extension (def: no)
--enable-lookahead-fsts Enable LookAheadFst classes (def: no)
+ --enable-ngram-fsts Enable NGramFst classes (def: no)
--enable-pdt Experimental push-down transducer extensions (def: no)
- --with-icu Use ICU to provide unicode support (def: no)
Configuring with --enable-bin=no gives very fast compiles, but
excludes the command line utilities.
diff --git a/README.android b/README.android
index 6d822ec..1e90058 100644
--- a/README.android
+++ b/README.android
@@ -1,4 +1,3 @@
-Openfst openfst-1.2.10
-Updated to Google CL27738347
+Copy of openfst-1.3.2-CL32004048 from Greco3.
-Modified to use stlport, Android.mk file added.
\ No newline at end of file
+Modified to use stlport, Android.mk file added.
diff --git a/configure b/configure
index 38c7241..0bd3a30 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.65 for OpenFst 1.2.10.
+# Generated by GNU Autoconf 2.65 for OpenFst 1.3.2.
#
# Report bugs to <help@www.openfst.org>.
#
@@ -701,8 +701,8 @@
# Identity of this package.
PACKAGE_NAME='OpenFst'
PACKAGE_TARNAME='openfst'
-PACKAGE_VERSION='1.2.10'
-PACKAGE_STRING='OpenFst 1.2.10'
+PACKAGE_VERSION='1.3.2'
+PACKAGE_STRING='OpenFst 1.3.2'
PACKAGE_BUGREPORT='help@www.openfst.org'
PACKAGE_URL=''
@@ -757,6 +757,8 @@
HAVE_PDT_TRUE
HAVE_LOOKAHEAD_FALSE
HAVE_LOOKAHEAD_TRUE
+HAVE_NGRAM_FALSE
+HAVE_NGRAM_TRUE
HAVE_FAR_FALSE
HAVE_FAR_TRUE
HAVE_CONST_FALSE
@@ -893,6 +895,7 @@
enable_compact_fsts
enable_const_fsts
enable_far
+enable_ngram_fsts
enable_lookahead_fsts
enable_pdt
with_icu
@@ -1451,7 +1454,7 @@
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures OpenFst 1.2.10 to adapt to many kinds of systems.
+\`configure' configures OpenFst 1.3.2 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1521,7 +1524,7 @@
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of OpenFst 1.2.10:";;
+ short | recursive ) echo "Configuration of OpenFst 1.3.2:";;
esac
cat <<\_ACEOF
@@ -1540,6 +1543,7 @@
--enable-compact-fsts enable all CompactFst extensions
--enable-const-fsts enable all ConstFst extensions
--enable-far enable FAR extensions
+ --enable-ngram-fsts enable NGramFst extension
--enable-lookahead-fsts enable LookAheadFst extensions
--enable-pdt enable PDT extensions
@@ -1630,7 +1634,7 @@
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-OpenFst configure 1.2.10
+OpenFst configure 1.3.2
generated by GNU Autoconf 2.65
Copyright (C) 2009 Free Software Foundation, Inc.
@@ -2083,7 +2087,7 @@
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by OpenFst $as_me 1.2.10, which was
+It was created by OpenFst $as_me 1.3.2, which was
generated by GNU Autoconf 2.65. Invocation command line was
$ $0 $@
@@ -2891,7 +2895,7 @@
# Define the identity of the package.
PACKAGE='openfst'
- VERSION='1.2.10'
+ VERSION='1.3.2'
cat >>confdefs.h <<_ACEOF
@@ -5007,13 +5011,13 @@
else
lt_cv_nm_interface="BSD nm"
echo "int some_variable = 0;" > conftest.$ac_ext
- (eval echo "\"\$as_me:5010: $ac_compile\"" >&5)
+ (eval echo "\"\$as_me:5014: $ac_compile\"" >&5)
(eval "$ac_compile" 2>conftest.err)
cat conftest.err >&5
- (eval echo "\"\$as_me:5013: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+ (eval echo "\"\$as_me:5017: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
(eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
cat conftest.err >&5
- (eval echo "\"\$as_me:5016: output\"" >&5)
+ (eval echo "\"\$as_me:5020: output\"" >&5)
cat conftest.out >&5
if $GREP 'External.*some_variable' conftest.out > /dev/null; then
lt_cv_nm_interface="MS dumpbin"
@@ -6219,7 +6223,7 @@
;;
*-*-irix6*)
# Find out which ABI we are using.
- echo '#line 6222 "configure"' > conftest.$ac_ext
+ echo '#line 6226 "configure"' > conftest.$ac_ext
if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
(eval $ac_compile) 2>&5
ac_status=$?
@@ -8246,11 +8250,11 @@
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:8249: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:8253: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:8253: \$? = $ac_status" >&5
+ echo "$as_me:8257: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -8585,11 +8589,11 @@
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:8588: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:8592: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:8592: \$? = $ac_status" >&5
+ echo "$as_me:8596: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -8690,11 +8694,11 @@
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:8693: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:8697: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:8697: \$? = $ac_status" >&5
+ echo "$as_me:8701: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -8745,11 +8749,11 @@
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:8748: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:8752: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:8752: \$? = $ac_status" >&5
+ echo "$as_me:8756: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -11129,7 +11133,7 @@
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 11132 "configure"
+#line 11136 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -11225,7 +11229,7 @@
lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
lt_status=$lt_dlunknown
cat > conftest.$ac_ext <<_LT_EOF
-#line 11228 "configure"
+#line 11232 "configure"
#include "confdefs.h"
#if HAVE_DLFCN_H
@@ -13181,11 +13185,11 @@
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:13184: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:13188: $lt_compile\"" >&5)
(eval "$lt_compile" 2>conftest.err)
ac_status=$?
cat conftest.err >&5
- echo "$as_me:13188: \$? = $ac_status" >&5
+ echo "$as_me:13192: \$? = $ac_status" >&5
if (exit $ac_status) && test -s "$ac_outfile"; then
# The compiler can only warn and ignore the option if not recognized
# So say no if there are warnings other than the usual output.
@@ -13280,11 +13284,11 @@
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:13283: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:13287: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:13287: \$? = $ac_status" >&5
+ echo "$as_me:13291: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -13332,11 +13336,11 @@
-e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
-e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
-e 's:$: $lt_compiler_flag:'`
- (eval echo "\"\$as_me:13335: $lt_compile\"" >&5)
+ (eval echo "\"\$as_me:13339: $lt_compile\"" >&5)
(eval "$lt_compile" 2>out/conftest.err)
ac_status=$?
cat out/conftest.err >&5
- echo "$as_me:13339: \$? = $ac_status" >&5
+ echo "$as_me:13343: \$? = $ac_status" >&5
if (exit $ac_status) && test -s out/conftest2.$ac_objext
then
# The compiler can only warn and ignore the option if not recognized
@@ -14303,7 +14307,7 @@
ac_config_headers="$ac_config_headers config.h src/include/fst/config.h"
-ac_config_files="$ac_config_files Makefile src/Makefile src/include/Makefile src/lib/Makefile src/bin/Makefile src/test/Makefile src/extensions/Makefile src/extensions/compact/Makefile src/extensions/const/Makefile src/extensions/far/Makefile src/extensions/lookahead/Makefile src/extensions/pdt/Makefile src/script/Makefile"
+ac_config_files="$ac_config_files Makefile src/Makefile src/include/Makefile src/lib/Makefile src/bin/Makefile src/test/Makefile src/extensions/Makefile src/extensions/compact/Makefile src/extensions/const/Makefile src/extensions/far/Makefile src/extensions/lookahead/Makefile src/extensions/ngram/Makefile src/extensions/pdt/Makefile src/script/Makefile"
ac_ext=cpp
@@ -14408,6 +14412,22 @@
fi
+# Check whether --enable-ngram-fsts was given.
+if test "${enable_ngram_fsts+set}" = set; then :
+ enableval=$enable_ngram_fsts;
+else
+ enable_ngram_fsts=no
+fi
+
+ if test "x$enable_ngram_fsts" != xno; then
+ HAVE_NGRAM_TRUE=
+ HAVE_NGRAM_FALSE='#'
+else
+ HAVE_NGRAM_TRUE='#'
+ HAVE_NGRAM_FALSE=
+fi
+
+
# Check whether --enable-lookahead-fsts was given.
if test "${enable_lookahead_fsts+set}" = set; then :
enableval=$enable_lookahead_fsts;
@@ -14706,6 +14726,10 @@
as_fn_error "conditional \"HAVE_FAR\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
fi
+if test -z "${HAVE_NGRAM_TRUE}" && test -z "${HAVE_NGRAM_FALSE}"; then
+ as_fn_error "conditional \"HAVE_NGRAM\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
if test -z "${HAVE_LOOKAHEAD_TRUE}" && test -z "${HAVE_LOOKAHEAD_FALSE}"; then
as_fn_error "conditional \"HAVE_LOOKAHEAD\" was never defined.
Usually this means the macro was only invoked conditionally." "$LINENO" 5
@@ -15122,7 +15146,7 @@
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by OpenFst $as_me 1.2.10, which was
+This file was extended by OpenFst $as_me 1.3.2, which was
generated by GNU Autoconf 2.65. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -15188,7 +15212,7 @@
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-OpenFst config.status 1.2.10
+OpenFst config.status 1.3.2
configured by $0, generated by GNU Autoconf 2.65,
with options \\"\$ac_cs_config\\"
@@ -15677,6 +15701,7 @@
"src/extensions/const/Makefile") CONFIG_FILES="$CONFIG_FILES src/extensions/const/Makefile" ;;
"src/extensions/far/Makefile") CONFIG_FILES="$CONFIG_FILES src/extensions/far/Makefile" ;;
"src/extensions/lookahead/Makefile") CONFIG_FILES="$CONFIG_FILES src/extensions/lookahead/Makefile" ;;
+ "src/extensions/ngram/Makefile") CONFIG_FILES="$CONFIG_FILES src/extensions/ngram/Makefile" ;;
"src/extensions/pdt/Makefile") CONFIG_FILES="$CONFIG_FILES src/extensions/pdt/Makefile" ;;
"src/script/Makefile") CONFIG_FILES="$CONFIG_FILES src/script/Makefile" ;;
diff --git a/configure.ac b/configure.ac
index 211dcd7..908a6d5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT([OpenFst], [1.2.10], [help@www.openfst.org])
+AC_INIT([OpenFst], [1.3.2], [help@www.openfst.org])
AM_INIT_AUTOMAKE([foreign nostdinc -Wall -Werror])
AC_PROG_CXX
@@ -19,6 +19,7 @@
src/extensions/const/Makefile
src/extensions/far/Makefile
src/extensions/lookahead/Makefile
+ src/extensions/ngram/Makefile
src/extensions/pdt/Makefile
src/script/Makefile
])
@@ -63,6 +64,15 @@
AM_CONDITIONAL([HAVE_FAR], [test "x$enable_far" != xno])
AC_ARG_ENABLE(
+ [ngram-fsts],
+ [AS_HELP_STRING(
+ [--enable-ngram-fsts],
+ [enable NGramFst extension])],
+ [],
+ [enable_ngram_fsts=no])
+AM_CONDITIONAL([HAVE_NGRAM], [test "x$enable_ngram_fsts" != xno])
+
+AC_ARG_ENABLE(
[lookahead-fsts],
[AS_HELP_STRING(
[--enable-lookahead-fsts],
diff --git a/fix_headers.sh b/fix_headers.sh
new file mode 100755
index 0000000..b1b0844
--- /dev/null
+++ b/fix_headers.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+INPUT_FILE=$1
+
+function fix_tr1() {
+ grep $1 $INPUT_FILE > /dev/null 2>&1
+ if [ $? -eq 0 ] ; then
+ echo "Modifying $INPUT_FILE ..."
+ sed -e "s/$1/$2/g" $INPUT_FILE > .tmp
+ if [ $? -ne 0 ] ; then
+ echo "sed failed!"
+ return 1
+ fi
+ mv -f .tmp $INPUT_FILE
+ fi
+ return 0
+}
+
+fix_tr1 "tr1\/unordered_map" "unordered_map"
+fix_tr1 "tr1/unordered_set" "unordered_set"
+fix_tr1 "tr1::hash" "hash"
diff --git a/src/bin/fstarcsort.cc b/src/bin/fstarcsort.cc
index 33e534d..9f1b420 100644
--- a/src/bin/fstarcsort.cc
+++ b/src/bin/fstarcsort.cc
@@ -38,7 +38,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
diff --git a/src/bin/fstclosure.cc b/src/bin/fstclosure.cc
index 569c708..f6b3324 100644
--- a/src/bin/fstclosure.cc
+++ b/src/bin/fstclosure.cc
@@ -34,7 +34,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstcompile.cc b/src/bin/fstcompile.cc
index db62f95..2e0bdd4 100644
--- a/src/bin/fstcompile.cc
+++ b/src/bin/fstcompile.cc
@@ -45,14 +45,14 @@
usage += " [text.fst [binary.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
}
const char *source = "standard input";
- istream *istrm = &std::cin;
+ istream *istrm = &cin;
if (argc > 1 && strcmp(argv[1], "-") != 0) {
source = argv[1];
istrm = new fst::ifstream(argv[1]);
@@ -63,13 +63,16 @@
}
const SymbolTable *isyms = 0, *osyms = 0, *ssyms = 0;
+ fst::SymbolTableTextOptions opts;
+ opts.allow_negative = FLAGS_allow_negative_labels;
+
if (!FLAGS_isymbols.empty()) {
- isyms = SymbolTable::ReadText(FLAGS_isymbols, FLAGS_allow_negative_labels);
+ isyms = SymbolTable::ReadText(FLAGS_isymbols, opts);
if (!isyms) exit(1);
}
if (!FLAGS_osymbols.empty()) {
- osyms = SymbolTable::ReadText(FLAGS_osymbols, FLAGS_allow_negative_labels);
+ osyms = SymbolTable::ReadText(FLAGS_osymbols, opts);
if (!osyms) exit(1);
}
@@ -85,7 +88,7 @@
FLAGS_acceptor, FLAGS_keep_isymbols, FLAGS_keep_osymbols,
FLAGS_keep_state_numbering, FLAGS_allow_negative_labels);
- if (istrm != &std::cin)
+ if (istrm != &cin)
delete istrm;
return 0;
diff --git a/src/bin/fstcompose.cc b/src/bin/fstcompose.cc
index 0ba8c47..996aca7 100644
--- a/src/bin/fstcompose.cc
+++ b/src/bin/fstcompose.cc
@@ -41,7 +41,7 @@
usage += " in1.fst in2.fst [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstconcat.cc b/src/bin/fstconcat.cc
index fbe9b68..4a02205 100644
--- a/src/bin/fstconcat.cc
+++ b/src/bin/fstconcat.cc
@@ -34,7 +34,7 @@
usage += " in1.fst in2.fst [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstconnect.cc b/src/bin/fstconnect.cc
index f774767..2900d1b 100644
--- a/src/bin/fstconnect.cc
+++ b/src/bin/fstconnect.cc
@@ -33,7 +33,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstconvert.cc b/src/bin/fstconvert.cc
index f7c4ad9..9fbb43e 100644
--- a/src/bin/fstconvert.cc
+++ b/src/bin/fstconvert.cc
@@ -33,7 +33,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstdeterminize.cc b/src/bin/fstdeterminize.cc
index aa5064f..ca7840e 100644
--- a/src/bin/fstdeterminize.cc
+++ b/src/bin/fstdeterminize.cc
@@ -41,7 +41,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstdifference.cc b/src/bin/fstdifference.cc
index 5459b03..8bdf89e 100644
--- a/src/bin/fstdifference.cc
+++ b/src/bin/fstdifference.cc
@@ -39,7 +39,7 @@
usage += " in1.fst in2.fst [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstdraw.cc b/src/bin/fstdraw.cc
index 51ebb2d..98f8ff4 100644
--- a/src/bin/fstdraw.cc
+++ b/src/bin/fstdraw.cc
@@ -26,8 +26,6 @@
DEFINE_string(osymbols, "", "Output label symbol table");
DEFINE_string(ssymbols, "", "State label symbol table");
DEFINE_bool(numeric, false, "Print numeric labels");
-DEFINE_string(save_isymbols, "", "Save input symbol table to file");
-DEFINE_string(save_osymbols, "", "Save output symbol table to file");
DEFINE_int32(precision, 5, "Set precision (number of char/float)");
DEFINE_bool(show_weight_one, false,
"Print/draw arc weights and final weights equal to Weight::One()");
@@ -51,10 +49,10 @@
string usage = "Prints out binary FSTs in dot text format.\n\n Usage: ";
usage += argv[0];
- usage += " [binary.fst [text.fst]]\n";
+ usage += " [binary.fst [text.dot]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
@@ -65,7 +63,7 @@
s::FstClass *fst = s::FstClass::Read(in_name);
if (!fst) return 1;
- ostream *ostrm = &std::cout;
+ ostream *ostrm = &cout;
string dest = "stdout";
if (argc == 3) {
dest = argv[2];
@@ -79,13 +77,16 @@
const SymbolTable *isyms = 0, *osyms = 0, *ssyms = 0;
+ fst::SymbolTableTextOptions opts;
+ opts.allow_negative = FLAGS_allow_negative_labels;
+
if (!FLAGS_isymbols.empty() && !FLAGS_numeric) {
- isyms = SymbolTable::ReadText(FLAGS_isymbols, FLAGS_allow_negative_labels);
+ isyms = SymbolTable::ReadText(FLAGS_isymbols, opts);
if (!isyms) exit(1);
}
if (!FLAGS_osymbols.empty() && !FLAGS_numeric) {
- osyms = SymbolTable::ReadText(FLAGS_osymbols, FLAGS_allow_negative_labels);
+ osyms = SymbolTable::ReadText(FLAGS_osymbols, opts);
if (!osyms) exit(1);
}
@@ -106,13 +107,7 @@
FLAGS_fontsize, FLAGS_precision,
FLAGS_show_weight_one, ostrm, dest);
- if (isyms && !FLAGS_save_isymbols.empty())
- isyms->WriteText(FLAGS_save_isymbols);
-
- if (osyms && !FLAGS_save_osymbols.empty())
- osyms->WriteText(FLAGS_save_osymbols);
-
- if (ostrm != &std::cout)
+ if (ostrm != &cout)
delete ostrm;
return 0;
}
diff --git a/src/bin/fstencode.cc b/src/bin/fstencode.cc
index 95dd49c..003ca24 100644
--- a/src/bin/fstencode.cc
+++ b/src/bin/fstencode.cc
@@ -39,7 +39,7 @@
usage += " in.fst codex [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstepsnormalize.cc b/src/bin/fstepsnormalize.cc
index c8813c6..d5c1a8d 100644
--- a/src/bin/fstepsnormalize.cc
+++ b/src/bin/fstepsnormalize.cc
@@ -35,7 +35,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstequal.cc b/src/bin/fstequal.cc
index 885b330..c64f6e3 100644
--- a/src/bin/fstequal.cc
+++ b/src/bin/fstequal.cc
@@ -33,7 +33,7 @@
usage += " in1.fst in2.fst\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc != 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstequivalent.cc b/src/bin/fstequivalent.cc
index e7e09f2..70010d7 100644
--- a/src/bin/fstequivalent.cc
+++ b/src/bin/fstequivalent.cc
@@ -43,7 +43,7 @@
usage += " in1.fst in2.fst\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc != 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstinfo.cc b/src/bin/fstinfo.cc
index 23816be..f02e0b3 100644
--- a/src/bin/fstinfo.cc
+++ b/src/bin/fstinfo.cc
@@ -41,7 +41,7 @@
usage += " [in.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 2) {
ShowUsage();
return 1;
diff --git a/src/bin/fstintersect.cc b/src/bin/fstintersect.cc
index b3558fc..3bdc26b 100644
--- a/src/bin/fstintersect.cc
+++ b/src/bin/fstintersect.cc
@@ -40,7 +40,7 @@
usage += " Flags: connect\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstinvert.cc b/src/bin/fstinvert.cc
index f9b348b..00dadc5 100644
--- a/src/bin/fstinvert.cc
+++ b/src/bin/fstinvert.cc
@@ -32,7 +32,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstmap.cc b/src/bin/fstmap.cc
index 231c725..81336b1 100644
--- a/src/bin/fstmap.cc
+++ b/src/bin/fstmap.cc
@@ -43,7 +43,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstminimize.cc b/src/bin/fstminimize.cc
index 8d9d635..2d4e7a9 100644
--- a/src/bin/fstminimize.cc
+++ b/src/bin/fstminimize.cc
@@ -36,7 +36,7 @@
usage += " [in.fst [out1.fst [out2.fst]]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstprint.cc b/src/bin/fstprint.cc
index b42bb7e..b5b23e5 100644
--- a/src/bin/fstprint.cc
+++ b/src/bin/fstprint.cc
@@ -44,7 +44,7 @@
usage += " [binary.fst [text.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
@@ -56,7 +56,7 @@
s::FstClass *fst = s::FstClass::Read(in_name);
if (!fst) return 1;
- ostream *ostrm = &std::cout;
+ ostream *ostrm = &cout;
string dest = "standard output";
if (argc == 3) {
dest = argv[2];
@@ -70,13 +70,16 @@
const SymbolTable *isyms = 0, *osyms = 0, *ssyms = 0;
+ fst::SymbolTableTextOptions opts;
+ opts.allow_negative = FLAGS_allow_negative_labels;
+
if (!FLAGS_isymbols.empty() && !FLAGS_numeric) {
- isyms = SymbolTable::ReadText(FLAGS_isymbols, FLAGS_allow_negative_labels);
+ isyms = SymbolTable::ReadText(FLAGS_isymbols, opts);
if (!isyms) exit(1);
}
if (!FLAGS_osymbols.empty() && !FLAGS_numeric) {
- osyms = SymbolTable::ReadText(FLAGS_osymbols, FLAGS_allow_negative_labels);
+ osyms = SymbolTable::ReadText(FLAGS_osymbols, opts);
if (!osyms) exit(1);
}
@@ -99,7 +102,7 @@
if (osyms && !FLAGS_save_osymbols.empty())
osyms->WriteText(FLAGS_save_osymbols);
- if (ostrm != &std::cout)
+ if (ostrm != &cout)
delete ostrm;
return 0;
}
diff --git a/src/bin/fstproject.cc b/src/bin/fstproject.cc
index a67f431..efd8dbf 100644
--- a/src/bin/fstproject.cc
+++ b/src/bin/fstproject.cc
@@ -35,7 +35,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstprune.cc b/src/bin/fstprune.cc
index 2041b7c..dedf237 100644
--- a/src/bin/fstprune.cc
+++ b/src/bin/fstprune.cc
@@ -38,7 +38,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstpush.cc b/src/bin/fstpush.cc
index 3b849b5..d40ff9f 100644
--- a/src/bin/fstpush.cc
+++ b/src/bin/fstpush.cc
@@ -41,7 +41,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstrandgen.cc b/src/bin/fstrandgen.cc
index f48d0cf..b3716e5 100644
--- a/src/bin/fstrandgen.cc
+++ b/src/bin/fstrandgen.cc
@@ -42,7 +42,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstrelabel.cc b/src/bin/fstrelabel.cc
index cc86450..e75db99 100644
--- a/src/bin/fstrelabel.cc
+++ b/src/bin/fstrelabel.cc
@@ -43,6 +43,7 @@
int main(int argc, char **argv) {
namespace s = fst::script;
using fst::SymbolTable;
+ using fst::SymbolTableTextOptions;
using fst::script::FstClass;
using fst::script::MutableFstClass;
@@ -58,7 +59,7 @@
usage += " -relabel_opairs opairs.txts\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
@@ -71,24 +72,24 @@
if (!fst) return 1;
// Relabel with symbol tables
+ SymbolTableTextOptions opts;
+ opts.allow_negative = FLAGS_allow_negative_labels;
if (!FLAGS_relabel_isymbols.empty() || !FLAGS_relabel_osymbols.empty()) {
bool attach_new_isymbols = (fst->InputSymbols() != 0);
const SymbolTable* old_isymbols = FLAGS_isymbols.empty()
? fst->InputSymbols()
- : SymbolTable::ReadText(FLAGS_isymbols, FLAGS_allow_negative_labels);
+ : SymbolTable::ReadText(FLAGS_isymbols, opts);
const SymbolTable* relabel_isymbols = FLAGS_relabel_isymbols.empty()
? NULL
- : SymbolTable::ReadText(FLAGS_relabel_isymbols,
- FLAGS_allow_negative_labels);
+ : SymbolTable::ReadText(FLAGS_relabel_isymbols, opts);
bool attach_new_osymbols = (fst->OutputSymbols() != 0);
const SymbolTable* old_osymbols = FLAGS_osymbols.empty()
? fst->OutputSymbols()
- : SymbolTable::ReadText(FLAGS_osymbols, FLAGS_allow_negative_labels);
+ : SymbolTable::ReadText(FLAGS_osymbols, opts);
const SymbolTable* relabel_osymbols = FLAGS_relabel_osymbols.empty()
? NULL
- : SymbolTable::ReadText(FLAGS_relabel_osymbols,
- FLAGS_allow_negative_labels);
+ : SymbolTable::ReadText(FLAGS_relabel_osymbols, opts);
s::Relabel(fst,
old_isymbols, relabel_isymbols, attach_new_isymbols,
diff --git a/src/bin/fstreplace.cc b/src/bin/fstreplace.cc
index 5d1d770..d6522fc 100644
--- a/src/bin/fstreplace.cc
+++ b/src/bin/fstreplace.cc
@@ -31,7 +31,7 @@
usage += " root.fst rootlabel [rule1.fst label1 ...] [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstreverse.cc b/src/bin/fstreverse.cc
index 7507ffa..a919f58 100644
--- a/src/bin/fstreverse.cc
+++ b/src/bin/fstreverse.cc
@@ -26,6 +26,7 @@
#include <fst/script/fst-class.h>
#include <iostream>
#include <fstream>
+#include <sstream>
int main(int argc, char **argv) {
using fst::script::FstClass;
@@ -37,7 +38,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstreweight.cc b/src/bin/fstreweight.cc
index 3913a6d..4e8333d 100644
--- a/src/bin/fstreweight.cc
+++ b/src/bin/fstreweight.cc
@@ -35,7 +35,7 @@
usage += " in.fst potential.txt [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/bin/fstrmepsilon.cc b/src/bin/fstrmepsilon.cc
index 63a80b8..da4e6d9 100644
--- a/src/bin/fstrmepsilon.cc
+++ b/src/bin/fstrmepsilon.cc
@@ -42,7 +42,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstshortestdistance.cc b/src/bin/fstshortestdistance.cc
index 01f5617..9103b72 100644
--- a/src/bin/fstshortestdistance.cc
+++ b/src/bin/fstshortestdistance.cc
@@ -41,7 +41,7 @@
usage += " [in.fst [distance.txt]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstshortestpath.cc b/src/bin/fstshortestpath.cc
index 4a9e720..6e33db8 100644
--- a/src/bin/fstshortestpath.cc
+++ b/src/bin/fstshortestpath.cc
@@ -42,7 +42,7 @@
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstsymbols.cc b/src/bin/fstsymbols.cc
index c07f1be..23a6447 100644
--- a/src/bin/fstsymbols.cc
+++ b/src/bin/fstsymbols.cc
@@ -48,7 +48,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
@@ -78,17 +78,18 @@
}
}
+ fst::SymbolTableTextOptions opts;
+ opts.allow_negative = FLAGS_allow_negative_labels;
+
if (FLAGS_clear_isymbols)
fst->SetInputSymbols(0);
else if (!FLAGS_isymbols.empty())
- fst->SetInputSymbols(
- SymbolTable::ReadText(FLAGS_isymbols, FLAGS_allow_negative_labels));
+ fst->SetInputSymbols(SymbolTable::ReadText(FLAGS_isymbols, opts));
if (FLAGS_clear_osymbols)
fst->SetOutputSymbols(0);
else if (!FLAGS_osymbols.empty())
- fst->SetOutputSymbols(
- SymbolTable::ReadText(FLAGS_osymbols, FLAGS_allow_negative_labels));
+ fst->SetOutputSymbols(SymbolTable::ReadText(FLAGS_osymbols, opts));
if (!FLAGS_relabel_ipairs.empty()) {
typedef int64 Label;
diff --git a/src/bin/fstsynchronize.cc b/src/bin/fstsynchronize.cc
index 7137504..8c226ad 100644
--- a/src/bin/fstsynchronize.cc
+++ b/src/bin/fstsynchronize.cc
@@ -32,7 +32,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fsttopsort.cc b/src/bin/fsttopsort.cc
index 6f04215..69896c7 100644
--- a/src/bin/fsttopsort.cc
+++ b/src/bin/fsttopsort.cc
@@ -32,7 +32,7 @@
usage += " [in.fst [out.fst]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/bin/fstunion.cc b/src/bin/fstunion.cc
index 41c4f83..ae7eb66 100644
--- a/src/bin/fstunion.cc
+++ b/src/bin/fstunion.cc
@@ -25,6 +25,7 @@
#include <fst/script/union.h>
#include <iostream>
#include <fstream>
+#include <sstream>
int main(int argc, char **argv) {
using fst::script::FstClass;
@@ -36,7 +37,7 @@
usage += " in1.fst in2.fst [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/extensions/Makefile.am b/src/extensions/Makefile.am
index 64d5483..d982c15 100644
--- a/src/extensions/Makefile.am
+++ b/src/extensions/Makefile.am
@@ -14,8 +14,12 @@
lookaheaddir = lookahead
endif
+if HAVE_NGRAM
+ngramdir = ngram
+endif
+
if HAVE_PDT
pdtdir = pdt
endif
-SUBDIRS = $(compactdir) $(constdir) $(fardir) $(lookaheaddir) $(pdtdir)
+SUBDIRS = $(compactdir) $(constdir) $(fardir) $(lookaheaddir) $(ngramdir) $(pdtdir)
diff --git a/src/extensions/Makefile.in b/src/extensions/Makefile.in
index 43a74c8..6bcf056 100644
--- a/src/extensions/Makefile.in
+++ b/src/extensions/Makefile.in
@@ -63,7 +63,7 @@
distdir
ETAGS = etags
CTAGS = ctags
-DIST_SUBDIRS = compact const far lookahead pdt
+DIST_SUBDIRS = compact const far lookahead ngram pdt
DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
am__relativize = \
dir0=`pwd`; \
@@ -216,8 +216,9 @@
@HAVE_CONST_TRUE@constdir = const
@HAVE_FAR_TRUE@fardir = far
@HAVE_LOOKAHEAD_TRUE@lookaheaddir = lookahead
+@HAVE_NGRAM_TRUE@ngramdir = ngram
@HAVE_PDT_TRUE@pdtdir = pdt
-SUBDIRS = $(compactdir) $(constdir) $(fardir) $(lookaheaddir) $(pdtdir)
+SUBDIRS = $(compactdir) $(constdir) $(fardir) $(lookaheaddir) $(ngramdir) $(pdtdir)
all: all-recursive
.SUFFIXES:
diff --git a/src/extensions/far/Makefile.am b/src/extensions/far/Makefile.am
index 61f7cc8..a7c2fff 100644
--- a/src/extensions/far/Makefile.am
+++ b/src/extensions/far/Makefile.am
@@ -12,12 +12,13 @@
libfstfar_la_LDFLAGS = -version-info 0:0:0
if HAVE_SCRIPT
-libfstfarscript_la_SOURCES = farscript.cc compile-strings.cc main.cc
+libfstfarscript_la_SOURCES = farscript.cc strings.cc main.cc
libfstfarscript_la_LDFLAGS = -version-info 0:0:0
endif
if HAVE_BIN
-bin_PROGRAMS = farcompilestrings farcreate farextract farinfo farprintstrings
+bin_PROGRAMS = farcompilestrings farcreate farequal farextract farinfo \
+ farprintstrings
LDADD = libfstfarscript.la libfstfar.la ../../script/libfstscript.la \
../../lib/libfst.la -lm -ldl
@@ -26,6 +27,8 @@
farcreate_SOURCES = farcreate.cc
+farequal_SOURCES = farequal.cc
+
farextract_SOURCES = farextract.cc
farinfo_SOURCES = farinfo.cc
diff --git a/src/extensions/far/Makefile.in b/src/extensions/far/Makefile.in
index 8e05cab..0635556 100644
--- a/src/extensions/far/Makefile.in
+++ b/src/extensions/far/Makefile.in
@@ -36,8 +36,9 @@
build_triplet = @build@
host_triplet = @host@
@HAVE_BIN_TRUE@bin_PROGRAMS = farcompilestrings$(EXEEXT) \
-@HAVE_BIN_TRUE@ farcreate$(EXEEXT) farextract$(EXEEXT) \
-@HAVE_BIN_TRUE@ farinfo$(EXEEXT) farprintstrings$(EXEEXT)
+@HAVE_BIN_TRUE@ farcreate$(EXEEXT) farequal$(EXEEXT) \
+@HAVE_BIN_TRUE@ farextract$(EXEEXT) farinfo$(EXEEXT) \
+@HAVE_BIN_TRUE@ farprintstrings$(EXEEXT)
subdir = src/extensions/far
DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
@@ -84,10 +85,9 @@
@HAVE_SCRIPT_FALSE@am_libfstfar_la_rpath = -rpath $(libfstdir)
@HAVE_SCRIPT_TRUE@am_libfstfar_la_rpath = -rpath $(libfstdir)
libfstfarscript_la_LIBADD =
-am__libfstfarscript_la_SOURCES_DIST = farscript.cc compile-strings.cc \
- main.cc
+am__libfstfarscript_la_SOURCES_DIST = farscript.cc strings.cc main.cc
@HAVE_SCRIPT_TRUE@am_libfstfarscript_la_OBJECTS = farscript.lo \
-@HAVE_SCRIPT_TRUE@ compile-strings.lo main.lo
+@HAVE_SCRIPT_TRUE@ strings.lo main.lo
libfstfarscript_la_OBJECTS = $(am_libfstfarscript_la_OBJECTS)
libfstfarscript_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
@@ -109,6 +109,13 @@
@HAVE_BIN_TRUE@farcreate_DEPENDENCIES = libfstfarscript.la \
@HAVE_BIN_TRUE@ libfstfar.la ../../script/libfstscript.la \
@HAVE_BIN_TRUE@ ../../lib/libfst.la
+am__farequal_SOURCES_DIST = farequal.cc
+@HAVE_BIN_TRUE@am_farequal_OBJECTS = farequal.$(OBJEXT)
+farequal_OBJECTS = $(am_farequal_OBJECTS)
+farequal_LDADD = $(LDADD)
+@HAVE_BIN_TRUE@farequal_DEPENDENCIES = libfstfarscript.la libfstfar.la \
+@HAVE_BIN_TRUE@ ../../script/libfstscript.la \
+@HAVE_BIN_TRUE@ ../../lib/libfst.la
am__farextract_SOURCES_DIST = farextract.cc
@HAVE_BIN_TRUE@am_farextract_OBJECTS = farextract.$(OBJEXT)
farextract_OBJECTS = $(am_farextract_OBJECTS)
@@ -145,13 +152,13 @@
$(LDFLAGS) -o $@
SOURCES = $(libfstfar_la_SOURCES) $(libfstfarscript_la_SOURCES) \
$(farcompilestrings_SOURCES) $(farcreate_SOURCES) \
- $(farextract_SOURCES) $(farinfo_SOURCES) \
+ $(farequal_SOURCES) $(farextract_SOURCES) $(farinfo_SOURCES) \
$(farprintstrings_SOURCES)
DIST_SOURCES = $(libfstfar_la_SOURCES) \
$(am__libfstfarscript_la_SOURCES_DIST) \
$(am__farcompilestrings_SOURCES_DIST) \
- $(am__farcreate_SOURCES_DIST) $(am__farextract_SOURCES_DIST) \
- $(am__farinfo_SOURCES_DIST) \
+ $(am__farcreate_SOURCES_DIST) $(am__farequal_SOURCES_DIST) \
+ $(am__farextract_SOURCES_DIST) $(am__farinfo_SOURCES_DIST) \
$(am__farprintstrings_SOURCES_DIST)
ETAGS = etags
CTAGS = ctags
@@ -283,13 +290,14 @@
@HAVE_SCRIPT_TRUE@libfst_LTLIBRARIES = libfstfarscript.la libfstfar.la
libfstfar_la_SOURCES = sttable.cc stlist.cc
libfstfar_la_LDFLAGS = -version-info 0:0:0
-@HAVE_SCRIPT_TRUE@libfstfarscript_la_SOURCES = farscript.cc compile-strings.cc main.cc
+@HAVE_SCRIPT_TRUE@libfstfarscript_la_SOURCES = farscript.cc strings.cc main.cc
@HAVE_SCRIPT_TRUE@libfstfarscript_la_LDFLAGS = -version-info 0:0:0
@HAVE_BIN_TRUE@LDADD = libfstfarscript.la libfstfar.la ../../script/libfstscript.la \
@HAVE_BIN_TRUE@ ../../lib/libfst.la -lm -ldl
@HAVE_BIN_TRUE@farcompilestrings_SOURCES = farcompilestrings.cc
@HAVE_BIN_TRUE@farcreate_SOURCES = farcreate.cc
+@HAVE_BIN_TRUE@farequal_SOURCES = farequal.cc
@HAVE_BIN_TRUE@farextract_SOURCES = farextract.cc
@HAVE_BIN_TRUE@farinfo_SOURCES = farinfo.cc
@HAVE_BIN_TRUE@farprintstrings_SOURCES = farprintstrings.cc
@@ -411,6 +419,9 @@
farcreate$(EXEEXT): $(farcreate_OBJECTS) $(farcreate_DEPENDENCIES)
@rm -f farcreate$(EXEEXT)
$(CXXLINK) $(farcreate_OBJECTS) $(farcreate_LDADD) $(LIBS)
+farequal$(EXEEXT): $(farequal_OBJECTS) $(farequal_DEPENDENCIES)
+ @rm -f farequal$(EXEEXT)
+ $(CXXLINK) $(farequal_OBJECTS) $(farequal_LDADD) $(LIBS)
farextract$(EXEEXT): $(farextract_OBJECTS) $(farextract_DEPENDENCIES)
@rm -f farextract$(EXEEXT)
$(CXXLINK) $(farextract_OBJECTS) $(farextract_LDADD) $(LIBS)
@@ -427,15 +438,16 @@
distclean-compile:
-rm -f *.tab.c
-@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/compile-strings.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farcompilestrings.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farcreate.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farequal.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farextract.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farinfo.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farprintstrings.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/farscript.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/stlist.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/strings.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sttable.Plo@am__quote@
.cc.o:
diff --git a/src/extensions/far/farcompilestrings.cc b/src/extensions/far/farcompilestrings.cc
index 0c9b352..765b908 100644
--- a/src/extensions/far/farcompilestrings.cc
+++ b/src/extensions/far/farcompilestrings.cc
@@ -25,12 +25,15 @@
#include <fst/extensions/far/main.h>
#include <iostream>
#include <fstream>
+#include <sstream>
DEFINE_string(key_prefix, "", "Prefix to append to keys");
DEFINE_string(key_suffix, "", "Suffix to append to keys");
DEFINE_int32(generate_keys, 0,
"Generate N digit numeric keys (def: use file basenames)");
-DEFINE_string(far_type, "default", "FAR file format type: one of: ");
+DEFINE_string(far_type, "default",
+ "FAR file format type: one of: \"default\", \"fst\""
+ "\"stlist\", \"sttable\"");
DEFINE_bool(allow_negative_labels, false,
"Allow negative labels (not recommended; may cause conflicts)");
DEFINE_string(arc_type, "standard", "Output arc type");
@@ -43,7 +46,11 @@
DEFINE_string(unknown_symbol, "", "");
DEFINE_bool(file_list_input, false,
"Each input files contains a list of files to be processed");
-
+DEFINE_bool(keep_symbols, false,
+ "Store symbol table in Far file");
+DEFINE_bool(initial_symbols, true,
+ "When keep_symbols==true, stores symbol table only for the first"
+ " Fst in archive.");
int main(int argc, char **argv) {
namespace s = fst::script;
@@ -51,23 +58,19 @@
string usage = "Compiles a set of strings as FSTs and stores them in";
usage += " a finite-state archive.\n\n Usage:";
usage += argv[0];
- usage += " in1.txt [in2.txt ...] out.far\n";
+ usage += " [in1.txt [[in2.txt ...] out.far]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
- if (argc < 3) {
- ShowUsage();
- return 1;
- }
+ vector<string> in_fnames;
+ for (unsigned i = 1; i < argc - 1; ++i)
+ in_fnames.push_back(strcmp(argv[i], "") != 0 ? argv[i] : "");
+ if (in_fnames.empty())
+ in_fnames.push_back(argc == 2 && strcmp(argv[1], "-") != 0 ? argv[1] : "");
- vector<string> in_fnames(argc - 2);
-
- for (unsigned i = 1; i < argc - 1; ++i) {
- in_fnames[i - 1] = argv[i];
- }
-
- string out_fname = argv[argc - 1];
+ string out_fname =
+ argc > 2 && strcmp(argv[argc - 1], "-") != 0 ? argv[argc - 1] : "";
fst::FarEntryType fet = fst::StringToFarEntryType(FLAGS_entry_type);
fst::FarTokenType ftt = fst::StringToFarTokenType(FLAGS_token_type);
@@ -76,6 +79,7 @@
s::FarCompileStrings(in_fnames, out_fname, FLAGS_arc_type, FLAGS_fst_type,
far_type, FLAGS_generate_keys, fet, ftt,
FLAGS_symbols, FLAGS_unknown_symbol,
+ FLAGS_keep_symbols, FLAGS_initial_symbols,
FLAGS_allow_negative_labels,
FLAGS_file_list_input, FLAGS_key_prefix,
FLAGS_key_suffix);
diff --git a/src/extensions/far/farcreate.cc b/src/extensions/far/farcreate.cc
index 009fc34..9ddbec3 100644
--- a/src/extensions/far/farcreate.cc
+++ b/src/extensions/far/farcreate.cc
@@ -29,7 +29,8 @@
DEFINE_int32(generate_keys, 0,
"Generate N digit numeric keys (def: use file basenames)");
DEFINE_string(far_type, "default",
- "FAR file format type: one of: \"default\", \"stlist\", \"sstable\", \"sttable\"");
+ "FAR file format type: one of: \"default\", "
+ "\"stlist\", \"sttable\"");
DEFINE_bool(file_list_input, false,
"Each input files contains a list of files to be processed");
@@ -38,21 +39,20 @@
string usage = "Creates a finite-state archive from input FSTs.\n\n Usage:";
usage += argv[0];
- usage += " in1.fst [in2.fst ...] out.far\n";
+ usage += " [in1.fst [[in2.fst ...] out.far]]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
-
- if (argc < 3) {
- ShowUsage();
- return 1;
- }
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
vector<string> in_fnames;
- for (int i = 1; i < argc - 1; ++i)
- in_fnames.push_back(argv[i]);
+ for (unsigned i = 1; i < argc - 1; ++i)
+ in_fnames.push_back(strcmp(argv[i], "") != 0 ? argv[i] : "");
+ if (in_fnames.empty())
+ in_fnames.push_back(argc == 2 && strcmp(argv[1], "-") != 0 ? argv[1] : "");
- string out_fname = argv[argc - 1];
+ string out_fname =
+ argc > 2 && strcmp(argv[argc - 1], "-") != 0 ? argv[argc - 1] : "";
+
string arc_type = fst::LoadArcTypeFromFst(in_fnames[0]);
fst::FarType far_type = fst::FarTypeFromString(FLAGS_far_type);
diff --git a/src/extensions/far/farequal.cc b/src/extensions/far/farequal.cc
new file mode 100644
index 0000000..94e607c
--- /dev/null
+++ b/src/extensions/far/farequal.cc
@@ -0,0 +1,57 @@
+// farequal.cc
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: allauzen@google.com (Cyril Allauzen)
+//
+// \file
+// Tests if two Far files contains the same (key,fst) pairs.
+
+#include <fst/extensions/far/main.h>
+#include <fst/extensions/far/farscript.h>
+
+DEFINE_string(begin_key, "",
+ "First key to extract (def: first key in archive)");
+DEFINE_string(end_key, "",
+ "Last key to extract (def: last key in archive)");
+DEFINE_double(delta, fst::kDelta, "Comparison/quantization delta");
+
+int main(int argc, char **argv) {
+ namespace s = fst::script;
+
+ string usage = "Prints some basic information about the FSTs in an FST ";
+ usage += "archive.\n\n Usage:";
+ usage += argv[0];
+ usage += " in1.far in2.far\n";
+ usage += " Flags: begin_key end_key";
+
+ std::set_new_handler(FailedNewHandler);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
+
+ if (argc != 3) {
+ ShowUsage();
+ return 1;
+ }
+
+ string filename1(argv[1]), filename2(argv[2]);
+
+ bool result = s::FarEqual(
+ filename1, filename2, fst::LoadArcTypeFromFar(filename1),
+ FLAGS_delta, FLAGS_begin_key, FLAGS_end_key);
+
+ if (!result)
+ VLOG(1) << "FARs are not equal.";
+
+ return result ? 0 : 2;
+}
diff --git a/src/extensions/far/farextract.cc b/src/extensions/far/farextract.cc
index 72b2d15..66b7840 100644
--- a/src/extensions/far/farextract.cc
+++ b/src/extensions/far/farextract.cc
@@ -38,19 +38,15 @@
string usage = "Extracts FSTs from a finite-state archive.\n\n Usage:";
usage += argv[0];
- usage += " in1.far [in2.far...]\n";
+ usage += " [in1.far in2.far...]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
-
- if (argc < 2) {
- ShowUsage();
- return 1;
- }
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
vector<string> ifilenames;
for (int i = 1; i < argc; ++i)
- ifilenames.push_back(argv[i]);
+ ifilenames.push_back(strcmp(argv[i], "") != 0 ? argv[i] : "");
+ if (ifilenames.empty()) ifilenames.push_back("");
const string &arc_type = fst::LoadArcTypeFromFar(ifilenames[0]);
diff --git a/src/extensions/far/farinfo.cc b/src/extensions/far/farinfo.cc
index e8036a9..2665141 100644
--- a/src/extensions/far/farinfo.cc
+++ b/src/extensions/far/farinfo.cc
@@ -36,20 +36,17 @@
string usage = "Prints some basic information about the FSTs in an FST ";
usage += "archive.\n\n Usage:";
usage += argv[0];
- usage += " in1.far [in2.far...]\n";
+ usage += " [in1.far in2.far...]\n";
usage += " Flags: begin_key end_key list_fsts";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
-
- if (argc < 2) {
- ShowUsage();
- return 1;
- }
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
vector<string> filenames;
for (int i = 1; i < argc; ++i)
- filenames.push_back(argv[i]);
+ filenames.push_back(strcmp(argv[i], "") != 0 ? argv[i] : "");
+ if (filenames.empty())
+ filenames.push_back("");
s::FarInfo(filenames, fst::LoadArcTypeFromFar(filenames[0]),
FLAGS_begin_key, FLAGS_end_key, FLAGS_list_fsts);
diff --git a/src/extensions/far/farprintstrings.cc b/src/extensions/far/farprintstrings.cc
index ab7d52b..0204ab7 100644
--- a/src/extensions/far/farprintstrings.cc
+++ b/src/extensions/far/farprintstrings.cc
@@ -31,11 +31,14 @@
"Last key to extract (def: last key in archive)");
// PrintStringsMain specific flag definitions.
DEFINE_bool(print_key, false, "Prefix each string by its key");
+DEFINE_bool(print_weight, false, "Suffix each string by its weight");
DEFINE_string(entry_type, "line", "Entry type: one of : "
"\"file\" (one FST per file), \"line\" (one FST per line)");
DEFINE_string(token_type, "symbol", "Token type: one of : "
"\"symbol\", \"byte\", \"utf8\"");
DEFINE_string(symbols, "", "Label symbol table");
+DEFINE_bool(initial_symbols, true,
+ "Uses symbol table from the first Fst in archive for all entries.");
int main(int argc, char **argv) {
@@ -43,27 +46,25 @@
string usage = "Print as string the string FSTs in an archive.\n\n Usage:";
usage += argv[0];
- usage += " in1.far [in2.far ...]\n";
+ usage += " [in1.far in2.far ...]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
-
- if (argc < 2) {
- ShowUsage();
- return 1;
- }
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
vector<string> ifilenames;
for (int i = 1; i < argc; ++i)
- ifilenames.push_back(argv[i]);
+ ifilenames.push_back(strcmp(argv[i], "") != 0 ? argv[i] : "");
+ if (ifilenames.empty()) ifilenames.push_back("");
string arc_type = fst::LoadArcTypeFromFar(ifilenames[0]);
s::FarPrintStrings(ifilenames, arc_type,
fst::StringToFarEntryType(FLAGS_entry_type),
fst::StringToFarTokenType(FLAGS_token_type),
- FLAGS_begin_key, FLAGS_end_key, FLAGS_print_key,
- FLAGS_symbols, FLAGS_generate_filenames,
+ FLAGS_begin_key, FLAGS_end_key,
+ FLAGS_print_key, FLAGS_print_weight,
+ FLAGS_symbols, FLAGS_initial_symbols,
+ FLAGS_generate_filenames,
FLAGS_filename_prefix, FLAGS_filename_suffix);
return 0;
diff --git a/src/extensions/far/farscript.cc b/src/extensions/far/farscript.cc
index a04645d..bc4ad0e 100644
--- a/src/extensions/far/farscript.cc
+++ b/src/extensions/far/farscript.cc
@@ -34,14 +34,17 @@
FarTokenType tt,
const string &symbols_fname,
const string &unknown_symbol,
+ bool keep_symbols,
+ bool initial_symbols,
bool allow_negative_labels,
bool file_list_input,
const string &key_prefix,
const string &key_suffix) {
FarCompileStringsArgs args(in_fnames, out_fname, fst_type, far_type,
generate_keys, fet, tt, symbols_fname,
- unknown_symbol, allow_negative_labels,
- file_list_input, key_prefix, key_suffix);
+ unknown_symbol, keep_symbols, initial_symbols,
+ allow_negative_labels, file_list_input,
+ key_prefix, key_suffix);
Apply<Operation<FarCompileStringsArgs> >("FarCompileStrings", arc_type,
&args);
@@ -61,6 +64,16 @@
Apply<Operation<FarCreateArgs> >("FarCreate", arc_type, &args);
}
+bool FarEqual(const string &filename1, const string &filename2,
+ const string &arc_type, float delta,
+ const string &begin_key, const string &end_key) {
+ FarEqualInnerArgs args(filename1, filename2, delta, begin_key, end_key);
+ FarEqualArgs args_with_retval(args);
+
+ Apply<Operation<FarEqualArgs> >("FarEqual", arc_type, &args_with_retval);
+ return args_with_retval.retval;
+}
+
void FarExtract(const vector<string> &ifilenames,
const string &arc_type,
int32 generate_filenames, const string &begin_key,
@@ -89,15 +102,16 @@
const string &begin_key,
const string &end_key,
const bool print_key,
+ const bool print_weight,
const string &symbols_fname,
+ const bool initial_symbols,
const int32 generate_filenames,
const string &filename_prefix,
const string &filename_suffix) {
- FarPrintStringsArgs args(ifilenames, entry_type, token_type, begin_key,
- end_key, print_key, symbols_fname,
- generate_filenames,
- filename_prefix,
- filename_suffix);
+ FarPrintStringsArgs args(ifilenames, entry_type, token_type,
+ begin_key, end_key, print_key, print_weight,
+ symbols_fname, initial_symbols, generate_filenames,
+ filename_prefix, filename_suffix);
Apply<Operation<FarPrintStringsArgs> >("FarPrintStrings", arc_type,
&args);
diff --git a/src/extensions/far/main.cc b/src/extensions/far/main.cc
index b01d639..855def6 100644
--- a/src/extensions/far/main.cc
+++ b/src/extensions/far/main.cc
@@ -26,6 +26,7 @@
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/extensions/far/main.h>
namespace fst {
@@ -33,7 +34,9 @@
// Return the 'FarType' value corresponding to a far type name.
FarType FarTypeFromString(const string &str) {
FarType type = FAR_DEFAULT;
- if (str == "stlist")
+ if (str == "fst")
+ type = FAR_FST;
+ else if (str == "stlist")
type = FAR_STLIST;
else if (str == "sttable")
type = FAR_STTABLE;
@@ -46,6 +49,8 @@
// Return the textual name corresponding to a 'FarType;.
string FarTypeToString(FarType type) {
switch (type) {
+ case FAR_FST:
+ return "fst";
case FAR_STLIST:
return "stlist";
case FAR_STTABLE:
@@ -85,19 +90,14 @@
string LoadArcTypeFromFar(const string &far_fname) {
FarHeader hdr;
- if (far_fname.empty()) {
- LOG(ERROR) << "Reading FAR from standard in not supported";
- return "";
- }
-
if (!hdr.Read(far_fname)) {
- LOG(ERROR) << "Error reading FAR: " << far_fname;
+ FSTERROR() << "Error reading FAR: " << far_fname;
return "";
}
string atype = hdr.ArcType();
if (atype == "unknown") {
- LOG(ERROR) << "Empty FST archive: " << far_fname;
+ FSTERROR() << "Empty FST archive: " << far_fname;
return "";
}
@@ -108,7 +108,7 @@
FstHeader hdr;
ifstream in(fst_fname.c_str(), ifstream::in | ifstream::binary);
if (!hdr.Read(in, fst_fname)) {
- LOG(ERROR) << "Error reading FST: " << fst_fname;
+ FSTERROR() << "Error reading FST: " << fst_fname;
return "";
}
diff --git a/src/extensions/far/compile-strings.cc b/src/extensions/far/strings.cc
similarity index 88%
rename from src/extensions/far/compile-strings.cc
rename to src/extensions/far/strings.cc
index e8a99cc..c62a45b 100644
--- a/src/extensions/far/compile-strings.cc
+++ b/src/extensions/far/strings.cc
@@ -17,6 +17,10 @@
#include <fst/extensions/far/compile-strings.h>
#include <iostream>
#include <fstream>
+#include <sstream>
+
+DEFINE_string(far_field_separator, "\t",
+ "Set of characters used as a separator between printed fields");
namespace fst {
diff --git a/src/extensions/ngram/Makefile.am b/src/extensions/ngram/Makefile.am
new file mode 100644
index 0000000..e909eaf
--- /dev/null
+++ b/src/extensions/ngram/Makefile.am
@@ -0,0 +1,10 @@
+AM_CPPFLAGS = -I$(srcdir)/../../include $(ICU_CPPFLAGS)
+
+libfstdir = @libfstdir@
+libfst_LTLIBRARIES = libfstngram.la ngram-fst.la
+
+ngram_fst_la_SOURCES = bitmap-index.cc ngram-fst.cc nthbit.cc
+ngram_fst_la_LDFLAGS = -module
+
+libfstngram_la_SOURCES = bitmap-index.cc ngram-fst.cc nthbit.cc
+libfstngram_la_LDFLAGS = -version-info 0:0:0
diff --git a/src/extensions/ngram/Makefile.in b/src/extensions/ngram/Makefile.in
new file mode 100644
index 0000000..cce1255
--- /dev/null
+++ b/src/extensions/ngram/Makefile.in
@@ -0,0 +1,543 @@
+# Makefile.in generated by automake 1.11.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+# 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation,
+# Inc.
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+VPATH = @srcdir@
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+subdir = src/extensions/ngram
+DIST_COMMON = $(srcdir)/Makefile.am $(srcdir)/Makefile.in
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_icu.m4 \
+ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \
+ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+ $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h \
+ $(top_builddir)/src/include/fst/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__installdirs = "$(DESTDIR)$(libfstdir)"
+LTLIBRARIES = $(libfst_LTLIBRARIES)
+libfstngram_la_LIBADD =
+am_libfstngram_la_OBJECTS = bitmap-index.lo ngram-fst.lo nthbit.lo
+libfstngram_la_OBJECTS = $(am_libfstngram_la_OBJECTS)
+libfstngram_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+ $(CXXFLAGS) $(libfstngram_la_LDFLAGS) $(LDFLAGS) -o $@
+ngram_fst_la_LIBADD =
+am_ngram_fst_la_OBJECTS = bitmap-index.lo ngram-fst.lo nthbit.lo
+ngram_fst_la_OBJECTS = $(am_ngram_fst_la_OBJECTS)
+ngram_fst_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \
+ $(CXXFLAGS) $(ngram_fst_la_LDFLAGS) $(LDFLAGS) -o $@
+DEFAULT_INCLUDES =
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__depfiles_maybe = depfiles
+am__mv = mv -f
+CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+ $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
+LTCXXCOMPILE = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+ --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
+ $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
+CXXLD = $(CXX)
+CXXLINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \
+ --mode=link $(CXXLD) $(AM_CXXFLAGS) $(CXXFLAGS) $(AM_LDFLAGS) \
+ $(LDFLAGS) -o $@
+SOURCES = $(libfstngram_la_SOURCES) $(ngram_fst_la_SOURCES)
+DIST_SOURCES = $(libfstngram_la_SOURCES) $(ngram_fst_la_SOURCES)
+ETAGS = etags
+CTAGS = ctags
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+AMTAR = @AMTAR@
+AR = @AR@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CFLAGS = @CFLAGS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+GREP = @GREP@
+ICU_CFLAGS = @ICU_CFLAGS@
+ICU_CONFIG = @ICU_CONFIG@
+ICU_CPPFLAGS = @ICU_CPPFLAGS@
+ICU_CXXFLAGS = @ICU_CXXFLAGS@
+ICU_LIBS = @ICU_LIBS@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+LD = @LD@
+LDFLAGS = @LDFLAGS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBTOOL = @LIBTOOL@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBOBJS = @LTLIBOBJS@
+MAKEINFO = @MAKEINFO@
+MKDIR_P = @MKDIR_P@
+NM = @NM@
+NMEDIT = @NMEDIT@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+RANLIB = @RANLIB@
+SED = @SED@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+STRIP = @STRIP@
+VERSION = @VERSION@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+libfstdir = @libfstdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+lt_ECHO = @lt_ECHO@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+srcdir = @srcdir@
+sysconfdir = @sysconfdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+AM_CPPFLAGS = -I$(srcdir)/../../include $(ICU_CPPFLAGS)
+libfst_LTLIBRARIES = libfstngram.la ngram-fst.la
+ngram_fst_la_SOURCES = bitmap-index.cc ngram-fst.cc nthbit.cc
+ngram_fst_la_LDFLAGS = -module
+libfstngram_la_SOURCES = bitmap-index.cc ngram-fst.cc nthbit.cc
+libfstngram_la_LDFLAGS = -version-info 0:0:0
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .cc .lo .o .obj
+$(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/extensions/ngram/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --foreign src/extensions/ngram/Makefile
+.PRECIOUS: Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__depfiles_maybe);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+install-libfstLTLIBRARIES: $(libfst_LTLIBRARIES)
+ @$(NORMAL_INSTALL)
+ test -z "$(libfstdir)" || $(MKDIR_P) "$(DESTDIR)$(libfstdir)"
+ @list='$(libfst_LTLIBRARIES)'; test -n "$(libfstdir)" || list=; \
+ list2=; for p in $$list; do \
+ if test -f $$p; then \
+ list2="$$list2 $$p"; \
+ else :; fi; \
+ done; \
+ test -z "$$list2" || { \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(libfstdir)'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(libfstdir)"; \
+ }
+
+uninstall-libfstLTLIBRARIES:
+ @$(NORMAL_UNINSTALL)
+ @list='$(libfst_LTLIBRARIES)'; test -n "$(libfstdir)" || list=; \
+ for p in $$list; do \
+ $(am__strip_dir) \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(libfstdir)/$$f'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(libfstdir)/$$f"; \
+ done
+
+clean-libfstLTLIBRARIES:
+ -test -z "$(libfst_LTLIBRARIES)" || rm -f $(libfst_LTLIBRARIES)
+ @list='$(libfst_LTLIBRARIES)'; for p in $$list; do \
+ dir="`echo $$p | sed -e 's|/[^/]*$$||'`"; \
+ test "$$dir" != "$$p" || dir=.; \
+ echo "rm -f \"$${dir}/so_locations\""; \
+ rm -f "$${dir}/so_locations"; \
+ done
+libfstngram.la: $(libfstngram_la_OBJECTS) $(libfstngram_la_DEPENDENCIES)
+ $(libfstngram_la_LINK) -rpath $(libfstdir) $(libfstngram_la_OBJECTS) $(libfstngram_la_LIBADD) $(LIBS)
+ngram-fst.la: $(ngram_fst_la_OBJECTS) $(ngram_fst_la_DEPENDENCIES)
+ $(ngram_fst_la_LINK) -rpath $(libfstdir) $(ngram_fst_la_OBJECTS) $(ngram_fst_la_LIBADD) $(LIBS)
+
+mostlyclean-compile:
+ -rm -f *.$(OBJEXT)
+
+distclean-compile:
+ -rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bitmap-index.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ngram-fst.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/nthbit.Plo@am__quote@
+
+.cc.o:
+@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ $<
+
+.cc.obj:
+@am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.cc.lo:
+@am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCXX_TRUE@ $(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+ -rm -f *.lo
+
+clean-libtool:
+ -rm -rf .libs _libs
+
+ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES)
+ list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | \
+ $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in files) print i; }; }'`; \
+ mkid -fID $$unique
+tags: TAGS
+
+TAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
+ $(TAGS_FILES) $(LISP)
+ set x; \
+ here=`pwd`; \
+ list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | \
+ $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in files) print i; }; }'`; \
+ shift; \
+ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+ test -n "$$unique" || unique=$$empty_fix; \
+ if test $$# -gt 0; then \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ "$$@" $$unique; \
+ else \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ $$unique; \
+ fi; \
+ fi
+ctags: CTAGS
+CTAGS: $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \
+ $(TAGS_FILES) $(LISP)
+ list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | \
+ $(AWK) '{ files[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in files) print i; }; }'`; \
+ test -z "$(CTAGS_ARGS)$$unique" \
+ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+ $$unique
+
+GTAGS:
+ here=`$(am__cd) $(top_builddir) && pwd` \
+ && $(am__cd) $(top_srcdir) \
+ && gtags -i $(GTAGS_ARGS) "$$here"
+
+distclean-tags:
+ -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+check: check-am
+all-am: Makefile $(LTLIBRARIES)
+installdirs:
+ for dir in "$(DESTDIR)$(libfstdir)"; do \
+ test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+ done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ `test -z '$(STRIP)' || \
+ echo "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'"` install
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libfstLTLIBRARIES clean-libtool \
+ mostlyclean-am
+
+distclean: distclean-am
+ -rm -rf ./$(DEPDIR)
+ -rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+ distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-libfstLTLIBRARIES
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am:
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+ -rm -rf ./$(DEPDIR)
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+ mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-libfstLTLIBRARIES
+
+.MAKE: install-am install-strip
+
+.PHONY: CTAGS GTAGS all all-am check check-am clean clean-generic \
+ clean-libfstLTLIBRARIES clean-libtool ctags distclean \
+ distclean-compile distclean-generic distclean-libtool \
+ distclean-tags distdir dvi dvi-am html html-am info info-am \
+ install install-am install-data install-data-am install-dvi \
+ install-dvi-am install-exec install-exec-am install-html \
+ install-html-am install-info install-info-am \
+ install-libfstLTLIBRARIES install-man install-pdf \
+ install-pdf-am install-ps install-ps-am install-strip \
+ installcheck installcheck-am installdirs maintainer-clean \
+ maintainer-clean-generic mostlyclean mostlyclean-compile \
+ mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \
+ tags uninstall uninstall-am uninstall-libfstLTLIBRARIES
+
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/extensions/ngram/bitmap-index.cc b/src/extensions/ngram/bitmap-index.cc
new file mode 100644
index 0000000..c6aaa14
--- /dev/null
+++ b/src/extensions/ngram/bitmap-index.cc
@@ -0,0 +1,187 @@
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: Jeffrey Soresnen (sorenj@google.com)
+
+#include <fst/extensions/ngram/bitmap-index.h>
+
+#include <algorithm>
+#include <iterator>
+
+#include <fst/extensions/ngram/nthbit.h>
+
+namespace fst {
+
+// These two internal classes implemented inverted views of the
+// primary and secondary indexes. That is, they provide iterators
+// that have operator*'s that return the number zeros rather than
+// the number of ones.
+
+class primary_index_inverted : public vector<uint32>::const_iterator {
+ public:
+ primary_index_inverted() {}
+ primary_index_inverted(vector<uint32>::const_iterator loc,
+ vector<uint32>::const_iterator begin) :
+ vector<uint32>::const_iterator(loc), begin_(begin) {}
+ uint32 operator*() {
+ return BitmapIndex::kStorageBitSize * BitmapIndex::kSecondaryBlockSize *
+ (1 + std::distance<vector<uint32>::const_iterator>(begin_, *this)) -
+ vector<uint32>::const_iterator::operator*();
+ }
+ private:
+ vector<uint32>::const_iterator begin_;
+};
+
+class secondary_index_inverted : public vector<uint16>::const_iterator {
+ public:
+ secondary_index_inverted() : vector<uint16>::const_iterator() {}
+ secondary_index_inverted(vector<uint16>::const_iterator loc,
+ vector<uint16>::const_iterator block_begin) :
+ vector<uint16>::const_iterator(loc), block_begin_(block_begin) {}
+ uint16 operator*() {
+ return ((1 + std::distance<vector<uint16>::const_iterator>(
+ block_begin_, *this)) << BitmapIndex::kStorageLogBitSize) -
+ vector<uint16>::const_iterator::operator*();
+ }
+ private:
+ vector<uint16>::const_iterator block_begin_;
+};
+
+size_t BitmapIndex::Rank1(size_t end) const {
+ if (end == 0) return 0;
+ CHECK_LE(end, Bits());
+ const uint32 end_word = (end - 1) >> BitmapIndex::kStorageLogBitSize;
+ const uint32 sum = get_index_ones_count(end_word);
+ const uint64 zero = 0;
+ const uint64 ones = ~zero;
+ return sum + __builtin_popcountll(bits_[end_word] &
+ (ones >> (kStorageBitSize - (end & kStorageBlockMask))));
+}
+
+size_t BitmapIndex::Select1(size_t bit_index) const {
+ if (bit_index >= GetOnesCount()) return Bits();
+ // search primary index for the relevant block
+ uint32 rembits = bit_index + 1;
+ const uint32 block = find_primary_block(bit_index + 1);
+ uint32 offset = 0;
+ if (block > 0) {
+ rembits -= primary_index_[block - 1];
+ offset += block * kSecondaryBlockSize;
+ }
+ // search the secondary index
+ uint32 word = find_secondary_block(offset, rembits);
+ if (word > 0) {
+ rembits -= secondary_index_[offset + word - 1];
+ offset += word;
+ }
+ int nth = nth_bit(bits_[offset], rembits);
+ return (offset << BitmapIndex::kStorageLogBitSize) + nth;
+}
+
+size_t BitmapIndex::Select0(size_t bit_index) const {
+ if (bit_index >= Bits() - GetOnesCount()) return Bits();
+ // search inverted primary index for relevant block
+ uint32 remzeros = bit_index + 1;
+ uint32 offset = 0;
+ const uint32 block = find_inverted_primary_block(bit_index + 1);
+ if (block > 0) {
+ remzeros -= *primary_index_inverted(primary_index_.begin() + block - 1,
+ primary_index_.begin());
+ offset += block * kSecondaryBlockSize;
+ }
+ // search the inverted secondary index
+ uint32 word = find_inverted_secondary_block(offset, remzeros);
+ if (word > 0) {
+ vector<uint16>::const_iterator block_begin =
+ secondary_index_.begin() + offset;
+ remzeros -= *secondary_index_inverted(block_begin + word - 1, block_begin);
+ offset += word;
+ }
+ int nth = nth_bit(~bits_[offset], remzeros);
+ return (offset << BitmapIndex::kStorageLogBitSize) + nth;
+}
+
+size_t BitmapIndex::get_index_ones_count(size_t array_index) const {
+ uint32 sum = 0;
+ if (array_index > 0) {
+ sum += secondary_index_[array_index-1];
+ uint32 end_block = (array_index - 1) / kSecondaryBlockSize;
+ if (end_block > 0) sum += primary_index_[end_block-1];
+ }
+ return sum;
+}
+
+void BitmapIndex::BuildIndex(const uint64 *bits, size_t size) {
+ bits_ = bits;
+ size_ = size;
+ secondary_index_.clear();
+ secondary_index_.reserve(ArraySize());
+ primary_index_.clear();
+ primary_index_.reserve(primary_index_size());
+ const uint64 zero = 0;
+ const uint64 ones = ~zero;
+ uint32 popcount = 0;
+ for (uint32 block_begin = 0; block_begin < ArraySize();
+ block_begin += kSecondaryBlockSize) {
+ uint32 block_popcount = 0;
+ uint32 block_end = block_begin + kSecondaryBlockSize;
+ if (block_end > ArraySize()) block_end = ArraySize();
+ for (uint32 j = block_begin; j < block_end; ++j) {
+ uint64 mask = ones;
+ if (j == ArraySize() - 1) {
+ mask = ones >> (-size_ & BitmapIndex::kStorageBlockMask);
+ }
+ block_popcount += __builtin_popcountll(bits_[j] & mask);
+ secondary_index_.push_back(block_popcount);
+ }
+ popcount += block_popcount;
+ primary_index_.push_back(popcount);
+ }
+}
+
+size_t BitmapIndex::find_secondary_block(
+ size_t block_begin, size_t rem_bit_index) const {
+ size_t block_end = block_begin + kSecondaryBlockSize;
+ if (block_end > secondary_index_.size()) block_end = secondary_index_.size();
+ return std::distance(secondary_index_.begin() + block_begin,
+ std::lower_bound(secondary_index_.begin() + block_begin,
+ secondary_index_.begin() + block_end,
+ rem_bit_index));
+}
+
+size_t BitmapIndex::find_inverted_secondary_block(
+ size_t block_begin, size_t rem_bit_index) const {
+ size_t block_end = block_begin + kSecondaryBlockSize;
+ if (block_end > secondary_index_.size()) block_end = secondary_index_.size();
+ secondary_index_inverted start(secondary_index_.begin() + block_begin,
+ secondary_index_.begin() + block_begin);
+ secondary_index_inverted end(secondary_index_.begin() + block_end,
+ secondary_index_.begin() + block_begin);
+ return std::distance(start,
+ std::lower_bound(start, end, rem_bit_index));
+}
+
+inline size_t BitmapIndex::find_primary_block(size_t bit_index) const {
+ return std::distance(primary_index_.begin(),
+ std::lower_bound(primary_index_.begin(),
+ primary_index_.end(), bit_index));
+}
+
+size_t BitmapIndex::find_inverted_primary_block(size_t bit_index) const {
+ primary_index_inverted start(primary_index_.begin(), primary_index_.begin());
+ primary_index_inverted end(primary_index_.end(), primary_index_.begin());
+ return std::distance(start, std::lower_bound(start, end, bit_index));
+}
+
+} // end namespace fst
diff --git a/src/extensions/ngram/ngram-fst.cc b/src/extensions/ngram/ngram-fst.cc
new file mode 100644
index 0000000..18f88db
--- /dev/null
+++ b/src/extensions/ngram/ngram-fst.cc
@@ -0,0 +1,28 @@
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: sorenj@google.com (Jeffrey Sorensen)
+
+#include <fst/extensions/ngram/ngram-fst.h>
+
+#include <sys/types.h>
+
+#include <fst/fstlib.h>
+
+using fst::NGramFst;
+using fst::StdArc;
+using fst::LogArc;
+
+REGISTER_FST(NGramFst, StdArc);
+REGISTER_FST(NGramFst, LogArc);
diff --git a/src/extensions/ngram/nthbit.cc b/src/extensions/ngram/nthbit.cc
new file mode 100644
index 0000000..7b006e7
--- /dev/null
+++ b/src/extensions/ngram/nthbit.cc
@@ -0,0 +1,107 @@
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: sorenj@google.com (Jeffrey Sorensen
+
+#include <fst/extensions/ngram/nthbit.h>
+
+//
+// This table is generate using:
+//
+// unsigned int nth_bit_scan(uint64 v, unsigned int r) {
+// int i=0;
+// for (; i<64; i++) {
+// if ((r -= v & 1) == 0) return i;
+// v >>= 1;
+// }
+// return i;
+// }
+//
+// for (size_t i = 0; i < 256; ++i) {
+// uint32 offsets = 0;
+// for (size_t b = 1; b <= 8; ++b) {
+// uint32 offset = min<uint32>(nth_bit_scan(i, b), 8);
+// offsets |= (offset << ((b - 1) << 2));
+// }
+// bit_offset = offsets;
+// printf("0x%x, ", bit_offset);
+// if (i % 4 == 3) printf("\n");
+// }
+//
+uint32 nth_bit_bit_offset[] = {
+0x88888888, 0x88888880, 0x88888881, 0x88888810,
+0x88888882, 0x88888820, 0x88888821, 0x88888210,
+0x88888883, 0x88888830, 0x88888831, 0x88888310,
+0x88888832, 0x88888320, 0x88888321, 0x88883210,
+0x88888884, 0x88888840, 0x88888841, 0x88888410,
+0x88888842, 0x88888420, 0x88888421, 0x88884210,
+0x88888843, 0x88888430, 0x88888431, 0x88884310,
+0x88888432, 0x88884320, 0x88884321, 0x88843210,
+0x88888885, 0x88888850, 0x88888851, 0x88888510,
+0x88888852, 0x88888520, 0x88888521, 0x88885210,
+0x88888853, 0x88888530, 0x88888531, 0x88885310,
+0x88888532, 0x88885320, 0x88885321, 0x88853210,
+0x88888854, 0x88888540, 0x88888541, 0x88885410,
+0x88888542, 0x88885420, 0x88885421, 0x88854210,
+0x88888543, 0x88885430, 0x88885431, 0x88854310,
+0x88885432, 0x88854320, 0x88854321, 0x88543210,
+0x88888886, 0x88888860, 0x88888861, 0x88888610,
+0x88888862, 0x88888620, 0x88888621, 0x88886210,
+0x88888863, 0x88888630, 0x88888631, 0x88886310,
+0x88888632, 0x88886320, 0x88886321, 0x88863210,
+0x88888864, 0x88888640, 0x88888641, 0x88886410,
+0x88888642, 0x88886420, 0x88886421, 0x88864210,
+0x88888643, 0x88886430, 0x88886431, 0x88864310,
+0x88886432, 0x88864320, 0x88864321, 0x88643210,
+0x88888865, 0x88888650, 0x88888651, 0x88886510,
+0x88888652, 0x88886520, 0x88886521, 0x88865210,
+0x88888653, 0x88886530, 0x88886531, 0x88865310,
+0x88886532, 0x88865320, 0x88865321, 0x88653210,
+0x88888654, 0x88886540, 0x88886541, 0x88865410,
+0x88886542, 0x88865420, 0x88865421, 0x88654210,
+0x88886543, 0x88865430, 0x88865431, 0x88654310,
+0x88865432, 0x88654320, 0x88654321, 0x86543210,
+0x88888887, 0x88888870, 0x88888871, 0x88888710,
+0x88888872, 0x88888720, 0x88888721, 0x88887210,
+0x88888873, 0x88888730, 0x88888731, 0x88887310,
+0x88888732, 0x88887320, 0x88887321, 0x88873210,
+0x88888874, 0x88888740, 0x88888741, 0x88887410,
+0x88888742, 0x88887420, 0x88887421, 0x88874210,
+0x88888743, 0x88887430, 0x88887431, 0x88874310,
+0x88887432, 0x88874320, 0x88874321, 0x88743210,
+0x88888875, 0x88888750, 0x88888751, 0x88887510,
+0x88888752, 0x88887520, 0x88887521, 0x88875210,
+0x88888753, 0x88887530, 0x88887531, 0x88875310,
+0x88887532, 0x88875320, 0x88875321, 0x88753210,
+0x88888754, 0x88887540, 0x88887541, 0x88875410,
+0x88887542, 0x88875420, 0x88875421, 0x88754210,
+0x88887543, 0x88875430, 0x88875431, 0x88754310,
+0x88875432, 0x88754320, 0x88754321, 0x87543210,
+0x88888876, 0x88888760, 0x88888761, 0x88887610,
+0x88888762, 0x88887620, 0x88887621, 0x88876210,
+0x88888763, 0x88887630, 0x88887631, 0x88876310,
+0x88887632, 0x88876320, 0x88876321, 0x88763210,
+0x88888764, 0x88887640, 0x88887641, 0x88876410,
+0x88887642, 0x88876420, 0x88876421, 0x88764210,
+0x88887643, 0x88876430, 0x88876431, 0x88764310,
+0x88876432, 0x88764320, 0x88764321, 0x87643210,
+0x88888765, 0x88887650, 0x88887651, 0x88876510,
+0x88887652, 0x88876520, 0x88876521, 0x88765210,
+0x88887653, 0x88876530, 0x88876531, 0x88765310,
+0x88876532, 0x88765320, 0x88765321, 0x87653210,
+0x88887654, 0x88876540, 0x88876541, 0x88765410,
+0x88876542, 0x88765420, 0x88765421, 0x87654210,
+0x88876543, 0x88765430, 0x88765431, 0x87654310,
+0x88765432, 0x87654320, 0x87654321, 0x76543210,
+};
diff --git a/src/extensions/pdt/pdtcompose.cc b/src/extensions/pdt/pdtcompose.cc
index 0bab405..0076e1d 100644
--- a/src/extensions/pdt/pdtcompose.cc
+++ b/src/extensions/pdt/pdtcompose.cc
@@ -41,7 +41,7 @@
usage += " in.fst in.pdt [out.pdt]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 3 || argc > 4) {
ShowUsage();
return 1;
diff --git a/src/extensions/pdt/pdtexpand.cc b/src/extensions/pdt/pdtexpand.cc
index 2134932..18d849d 100644
--- a/src/extensions/pdt/pdtexpand.cc
+++ b/src/extensions/pdt/pdtexpand.cc
@@ -17,7 +17,7 @@
// Modified: jpr@google.com (Jake Ratkiewicz) to use FstClass
//
// \file
-// Expands a PDT and an FST.
+// Expands a (bounded-stack) PDT as an FST.
//
#include <fst/extensions/pdt/pdtscript.h>
@@ -32,12 +32,12 @@
int main(int argc, char **argv) {
namespace s = fst::script;
- string usage = "Expand a PDT and an FST.\n\n Usage: ";
+ string usage = "Expand a (bounded-stack) PDT as an FST.\n\n Usage: ";
usage += argv[0];
usage += " in.pdt [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/extensions/pdt/pdtinfo.cc b/src/extensions/pdt/pdtinfo.cc
index 89227d3..71c0ed0 100644
--- a/src/extensions/pdt/pdtinfo.cc
+++ b/src/extensions/pdt/pdtinfo.cc
@@ -34,7 +34,7 @@
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 2) {
ShowUsage();
return 1;
diff --git a/src/extensions/pdt/pdtreplace.cc b/src/extensions/pdt/pdtreplace.cc
index 543b6ea..9f8aa86 100644
--- a/src/extensions/pdt/pdtreplace.cc
+++ b/src/extensions/pdt/pdtreplace.cc
@@ -15,6 +15,7 @@
// Copyright 2005-2010 Google, Inc.
// Author: riley@google.com (Michael Riley)
//
+// Converts an RTN represented by FSTs and non-terminal labels into a PDT .
#include <utility>
using std::pair; using std::make_pair;
@@ -29,13 +30,14 @@
int main(int argc, char **argv) {
namespace s = fst::script;
- string usage = "Recursively replace Fst arcs with other Fst(s).\n";
+ string usage = "Converts an RTN represented by FSTs";
+ usage += " and non-terminal labels into PDT";
usage += " Usage: ";
usage += argv[0];
usage += " root.fst rootlabel [rule1.fst label1 ...] [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc < 4) {
ShowUsage();
return 1;
diff --git a/src/extensions/pdt/pdtreverse.cc b/src/extensions/pdt/pdtreverse.cc
index e79cd7f..48f412d 100644
--- a/src/extensions/pdt/pdtreverse.cc
+++ b/src/extensions/pdt/pdtreverse.cc
@@ -32,7 +32,7 @@
usage += " in.pdt [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/extensions/pdt/pdtshortestpath.cc b/src/extensions/pdt/pdtshortestpath.cc
index ad67969..5973db9 100644
--- a/src/extensions/pdt/pdtshortestpath.cc
+++ b/src/extensions/pdt/pdtshortestpath.cc
@@ -17,7 +17,7 @@
// Modified: jpr@google.com (Jake Ratkiewicz) to use FstClass
//
// \file
-// Return the shortest path in a PDT.
+// Return the shortest path in a (bounded-stack) PDT.
//
#include <fst/extensions/pdt/pdtscript.h>
@@ -33,12 +33,12 @@
int main(int argc, char **argv) {
namespace s = fst::script;
- string usage = "Shortest path in a PDT.\n\n Usage: ";
+ string usage = "Shortest path in a (bounded-stack) PDT.\n\n Usage: ";
usage += argv[0];
usage += " in.pdt [out.fst]\n";
std::set_new_handler(FailedNewHandler);
- SetFlags(usage.c_str(), &argc, &argv, true);
+ SET_FLAGS(usage.c_str(), &argc, &argv, true);
if (argc > 3) {
ShowUsage();
return 1;
diff --git a/src/include/Makefile.am b/src/include/Makefile.am
index b4e6213..e9eb92f 100644
--- a/src/include/Makefile.am
+++ b/src/include/Makefile.am
@@ -1,10 +1,17 @@
if HAVE_FAR
far_include_headers = fst/extensions/far/compile-strings.h \
-fst/extensions/far/create.h fst/extensions/far/extract.h \
-fst/extensions/far/far.h fst/extensions/far/farlib.h \
-fst/extensions/far/farscript.h fst/extensions/far/info.h \
-fst/extensions/far/main.h fst/extensions/far/print-strings.h \
-fst/extensions/far/stlist.h fst/extensions/far/sttable.h
+fst/extensions/far/create.h fst/extensions/far/equal.h \
+fst/extensions/far/extract.h fst/extensions/far/far.h \
+fst/extensions/far/farlib.h fst/extensions/far/farscript.h \
+fst/extensions/far/info.h fst/extensions/far/main.h \
+fst/extensions/far/print-strings.h fst/extensions/far/stlist.h \
+fst/extensions/far/sttable.h
+endif
+
+if HAVE_NGRAM
+ngram_include_headers = fst/extensions/ngram/bitmap-index.h \
+fst/extensions/ngram/ngram-fst.h \
+fst/extensions/ngram/nthbit.h
endif
if HAVE_PDT
@@ -62,6 +69,6 @@
fst/sparse-tuple-weight.h fst/sparse-power-weight.h fst/expectation-weight.h \
fst/symbol-table-ops.h fst/bi-table.h \
$(far_include_headers) \
+$(ngram_include_headers) \
$(pdt_include_headers) \
$(script_include_headers)
-
diff --git a/src/include/Makefile.in b/src/include/Makefile.in
index ab6c28d..b4e3f86 100644
--- a/src/include/Makefile.in
+++ b/src/include/Makefile.in
@@ -82,11 +82,14 @@
fst/sparse-power-weight.h fst/expectation-weight.h \
fst/symbol-table-ops.h fst/bi-table.h \
fst/extensions/far/compile-strings.h \
- fst/extensions/far/create.h fst/extensions/far/extract.h \
- fst/extensions/far/far.h fst/extensions/far/farlib.h \
- fst/extensions/far/farscript.h fst/extensions/far/info.h \
- fst/extensions/far/main.h fst/extensions/far/print-strings.h \
- fst/extensions/far/stlist.h fst/extensions/far/sttable.h \
+ fst/extensions/far/create.h fst/extensions/far/equal.h \
+ fst/extensions/far/extract.h fst/extensions/far/far.h \
+ fst/extensions/far/farlib.h fst/extensions/far/farscript.h \
+ fst/extensions/far/info.h fst/extensions/far/main.h \
+ fst/extensions/far/print-strings.h fst/extensions/far/stlist.h \
+ fst/extensions/far/sttable.h \
+ fst/extensions/ngram/bitmap-index.h \
+ fst/extensions/ngram/ngram-fst.h fst/extensions/ngram/nthbit.h \
fst/extensions/pdt/collection.h fst/extensions/pdt/compose.h \
fst/extensions/pdt/expand.h fst/extensions/pdt/info.h \
fst/extensions/pdt/paren.h fst/extensions/pdt/pdt.h \
@@ -264,11 +267,16 @@
top_builddir = @top_builddir@
top_srcdir = @top_srcdir@
@HAVE_FAR_TRUE@far_include_headers = fst/extensions/far/compile-strings.h \
-@HAVE_FAR_TRUE@fst/extensions/far/create.h fst/extensions/far/extract.h \
-@HAVE_FAR_TRUE@fst/extensions/far/far.h fst/extensions/far/farlib.h \
-@HAVE_FAR_TRUE@fst/extensions/far/farscript.h fst/extensions/far/info.h \
-@HAVE_FAR_TRUE@fst/extensions/far/main.h fst/extensions/far/print-strings.h \
-@HAVE_FAR_TRUE@fst/extensions/far/stlist.h fst/extensions/far/sttable.h
+@HAVE_FAR_TRUE@fst/extensions/far/create.h fst/extensions/far/equal.h \
+@HAVE_FAR_TRUE@fst/extensions/far/extract.h fst/extensions/far/far.h \
+@HAVE_FAR_TRUE@fst/extensions/far/farlib.h fst/extensions/far/farscript.h \
+@HAVE_FAR_TRUE@fst/extensions/far/info.h fst/extensions/far/main.h \
+@HAVE_FAR_TRUE@fst/extensions/far/print-strings.h fst/extensions/far/stlist.h \
+@HAVE_FAR_TRUE@fst/extensions/far/sttable.h
+
+@HAVE_NGRAM_TRUE@ngram_include_headers = fst/extensions/ngram/bitmap-index.h \
+@HAVE_NGRAM_TRUE@fst/extensions/ngram/ngram-fst.h \
+@HAVE_NGRAM_TRUE@fst/extensions/ngram/nthbit.h
@HAVE_PDT_TRUE@pdt_include_headers = fst/extensions/pdt/collection.h \
@HAVE_PDT_TRUE@fst/extensions/pdt/compose.h fst/extensions/pdt/expand.h \
@@ -323,6 +331,7 @@
fst/sparse-tuple-weight.h fst/sparse-power-weight.h fst/expectation-weight.h \
fst/symbol-table-ops.h fst/bi-table.h \
$(far_include_headers) \
+$(ngram_include_headers) \
$(pdt_include_headers) \
$(script_include_headers)
diff --git a/src/include/fst/accumulator.h b/src/include/fst/accumulator.h
index fcb960c..9801b93 100644
--- a/src/include/fst/accumulator.h
+++ b/src/include/fst/accumulator.h
@@ -258,7 +258,7 @@
for(StateIterator<F> siter(fst); !siter.Done(); siter.Next()) {
StateId s = siter.Value();
if (fst.NumArcs(s) >= arc_limit_) {
- double sum = FloatLimits<double>::kPosInfinity;
+ double sum = FloatLimits<double>::PosInfinity();
weight_positions.push_back(weight_position);
weights.push_back(sum);
++weight_position;
@@ -282,12 +282,12 @@
private:
double LogPosExp(double x) {
- return x == FloatLimits<double>::kPosInfinity ?
+ return x == FloatLimits<double>::PosInfinity() ?
0.0 : log(1.0F + exp(-x));
}
double LogMinusExp(double x) {
- return x == FloatLimits<double>::kPosInfinity ?
+ return x == FloatLimits<double>::PosInfinity() ?
0.0 : log(1.0F - exp(-x));
}
@@ -302,7 +302,7 @@
double LogPlus(double f1, Weight v) {
double f2 = to_log_weight_(v).Value();
- if (f1 == FloatLimits<double>::kPosInfinity)
+ if (f1 == FloatLimits<double>::PosInfinity())
return f2;
else if (f1 > f2)
return f2 - LogPosExp(f1 - f2);
@@ -317,7 +317,7 @@
error_ = true;
return Weight::NoWeight();
}
- if (f2 == FloatLimits<double>::kPosInfinity)
+ if (f2 == FloatLimits<double>::PosInfinity())
return to_weight_(f1);
else
return to_weight_(f1 - LogMinusExp(f2 - f1));
@@ -485,7 +485,7 @@
if ((weights_ == 0) && (fst_->NumArcs(s) >= arc_limit_)) {
weights_ = new vector<double>;
weights_->reserve(fst_->NumArcs(s) + 1);
- weights_->push_back(FloatLimits<double>::kPosInfinity);
+ weights_->push_back(FloatLimits<double>::PosInfinity());
data_->AddWeights(s, weights_);
}
}
@@ -524,7 +524,7 @@
- weights_->begin() - 1;
} else {
size_t n = 0;
- double x = FloatLimits<double>::kPosInfinity;
+ double x = FloatLimits<double>::PosInfinity();
for(aiter->Reset(); !aiter->Done(); aiter->Next(), ++n) {
x = LogPlus(x, aiter->Value().weight);
if (x < w) break;
@@ -537,12 +537,12 @@
private:
double LogPosExp(double x) {
- return x == FloatLimits<double>::kPosInfinity ?
+ return x == FloatLimits<double>::PosInfinity() ?
0.0 : log(1.0F + exp(-x));
}
double LogMinusExp(double x) {
- return x == FloatLimits<double>::kPosInfinity ?
+ return x == FloatLimits<double>::PosInfinity() ?
0.0 : log(1.0F - exp(-x));
}
@@ -557,7 +557,7 @@
double LogPlus(double f1, Weight v) {
double f2 = to_log_weight_(v).Value();
- if (f1 == FloatLimits<double>::kPosInfinity)
+ if (f1 == FloatLimits<double>::PosInfinity())
return f2;
else if (f1 > f2)
return f2 - LogPosExp(f1 - f2);
@@ -572,7 +572,7 @@
error_ = true;
return Weight::NoWeight();
}
- if (f2 == FloatLimits<double>::kPosInfinity)
+ if (f2 == FloatLimits<double>::PosInfinity())
return to_weight_(f1);
else
return to_weight_(f1 - LogMinusExp(f2 - f1));
diff --git a/src/include/fst/arc.h b/src/include/fst/arc.h
index 56086c9..5f4014b 100644
--- a/src/include/fst/arc.h
+++ b/src/include/fst/arc.h
@@ -34,6 +34,7 @@
#include <fst/sparse-power-weight.h>
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/string-weight.h>
diff --git a/src/include/fst/arcsort.h b/src/include/fst/arcsort.h
index 38f4f95..37a51dc 100644
--- a/src/include/fst/arcsort.h
+++ b/src/include/fst/arcsort.h
@@ -118,9 +118,11 @@
// and exclusive of caching.
template <class A, class C>
class ArcSortFst : public StateMapFst<A, A, ArcSortMapper<A, C> > {
+ using StateMapFst<A, A, ArcSortMapper<A, C> >::GetImpl;
public:
typedef A Arc;
- typedef ArcSortMapper<A, C> M;
+ typedef typename Arc::StateId StateId;
+ typedef ArcSortMapper<A, C> M;
ArcSortFst(const Fst<A> &fst, const C &comp)
: StateMapFst<A, A, M>(fst, ArcSortMapper<A, C>(fst, comp)) {}
@@ -136,6 +138,18 @@
virtual ArcSortFst<A, C> *Copy(bool safe = false) const {
return new ArcSortFst(*this, safe);
}
+
+ virtual size_t NumArcs(StateId s) const {
+ return GetImpl()->GetFst().NumArcs(s);
+ }
+
+ virtual size_t NumInputEpsilons(StateId s) const {
+ return GetImpl()->GetFst().NumInputEpsilons(s);
+ }
+
+ virtual size_t NumOutputEpsilons(StateId s) const {
+ return GetImpl()->GetFst().NumOutputEpsilons(s);
+ }
};
diff --git a/src/include/fst/bi-table.h b/src/include/fst/bi-table.h
index dbb436c..bd37781 100644
--- a/src/include/fst/bi-table.h
+++ b/src/include/fst/bi-table.h
@@ -39,8 +39,9 @@
// // Required constructors.
// BiTable();
//
-// // Lookup integer ID from entry. If it doesn't exist, then add it.
-// I FindId(const T &entry);
+// // Lookup integer ID from entry. If it doesn't exist and 'insert'
+// / is true, then add it. Otherwise return -1.
+// I FindId(const T &entry, bool insert = true);
// // Lookup entry from integer ID.
// const T &FindEntry(I) const;
// // # of stored entries.
@@ -58,11 +59,15 @@
T empty_entry;
}
- I FindId(const T &entry) {
+ I FindId(const T &entry, bool insert = true) {
I &id_ref = entry2id_[entry];
- if (id_ref == 0) { // T not found; store and assign it a new ID.
- id2entry_.push_back(entry);
- id_ref = id2entry_.size();
+ if (id_ref == 0) { // T not found
+ if (insert) { // store and assign it a new ID
+ id2entry_.push_back(entry);
+ id_ref = id2entry_.size();
+ } else {
+ return -1;
+ }
}
return id_ref - 1; // NB: id_ref = ID + 1
}
@@ -109,14 +114,18 @@
id2entry_.reserve(table_size);
}
- I FindId(const T &entry) {
+ I FindId(const T &entry, bool insert = true) {
current_entry_ = &entry;
typename KeyHashSet::const_iterator it = keys_.find(kCurrentKey);
- if (it == keys_.end()) {
- I key = id2entry_.size();
- id2entry_.push_back(entry);
- keys_.insert(key);
- return key;
+ if (it == keys_.end()) { // T not found
+ if (insert) { // store and assign it a new ID
+ I key = id2entry_.size();
+ id2entry_.push_back(entry);
+ keys_.insert(key);
+ return key;
+ } else {
+ return -1;
+ }
} else {
return *it;
}
@@ -191,14 +200,18 @@
~VectorBiTable() { delete fp_; }
- I FindId(const T &entry) {
+ I FindId(const T &entry, bool insert = true) {
ssize_t fp = (*fp_)(entry);
if (fp >= fp2id_.size())
fp2id_.resize(fp + 1);
I &id_ref = fp2id_[fp];
- if (id_ref == 0) { // T not found; store and assign it a new ID.
- id2entry_.push_back(entry);
- id_ref = id2entry_.size();
+ if (id_ref == 0) { // T not found
+ if (insert) { // store and assign it a new ID
+ id2entry_.push_back(entry);
+ id_ref = id2entry_.size();
+ } else {
+ return -1;
+ }
}
return id_ref - 1; // NB: id_ref = ID + 1
}
@@ -251,24 +264,32 @@
delete h_;
}
- I FindId(const T &entry) {
+ I FindId(const T &entry, bool insert = true) {
if ((*selector_)(entry)) { // Use the vector if 'selector_(entry) == true'
uint64 fp = (*fp_)(entry);
if (fp2id_.size() <= fp)
fp2id_.resize(fp + 1, 0);
- if (fp2id_[fp] == 0) {
- id2entry_.push_back(entry);
- fp2id_[fp] = id2entry_.size();
+ if (fp2id_[fp] == 0) { // T not found
+ if (insert) { // store and assign it a new ID
+ id2entry_.push_back(entry);
+ fp2id_[fp] = id2entry_.size();
+ } else {
+ return -1;
+ }
}
return fp2id_[fp] - 1; // NB: assoc_value = ID + 1
} else { // Use the hash table otherwise.
current_entry_ = &entry;
typename KeyHashSet::const_iterator it = keys_.find(kCurrentKey);
if (it == keys_.end()) {
- I key = id2entry_.size();
- id2entry_.push_back(entry);
- keys_.insert(key);
- return key;
+ if (insert) {
+ I key = id2entry_.size();
+ id2entry_.push_back(entry);
+ keys_.insert(key);
+ return key;
+ } else {
+ return -1;
+ }
} else {
return *it;
}
@@ -357,11 +378,15 @@
public:
ErasableBiTable() : first_(0) {}
- I FindId(const T &entry) {
+ I FindId(const T &entry, bool insert = true) {
I &id_ref = entry2id_[entry];
- if (id_ref == 0) { // T not found; store and assign it a new ID.
- id2entry_.push_back(entry);
- id_ref = id2entry_.size() + first_;
+ if (id_ref == 0) { // T not found
+ if (insert) { // store and assign it a new ID
+ id2entry_.push_back(entry);
+ id_ref = id2entry_.size() + first_;
+ } else {
+ return -1;
+ }
}
return id_ref - 1; // NB: id_ref = ID + 1
}
diff --git a/src/include/fst/cache.h b/src/include/fst/cache.h
index a6a92d4..0177396 100644
--- a/src/include/fst/cache.h
+++ b/src/include/fst/cache.h
@@ -292,13 +292,13 @@
void DeleteArcs(StateId s, size_t n) {
S *state = ExtendState(s);
- const vector<Arc> &arcs = GetState(s)->arcs;
+ const vector<Arc> &arcs = state->arcs;
for (size_t i = 0; i < n; ++i) {
size_t j = arcs.size() - i - 1;
if (arcs[j].ilabel == 0)
- --GetState(s)->niepsilons;
+ --state->niepsilons;
if (arcs[j].olabel == 0)
- --GetState(s)->noepsilons;
+ --state->noepsilons;
}
state->arcs.resize(arcs.size() - n);
SetProperties(DeleteArcsProperties(Properties()));
@@ -503,9 +503,6 @@
size_t noepsilons; // # of output epsilons
mutable uint32 flags;
mutable int ref_count;
-
- private:
- DISALLOW_COPY_AND_ASSIGN(CacheState);
};
// A CacheBaseImpl with a commonly used CacheState.
diff --git a/src/include/fst/compact-fst.h b/src/include/fst/compact-fst.h
index efa567a..57c927e 100644
--- a/src/include/fst/compact-fst.h
+++ b/src/include/fst/compact-fst.h
@@ -175,10 +175,10 @@
bool Error() const { return error_; }
- private:
// Byte alignment for states and arcs in file format (version 1 only)
static const int kFileAlign = 16;
+ private:
Unsigned *states_;
CompactElement *compacts_;
size_t nstates_;
@@ -539,17 +539,16 @@
}
Weight Final(StateId s) {
- if (!HasFinal(s)) {
- Arc arc(kNoLabel, kNoLabel, Weight::Zero(), kNoStateId);
- if ((compactor_->Size() != -1) ||
- (data_->States(s) != data_->States(s + 1)))
- arc = ComputeArc(s,
- compactor_->Size() == -1
- ? data_->States(s)
- : s * compactor_->Size());
- SetFinal(s, arc.ilabel == kNoLabel ? arc.weight : Weight::Zero());
- }
- return CacheImpl<A>::Final(s);
+ if (HasFinal(s))
+ return CacheImpl<A>::Final(s);
+ Arc arc(kNoLabel, kNoLabel, Weight::Zero(), kNoStateId);
+ if ((compactor_->Size() != -1) ||
+ (data_->States(s) != data_->States(s + 1)))
+ arc = ComputeArc(s,
+ compactor_->Size() == -1
+ ? data_->States(s)
+ : s * compactor_->Size());
+ return arc.ilabel == kNoLabel ? arc.weight : Weight::Zero();
}
StateId NumStates() const {
@@ -650,7 +649,6 @@
// Ensures compatibility
int file_version = opts.align ? kAlignedFileVersion : kFileVersion;
WriteHeader(strm, opts, file_version, &hdr);
-
compactor_->Write(strm);
return data_->Write(strm, opts);
}
@@ -678,9 +676,13 @@
data_->States(s + 1) : (s + 1) * compactor_->Size();
for (size_t i = begin; i < end; ++i) {
const Arc &arc = ComputeArc(s, i);
- if (arc.ilabel == kNoLabel) continue;
- PushArc(s, arc);
+ if (arc.ilabel == kNoLabel)
+ SetFinal(s, arc.weight);
+ else
+ PushArc(s, arc);
}
+ if (!HasFinal(s))
+ SetFinal(s, Weight::Zero());
SetArcs(s);
}
@@ -694,6 +696,9 @@
C *GetCompactor() const { return compactor_; }
CompactFstData<CompactElement, U> *Data() const { return data_; }
+ // Properties always true of this Fst class
+ static const uint64 kStaticProperties = kExpanded;
+
protected:
template <class B, class D>
explicit CompactFstImpl(const CompactFstImpl<B, D, U> &impl)
@@ -710,6 +715,8 @@
}
private:
+ friend class CompactFst<A, C, U>; // allow access during write.
+
void Init(const Fst<Arc> &fst) {
string type = "compact";
if (sizeof(U) != sizeof(uint32)) {
@@ -751,8 +758,6 @@
SetProperties(kError, kError);
}
- // Properties always true of this Fst class
- static const uint64 kStaticProperties = kExpanded;
// Current unaligned file format version
static const int kFileVersion = 2;
// Current aligned file format version
@@ -863,6 +868,10 @@
return Fst<A>::WriteFile(filename);
}
+ template <class F>
+ static bool WriteFst(const F &fst, const C &compactor, ostream &strm,
+ const FstWriteOptions &opts);
+
virtual void InitStateIterator(StateIteratorData<A> *data) const {
GetImpl()->InitStateIterator(data);
}
@@ -893,6 +902,115 @@
void operator=(const CompactFst<A, C, U> &fst); // disallow
};
+// Writes Fst in Compact format, potentially with a pass over the machine
+// before writing to compute the number of states and arcs.
+//
+template <class A, class C, class U>
+template <class F>
+bool CompactFst<A, C, U>::WriteFst(const F &fst,
+ const C &compactor,
+ ostream &strm,
+ const FstWriteOptions &opts) {
+ typedef U Unsigned;
+ typedef typename C::Element CompactElement;
+ typedef typename A::Weight Weight;
+ static const int kFileAlign =
+ CompactFstData<CompactElement, U>::kFileAlign;
+ int file_version = opts.align ?
+ CompactFstImpl<A, C, U>::kAlignedFileVersion :
+ CompactFstImpl<A, C, U>::kFileVersion;
+ size_t num_arcs = -1, num_states = -1, num_compacts = -1;
+ C first_pass_compactor = compactor;
+ if (fst.Type() == CompactFst<A, C, U>().Type()) {
+ const CompactFst<A, C, U> *compact_fst =
+ reinterpret_cast<const CompactFst<A, C, U> *>(&fst);
+ num_arcs = compact_fst->GetImpl()->Data()->NumArcs();
+ num_states = compact_fst->GetImpl()->Data()->NumStates();
+ num_compacts = compact_fst->GetImpl()->Data()->NumCompacts();
+ first_pass_compactor = *compact_fst->GetImpl()->GetCompactor();
+ } else {
+ // A first pass is needed to compute the state of the compactor, which
+ // is saved ahead of the rest of the data structures. This unfortunately
+ // means forcing a complete double compaction when writing in this format.
+ // TODO(allauzen): eliminate mutable state from compactors.
+ num_arcs = 0;
+ num_states = 0;
+ for (StateIterator<F> siter(fst); !siter.Done(); siter.Next()) {
+ const StateId s = siter.Value();
+ ++num_states;
+ if (fst.Final(s) != Weight::Zero()) {
+ first_pass_compactor.Compact(
+ s, A(kNoLabel, kNoLabel, fst.Final(s), kNoStateId));
+ }
+ for (ArcIterator<F> aiter(fst, s); !aiter.Done(); aiter.Next()) {
+ ++num_arcs;
+ first_pass_compactor.Compact(s, aiter.Value());
+ }
+ }
+ }
+ FstHeader hdr;
+ hdr.SetStart(fst.Start());
+ hdr.SetNumStates(num_states);
+ hdr.SetNumArcs(num_arcs);
+ string type = "compact";
+ if (sizeof(U) != sizeof(uint32)) {
+ string size;
+ Int64ToStr(8 * sizeof(U), &size);
+ type += size;
+ }
+ type += "_";
+ type += C::Type();
+ uint64 copy_properties = fst.Properties(kCopyProperties, true);
+ if ((copy_properties & kError) || !compactor.Compatible(fst)) {
+ LOG(ERROR) << "fst incompatible with compactor";
+ return false;
+ }
+ uint64 properties = copy_properties |
+ CompactFstImpl<A, C, U>::kStaticProperties;
+ FstImpl<A>::WriteFstHeader(fst, strm, opts, file_version, type, properties,
+ &hdr);
+ first_pass_compactor.Write(strm);
+ if (first_pass_compactor.Size() == -1) {
+ if (opts.align && !AlignOutput(strm, kFileAlign)) {
+ LOG(ERROR) << "CompactFst::Write: Alignment failed: " << opts.source;
+ return false;
+ }
+ Unsigned compacts = 0;
+ for (StateIterator<F> siter(fst); !siter.Done(); siter.Next()) {
+ const StateId s = siter.Value();
+ strm.write(reinterpret_cast<const char *>(&compacts), sizeof(compacts));
+ if (fst.Final(s) != Weight::Zero()) {
+ ++compacts;
+ }
+ compacts += fst.NumArcs(s);
+ }
+ strm.write(reinterpret_cast<const char *>(&compacts), sizeof(compacts));
+ }
+ if (opts.align && !AlignOutput(strm, kFileAlign)) {
+ LOG(ERROR) << "Could not align file during write after writing states";
+ }
+ C second_pass_compactor = compactor;
+ CompactElement element;
+ for (StateIterator<F> siter(fst); !siter.Done(); siter.Next()) {
+ const StateId s = siter.Value();
+ if (fst.Final(s) != Weight::Zero()) {
+ element = second_pass_compactor.Compact(
+ s, A(kNoLabel, kNoLabel, fst.Final(s), kNoStateId));
+ strm.write(reinterpret_cast<const char *>(&element), sizeof(element));
+ }
+ for (ArcIterator<F> aiter(fst, s); !aiter.Done(); aiter.Next()) {
+ element = second_pass_compactor.Compact(s, aiter.Value());
+ strm.write(reinterpret_cast<const char *>(&element), sizeof(element));
+ }
+ }
+ strm.flush();
+ if (!strm) {
+ LOG(ERROR) << "CompactFst write failed: " << opts.source;
+ return false;
+ }
+ return true;
+}
+
// Specialization for CompactFst; see generic version in fst.h
// for sample usage (but use the CompactFst type!). This version
diff --git a/src/include/fst/compat.h b/src/include/fst/compat.h
index 034b57e..3b5275d 100644
--- a/src/include/fst/compat.h
+++ b/src/include/fst/compat.h
@@ -39,10 +39,7 @@
#include <fst/lock.h>
#include <fst/flags.h>
#include <fst/log.h>
-
-#ifdef HAVE_ICU
#include <fst/icu.h>
-#endif
using std::cin;
using std::cout;
@@ -87,7 +84,7 @@
void Reset() {
count_ = 0;
for (int i = 0; i < kCheckSumLength; ++i)
- check_sum_[0] = '\0';
+ check_sum_[i] = '\0';
}
void Update(void const *data, int size) {
@@ -113,24 +110,6 @@
DISALLOW_COPY_AND_ASSIGN(CheckSummer);
};
-// Define the UTF8 string conversion function to throw an error
-// when the ICU Library is missing or disabled.
-#ifndef HAVE_ICU
-
-template <class Label>
-bool UTF8StringToLabels(const string&, std::vector<Label>*) {
- LOG(ERROR) << "UTF8StringToLabels: ICU Library required for UTF8 handling";
- return false;
-}
-
-template <class Label>
-bool LabelsToUTF8String(const std::vector<Label>&, string*) {
- LOG(ERROR) << "LabelsToUTF8String: ICU Library required for UTF8 handling";
- return false;
-}
-
-#endif // HAVE_ICU
-
} // namespace fst
diff --git a/src/include/fst/compose.h b/src/include/fst/compose.h
index c0bf4b1..dfdff0a 100644
--- a/src/include/fst/compose.h
+++ b/src/include/fst/compose.h
@@ -360,6 +360,10 @@
return Times(final1, final2);
}
+ // Identifies and verifies the capabilities of the matcher to be used for
+ // composition.
+ void SetMatchType();
+
F *filter_;
Matcher1 *matcher1_;
Matcher2 *matcher2_;
@@ -385,23 +389,15 @@
fst2_(matcher2_->GetFst()),
state_table_(opts.state_table ? opts.state_table :
new T(fst1_, fst2_)) {
- MatchType type1 = matcher1_->Type(false);
- MatchType type2 = matcher2_->Type(false);
- if (type1 == MATCH_OUTPUT && type2 == MATCH_INPUT) {
- match_type_ = MATCH_BOTH;
- } else if (type1 == MATCH_OUTPUT) {
- match_type_ = MATCH_OUTPUT;
- } else if (type2 == MATCH_INPUT) {
- match_type_ = MATCH_INPUT;
- } else if (matcher1_->Type(true) == MATCH_OUTPUT) {
- match_type_ = MATCH_OUTPUT;
- } else if (matcher2_->Type(true) == MATCH_INPUT) {
- match_type_ = MATCH_INPUT;
- } else {
- FSTERROR() << "ComposeFst: 1st argument cannot match on output labels "
- << "and 2nd argument cannot match on input labels (sort?).";
+ SetMatchType();
+ if (match_type_ == MATCH_NONE)
SetProperties(kError, kError);
- }
+ VLOG(2) << "ComposeFst(" << this << "): Match type: "
+ << (match_type_ == MATCH_OUTPUT ? "output" :
+ (match_type_ == MATCH_INPUT ? "input" :
+ (match_type_ == MATCH_BOTH ? "both" :
+ (match_type_ == MATCH_NONE ? "none" : "unknown"))));
+
uint64 fprops1 = fst1.Properties(kFstProperties, false);
uint64 fprops2 = fst2.Properties(kFstProperties, false);
uint64 mprops1 = matcher1_->Properties(fprops1);
@@ -412,6 +408,55 @@
VLOG(2) << "ComposeFst(" << this << "): Initialized";
}
+template <class M1, class M2, class F, class T>
+void ComposeFstImpl<M1, M2, F, T>::SetMatchType() {
+ MatchType type1 = matcher1_->Type(false);
+ MatchType type2 = matcher2_->Type(false);
+ uint32 flags1 = matcher1_->Flags();
+ uint32 flags2 = matcher2_->Flags();
+ if (flags1 & flags2 & kRequireMatch) {
+ FSTERROR() << "ComposeFst: only one argument can require matching.";
+ match_type_ = MATCH_NONE;
+ } else if (flags1 & kRequireMatch) {
+ if (matcher1_->Type(true) != MATCH_OUTPUT) {
+ FSTERROR() << "ComposeFst: 1st argument requires matching but cannot.";
+ match_type_ = MATCH_NONE;
+ }
+ match_type_ = MATCH_OUTPUT;
+ } else if (flags2 & kRequireMatch) {
+ if (matcher2_->Type(true) != MATCH_INPUT) {
+ FSTERROR() << "ComposeFst: 2nd argument requires matching but cannot.";
+ match_type_ = MATCH_NONE;
+ }
+ match_type_ = MATCH_INPUT;
+ } else if (flags1 & flags2 & kPreferMatch &&
+ type1 == MATCH_OUTPUT && type2 == MATCH_INPUT) {
+ match_type_ = MATCH_BOTH;
+ } else if (flags1 & kPreferMatch && type1 == MATCH_OUTPUT) {
+ match_type_ = MATCH_OUTPUT;
+ } else if (flags2 & kPreferMatch && type2 == MATCH_INPUT) {
+ match_type_ = MATCH_INPUT;
+ } else if (type1 == MATCH_OUTPUT && type2 == MATCH_INPUT) {
+ match_type_ = MATCH_BOTH;
+ } else if (type1 == MATCH_OUTPUT) {
+ match_type_ = MATCH_OUTPUT;
+ } else if (type2 == MATCH_INPUT) {
+ match_type_ = MATCH_INPUT;
+ } else if (flags1 & kPreferMatch && matcher1_->Type(true) == MATCH_OUTPUT) {
+ match_type_ = MATCH_OUTPUT;
+ } else if (flags2 & kPreferMatch && matcher2_->Type(true) == MATCH_INPUT) {
+ match_type_ = MATCH_INPUT;
+ } else if (matcher1_->Type(true) == MATCH_OUTPUT) {
+ match_type_ = MATCH_OUTPUT;
+ } else if (matcher2_->Type(true) == MATCH_INPUT) {
+ match_type_ = MATCH_INPUT;
+ } else {
+ FSTERROR() << "ComposeFst: 1st argument cannot match on output labels "
+ << "and 2nd argument cannot match on input labels (sort?).";
+ match_type_ = MATCH_NONE;
+ }
+}
+
// Computes the composition of two transducers. This version is a
// delayed Fst. If FST1 transduces string x to y with weight a and FST2
@@ -539,16 +584,19 @@
switch (LookAheadMatchType(fst1, fst2)) { // Check for lookahead matchers
default:
case MATCH_NONE: { // Default composition (no look-ahead)
+ VLOG(2) << "ComposeFst: Default composition (no look-ahead)";
ComposeFstOptions<Arc> nopts(opts);
return CreateBase1(fst1, fst2, nopts);
}
case MATCH_OUTPUT: { // Lookahead on fst1
+ VLOG(2) << "ComposeFst: Lookahead on fst1";
typedef typename DefaultLookAhead<Arc, MATCH_OUTPUT>::FstMatcher M;
typedef typename DefaultLookAhead<Arc, MATCH_OUTPUT>::ComposeFilter F;
ComposeFstOptions<Arc, M, F> nopts(opts);
return CreateBase1(fst1, fst2, nopts);
}
case MATCH_INPUT: { // Lookahead on fst2
+ VLOG(2) << "ComposeFst: Lookahead on fst2";
typedef typename DefaultLookAhead<Arc, MATCH_INPUT>::FstMatcher M;
typedef typename DefaultLookAhead<Arc, MATCH_INPUT>::ComposeFilter F;
ComposeFstOptions<Arc, M, F> nopts(opts);
diff --git a/src/include/fst/config.h b/src/include/fst/config.h
index 046b49c..47e472e 100644
--- a/src/include/fst/config.h
+++ b/src/include/fst/config.h
@@ -4,7 +4,7 @@
/* Define to 1 if you have the ICU library. */
/* #undef HAVE_ICU */
-/* Define to 1 if the system has the type `std::tr1::hash<long long
+/* Define to 1 if the system has the type `std::hash<long long
unsigned>'. */
#define HAVE_STD__TR1__HASH_LONG_LONG_UNSIGNED_ 1
diff --git a/src/include/fst/const-fst.h b/src/include/fst/const-fst.h
index f68e8ed..80efc8d 100644
--- a/src/include/fst/const-fst.h
+++ b/src/include/fst/const-fst.h
@@ -87,8 +87,6 @@
static ConstFstImpl<A, U> *Read(istream &strm, const FstReadOptions &opts);
- bool Write(ostream &strm, const FstWriteOptions &opts) const;
-
A *Arcs(StateId s) { return arcs_ + states_[s].pos; }
// Provide information needed for generic state iterator
@@ -330,23 +328,24 @@
template <class F>
bool ConstFst<A, U>::WriteFst(const F &fst, ostream &strm,
const FstWriteOptions &opts) {
- static const int kFileVersion = 2;
- static const int kAlignedFileVersion = 1;
- static const int kFileAlign = 16;
- int file_version = opts.align ? kAlignedFileVersion : kFileVersion;
+ int file_version = opts.align ? ConstFstImpl<A, U>::kAlignedFileVersion :
+ ConstFstImpl<A, U>::kFileVersion;
size_t num_arcs = -1, num_states = -1;
size_t start_offset = 0;
bool update_header = true;
if (fst.Type() == ConstFst<A, U>().Type()) {
- const ConstFst<A, U> *const_fst = static_cast<const ConstFst<A, U> *>(&fst);
+ const ConstFst<A, U> *const_fst =
+ reinterpret_cast<const ConstFst<A, U> *>(&fst);
num_arcs = const_fst->GetImpl()->narcs_;
num_states = const_fst->GetImpl()->nstates_;
update_header = false;
} else if ((start_offset = strm.tellp()) == -1) {
// precompute values needed for header when we cannot seek to rewrite it.
+ num_arcs = 0;
+ num_states = 0;
for (StateIterator<F> siter(fst); !siter.Done(); siter.Next()) {
num_arcs += fst.NumArcs(siter.Value());
- num_states++;
+ ++num_states;
}
update_header = false;
}
@@ -360,8 +359,11 @@
Int64ToStr(8 * sizeof(U), &size);
type += size;
}
- FstImpl<A>::WriteFstHeader(fst, strm, opts, file_version, type, &hdr);
- if (opts.align && !AlignOutput(strm, kFileAlign)) {
+ uint64 properties = fst.Properties(kCopyProperties, true) |
+ ConstFstImpl<A, U>::kStaticProperties;
+ FstImpl<A>::WriteFstHeader(fst, strm, opts, file_version, type, properties,
+ &hdr);
+ if (opts.align && !AlignOutput(strm, ConstFstImpl<A, U>::kFileAlign)) {
LOG(ERROR) << "Could not align file during write after header";
return false;
}
@@ -375,11 +377,11 @@
state.noepsilons = fst.NumOutputEpsilons(siter.Value());
strm.write(reinterpret_cast<const char *>(&state), sizeof(state));
pos += state.narcs;
- states++;
+ ++states;
}
hdr.SetNumStates(states);
hdr.SetNumArcs(pos);
- if (opts.align && !AlignOutput(strm, kFileAlign)) {
+ if (opts.align && !AlignOutput(strm, ConstFstImpl<A, U>::kFileAlign)) {
LOG(ERROR) << "Could not align file during write after writing states";
}
for (StateIterator<F> siter(fst); !siter.Done(); siter.Next()) {
@@ -391,12 +393,12 @@
}
strm.flush();
if (!strm) {
- LOG(ERROR) << "WriteAsVectorFst write failed: " << opts.source;
+ LOG(ERROR) << "ConstFst Write write failed: " << opts.source;
return false;
}
if (update_header) {
return FstImpl<A>::UpdateFstHeader(fst, strm, opts, file_version, type,
- &hdr, start_offset);
+ properties, &hdr, start_offset);
} else {
if (hdr.NumStates() != num_states) {
LOG(ERROR) << "Inconsistent number of states observed during write";
diff --git a/src/include/fst/dfs-visit.h b/src/include/fst/dfs-visit.h
index b47c78d..4d93a39 100644
--- a/src/include/fst/dfs-visit.h
+++ b/src/include/fst/dfs-visit.h
@@ -177,7 +177,8 @@
// Find next tree root
for (root = root == start ? 0 : root + 1;
root < nstates && state_color[root] != kDfsWhite;
- ++root);
+ ++root) {
+ }
// Check for a state beyond the largest known state
if (!expanded && root == nstates) {
diff --git a/src/include/fst/expanded-fst.h b/src/include/fst/expanded-fst.h
index b44b81c..676ceb3 100644
--- a/src/include/fst/expanded-fst.h
+++ b/src/include/fst/expanded-fst.h
@@ -82,7 +82,7 @@
}
return Read(strm, FstReadOptions(filename));
} else {
- return Read(std::cin, FstReadOptions("standard input"));
+ return Read(cin, FstReadOptions("standard input"));
}
}
};
@@ -154,7 +154,7 @@
}
return I::Read(strm, FstReadOptions(filename));
} else {
- return I::Read(std::cin, FstReadOptions("standard input"));
+ return I::Read(cin, FstReadOptions("standard input"));
}
}
diff --git a/src/include/fst/extensions/far/compile-strings.h b/src/include/fst/extensions/far/compile-strings.h
index d7f4d6b..ca247db 100644
--- a/src/include/fst/extensions/far/compile-strings.h
+++ b/src/include/fst/extensions/far/compile-strings.h
@@ -56,7 +56,7 @@
const SymbolTable *syms = 0,
Label unknown_label = kNoStateId)
: nline_(0), strm_(istrm), source_(source), entry_type_(entry_type),
- token_type_(token_type), done_(false),
+ token_type_(token_type), symbols_(syms), done_(false),
compiler_(token_type, syms, unknown_label, allow_negative_labels) {
Next(); // Initialize the reader to the first input.
}
@@ -87,8 +87,12 @@
done_ = true; // whitespace at the end of a file.
}
- VectorFst<A> *GetVectorFst() {
+ VectorFst<A> *GetVectorFst(bool keep_symbols = false) {
VectorFst<A> *fst = new VectorFst<A>;
+ if (keep_symbols) {
+ fst->SetInputSymbols(symbols_);
+ fst->SetOutputSymbols(symbols_);
+ }
if (compiler_(content_, fst)) {
return fst;
} else {
@@ -97,9 +101,16 @@
}
}
- CompactFst<A, StringCompactor<A> > *GetCompactFst() {
- CompactFst<A, StringCompactor<A> > *fst =
- new CompactFst<A, StringCompactor<A> >;
+ CompactFst<A, StringCompactor<A> > *GetCompactFst(bool keep_symbols = false) {
+ CompactFst<A, StringCompactor<A> > *fst;
+ if (keep_symbols) {
+ VectorFst<A> tmp;
+ tmp.SetInputSymbols(symbols_);
+ tmp.SetOutputSymbols(symbols_);
+ fst = new CompactFst<A, StringCompactor<A> >(tmp);
+ } else {
+ fst = new CompactFst<A, StringCompactor<A> >;
+ }
if (compiler_(content_, fst)) {
return fst;
} else {
@@ -114,6 +125,7 @@
string source_;
EntryType entry_type_;
TokenType token_type_;
+ const SymbolTable *symbols_;
bool done_;
StringCompiler<A> compiler_;
string content_; // The actual content of the input stream's next FST.
@@ -135,6 +147,8 @@
FarTokenType tt,
const string &symbols_fname,
const string &unknown_symbol,
+ bool keep_symbols,
+ bool initial_symbols,
bool allow_negative_labels,
bool file_list_input,
const string &key_prefix,
@@ -175,8 +189,9 @@
const SymbolTable *syms = 0;
typename Arc::Label unknown_label = kNoLabel;
if (!symbols_fname.empty()) {
- syms = SymbolTable::ReadText(symbols_fname,
- allow_negative_labels);
+ SymbolTableTextOptions opts;
+ opts.allow_negative = allow_negative_labels;
+ syms = SymbolTable::ReadText(symbols_fname, opts);
if (!syms) {
FSTERROR() << "FarCompileStrings: error reading symbol table: "
<< symbols_fname;
@@ -199,32 +214,47 @@
vector<string> inputs;
if (file_list_input) {
for (int i = 1; i < in_fnames.size(); ++i) {
- ifstream istrm(in_fnames[i].c_str());
+ istream *istrm = in_fnames.empty() ? &cin :
+ new ifstream(in_fnames[i].c_str());
string str;
- while (getline(istrm, str))
+ while (getline(*istrm, str))
inputs.push_back(str);
+ if (!in_fnames.empty())
+ delete istrm;
}
} else {
inputs = in_fnames;
}
for (int i = 0, n = 0; i < inputs.size(); ++i) {
+ if (generate_keys == 0 && inputs[i].empty()) {
+ FSTERROR() << "FarCompileStrings: read from a file instead of stdin or"
+ << " set the --generate_keys flags.";
+ delete far_writer;
+ delete syms;
+ return;
+ }
int key_size = generate_keys ? generate_keys :
(entry_type == StringReader<Arc>::FILE ? 1 :
KeySize(inputs[i].c_str()));
- ifstream istrm(inputs[i].c_str());
+ istream *istrm = inputs[i].empty() ? &cin :
+ new ifstream(inputs[i].c_str());
+ bool keep_syms = keep_symbols;
for (StringReader<Arc> reader(
- istrm, inputs[i], entry_type, token_type,
- allow_negative_labels, syms, unknown_label);
+ *istrm, inputs[i].empty() ? "stdin" : inputs[i],
+ entry_type, token_type, allow_negative_labels,
+ syms, unknown_label);
!reader.Done();
reader.Next()) {
++n;
const Fst<Arc> *fst;
if (compact)
- fst = reader.GetCompactFst();
+ fst = reader.GetCompactFst(keep_syms);
else
- fst = reader.GetVectorFst();
+ fst = reader.GetVectorFst(keep_syms);
+ if (initial_symbols)
+ keep_syms = false;
if (!fst) {
FSTERROR() << "FarCompileStrings: compiling string number " << n
<< " in file " << inputs[i] << " failed with token_type = "
@@ -236,6 +266,7 @@
(fet == FET_FILE ? "file" : "unknown"));
delete far_writer;
delete syms;
+ if (!inputs[i].empty()) delete istrm;
return;
}
ostringstream keybuf;
@@ -260,6 +291,8 @@
}
if (generate_keys == 0)
n = 0;
+ if (!inputs[i].empty())
+ delete istrm;
}
delete far_writer;
diff --git a/src/include/fst/extensions/far/equal.h b/src/include/fst/extensions/far/equal.h
new file mode 100644
index 0000000..be82e2d
--- /dev/null
+++ b/src/include/fst/extensions/far/equal.h
@@ -0,0 +1,99 @@
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: allauzen@google.com (Cyril Allauzen)
+
+#ifndef FST_EXTENSIONS_FAR_EQUAL_H_
+#define FST_EXTENSIONS_FAR_EQUAL_H_
+
+#include <string>
+
+#include <fst/extensions/far/far.h>
+#include <fst/equal.h>
+
+namespace fst {
+
+template <class Arc>
+bool FarEqual(const string &filename1,
+ const string &filename2,
+ float delta = kDelta,
+ const string &begin_key = string(),
+ const string &end_key = string()) {
+
+ FarReader<Arc> *reader1 = FarReader<Arc>::Open(filename1);
+ FarReader<Arc> *reader2 = FarReader<Arc>::Open(filename2);
+ if (!reader1 || !reader2) {
+ delete reader1;
+ delete reader2;
+ VLOG(1) << "FarEqual: cannot open input Far file(s)";
+ return false;
+ }
+
+ if (!begin_key.empty()) {
+ bool find_begin1 = reader1->Find(begin_key);
+ bool find_begin2 = reader2->Find(begin_key);
+ if (!find_begin1 || !find_begin2) {
+ bool ret = !find_begin1 && !find_begin2;
+ if (!ret) {
+ VLOG(1) << "FarEqual: key \"" << begin_key << "\" missing from "
+ << (find_begin1 ? "second" : "first") << " archive.";
+ }
+ delete reader1;
+ delete reader2;
+ return ret;
+ }
+ }
+
+ for(; !reader1->Done() && !reader2->Done();
+ reader1->Next(), reader2->Next()) {
+ const string key1 = reader1->GetKey();
+ const string key2 = reader2->GetKey();
+ if (!end_key.empty() && end_key < key1 && end_key < key2) {
+ delete reader1;
+ delete reader2;
+ return true;
+ }
+ if (key1 != key2) {
+ VLOG(1) << "FarEqual: mismatched keys \""
+ << key1 << "\" <> \"" << key2 << "\".";
+ delete reader1;
+ delete reader2;
+ return false;
+ }
+ if (!Equal(reader1->GetFst(), reader2->GetFst(), delta)) {
+ VLOG(1) << "FarEqual: Fsts for key \"" << key1 << "\" are not equal.";
+ delete reader1;
+ delete reader2;
+ return false;
+ }
+ }
+
+ if (!reader1->Done() || !reader2->Done()) {
+ VLOG(1) << "FarEqual: key \""
+ << (reader1->Done() ? reader2->GetKey() : reader1->GetKey())
+ << "\" missing form " << (reader2->Done() ? "first" : "second")
+ << " archive.";
+ delete reader1;
+ delete reader2;
+ return false;
+ }
+
+ delete reader1;
+ delete reader2;
+ return true;
+}
+
+} // namespace fst
+
+#endif // FST_EXTENSIONS_FAR_EQUAL_H_
diff --git a/src/include/fst/extensions/far/extract.h b/src/include/fst/extensions/far/extract.h
index 022ca60..d6f92ff 100644
--- a/src/include/fst/extensions/far/extract.h
+++ b/src/include/fst/extensions/far/extract.h
@@ -70,7 +70,7 @@
if (nrep > 0) {
ostringstream tmp;
tmp << '.' << nrep;
- key += tmp.str();
+ key.append(tmp.str().data(), tmp.str().size());
}
ofilename = key;
}
diff --git a/src/include/fst/extensions/far/far.h b/src/include/fst/extensions/far/far.h
index 82b9e5c..acce76e 100644
--- a/src/include/fst/extensions/far/far.h
+++ b/src/include/fst/extensions/far/far.h
@@ -32,6 +32,13 @@
enum FarEntryType { FET_LINE, FET_FILE };
enum FarTokenType { FTT_SYMBOL, FTT_BYTE, FTT_UTF8 };
+inline bool IsFst(const string &filename) {
+ ifstream strm(filename.c_str());
+ if (!strm)
+ return false;
+ return IsFstHeader(strm, filename);
+}
+
// FST archive header class
class FarHeader {
public:
@@ -40,8 +47,11 @@
bool Read(const string &filename) {
FstHeader fsthdr;
- if (filename.empty()) { // Header reading unsupported on stdin.
- return false;
+ if (filename.empty()) {
+ // Header reading unsupported on stdin. Assumes STList and StdArc.
+ fartype_ = "stlist";
+ arctype_ = "standard";
+ return true;
} else if (IsSTTable(filename)) { // Check if STTable
ReadSTTableHeader(filename, &fsthdr);
fartype_ = "sttable";
@@ -52,6 +62,12 @@
fartype_ = "sttable";
arctype_ = fsthdr.ArcType().empty() ? "unknown" : fsthdr.ArcType();
return true;
+ } else if (IsFst(filename)) { // Check if Fst
+ ifstream istrm(filename.c_str());
+ fsthdr.Read(istrm, filename);
+ fartype_ = "fst";
+ arctype_ = fsthdr.ArcType().empty() ? "unknown" : fsthdr.ArcType();
+ return true;
}
return false;
}
@@ -61,8 +77,12 @@
string arctype_;
};
-enum FarType { FAR_DEFAULT = 0, FAR_STTABLE = 1, FAR_STLIST = 2,
- FAR_SSTABLE = 3 };
+enum FarType {
+ FAR_DEFAULT = 0,
+ FAR_STTABLE = 1,
+ FAR_STLIST = 2,
+ FAR_FST = 3,
+};
// This class creates an archive of FSTs.
template <class A>
@@ -153,7 +173,7 @@
public:
typedef A Arc;
- static STTableFarWriter *Create(const string filename) {
+ static STTableFarWriter *Create(const string &filename) {
STTableWriter<Fst<A>, FstWriter<A> > *writer =
STTableWriter<Fst<A>, FstWriter<A> >::Create(filename);
return new STTableFarWriter(writer);
@@ -183,7 +203,7 @@
public:
typedef A Arc;
- static STListFarWriter *Create(const string filename) {
+ static STListFarWriter *Create(const string &filename) {
STListWriter<Fst<A>, FstWriter<A> > *writer =
STListWriter<Fst<A>, FstWriter<A> >::Create(filename);
return new STListFarWriter(writer);
@@ -209,6 +229,43 @@
template <class A>
+class FstFarWriter : public FarWriter<A> {
+ public:
+ typedef A Arc;
+
+ explicit FstFarWriter(const string &filename)
+ : filename_(filename), error_(false), written_(false) {}
+
+ static FstFarWriter *Create(const string &filename) {
+ return new FstFarWriter(filename);
+ }
+
+ void Add(const string &key, const Fst<A> &fst) {
+ if (written_) {
+ LOG(WARNING) << "FstFarWriter::Add: only one Fst supported,"
+ << " subsequent entries discarded.";
+ } else {
+ error_ = !fst.Write(filename_);
+ written_ = true;
+ }
+ }
+
+ FarType Type() const { return FAR_FST; }
+
+ bool Error() const { return error_; }
+
+ ~FstFarWriter() {}
+
+ private:
+ string filename_;
+ bool error_;
+ bool written_;
+
+ DISALLOW_COPY_AND_ASSIGN(FstFarWriter);
+};
+
+
+template <class A>
FarWriter<A> *FarWriter<A>::Create(const string &filename, FarType type) {
switch(type) {
case FAR_DEFAULT:
@@ -220,6 +277,9 @@
case FAR_STLIST:
return STListFarWriter<A>::Create(filename);
break;
+ case FAR_FST:
+ return FstFarWriter<A>::Create(filename);
+ break;
default:
LOG(ERROR) << "FarWriter::Create: unknown far type";
return 0;
@@ -331,6 +391,114 @@
DISALLOW_COPY_AND_ASSIGN(STListFarReader);
};
+template <class A>
+class FstFarReader : public FarReader<A> {
+ public:
+ typedef A Arc;
+
+ static FstFarReader *Open(const string &filename) {
+ vector<string> filenames;
+ filenames.push_back(filename);
+ return new FstFarReader<A>(filenames);
+ }
+
+ static FstFarReader *Open(const vector<string> &filenames) {
+ return new FstFarReader<A>(filenames);
+ }
+
+ FstFarReader(const vector<string> &filenames)
+ : keys_(filenames), has_stdin_(false), pos_(0), fst_(0), error_(false) {
+ sort(keys_.begin(), keys_.end());
+ streams_.resize(keys_.size(), 0);
+ for (size_t i = 0; i < keys_.size(); ++i) {
+ if (keys_[i].empty()) {
+ if (!has_stdin_) {
+ streams_[i] = &cin;
+ //sources_[i] = "stdin";
+ has_stdin_ = true;
+ } else {
+ FSTERROR() << "FstFarReader::FstFarReader: stdin should only "
+ << "appear once in the input file list.";
+ error_ = true;
+ return;
+ }
+ } else {
+ streams_[i] = new ifstream(
+ keys_[i].c_str(), ifstream::in | ifstream::binary);
+ }
+ }
+ if (pos_ >= keys_.size()) return;
+ ReadFst();
+ }
+
+ void Reset() {
+ if (has_stdin_) {
+ FSTERROR() << "FstFarReader::Reset: operation not supported on stdin";
+ error_ = true;
+ return;
+ }
+ pos_ = 0;
+ ReadFst();
+ }
+
+ bool Find(const string &key) {
+ if (has_stdin_) {
+ FSTERROR() << "FstFarReader::Find: operation not supported on stdin";
+ error_ = true;
+ return false;
+ }
+ pos_ = 0;//TODO
+ ReadFst();
+ return true;
+ }
+
+ bool Done() const { return error_ || pos_ >= keys_.size(); }
+
+ void Next() {
+ ++pos_;
+ ReadFst();
+ }
+
+ const string &GetKey() const {
+ return keys_[pos_];
+ }
+
+ const Fst<A> &GetFst() const {
+ return *fst_;
+ }
+
+ FarType Type() const { return FAR_FST; }
+
+ bool Error() const { return error_; }
+
+ ~FstFarReader() {
+ if (fst_) delete fst_;
+ for (size_t i = 0; i < keys_.size(); ++i)
+ delete streams_[i];
+ }
+
+ private:
+ void ReadFst() {
+ if (fst_) delete fst_;
+ if (pos_ >= keys_.size()) return;
+ streams_[pos_]->seekg(0);
+ fst_ = Fst<A>::Read(*streams_[pos_], FstReadOptions());
+ if (!fst_) {
+ FSTERROR() << "FstFarReader: error reading Fst from: " << keys_[pos_];
+ error_ = true;
+ }
+ }
+
+ private:
+ vector<string> keys_;
+ vector<istream*> streams_;
+ bool has_stdin_;
+ size_t pos_;
+ mutable Fst<A> *fst_;
+ mutable bool error_;
+
+ DISALLOW_COPY_AND_ASSIGN(FstFarReader);
+};
template <class A>
FarReader<A> *FarReader<A>::Open(const string &filename) {
@@ -340,6 +508,8 @@
return STTableFarReader<A>::Open(filename);
else if (IsSTList(filename))
return STListFarReader<A>::Open(filename);
+ else if (IsFst(filename))
+ return FstFarReader<A>::Open(filename);
return 0;
}
@@ -352,6 +522,8 @@
return STTableFarReader<A>::Open(filenames);
else if (!filenames.empty() && IsSTList(filenames[0]))
return STListFarReader<A>::Open(filenames);
+ else if (!filenames.empty() && IsFst(filenames[0]))
+ return FstFarReader<A>::Open(filenames);
return 0;
}
diff --git a/src/include/fst/extensions/far/farscript.h b/src/include/fst/extensions/far/farscript.h
index 9c3b1ca..3a9c145 100644
--- a/src/include/fst/extensions/far/farscript.h
+++ b/src/include/fst/extensions/far/farscript.h
@@ -27,6 +27,7 @@
#include <fst/script/arg-packs.h>
#include <fst/extensions/far/compile-strings.h>
#include <fst/extensions/far/create.h>
+#include <fst/extensions/far/equal.h>
#include <fst/extensions/far/extract.h>
#include <fst/extensions/far/info.h>
#include <fst/extensions/far/print-strings.h>
@@ -51,6 +52,8 @@
const FarTokenType tt;
const string &symbols_fname;
const string &unknown_symbol;
+ const bool keep_symbols;
+ const bool initial_symbols;
const bool allow_negative_labels;
const bool file_list_input;
const string &key_prefix;
@@ -65,6 +68,8 @@
FarTokenType tt,
const string &symbols_fname,
const string &unknown_symbol,
+ bool keep_symbols,
+ bool initial_symbols,
bool allow_negative_labels,
bool file_list_input,
const string &key_prefix,
@@ -72,6 +77,7 @@
in_fnames(in_fnames), out_fname(out_fname), fst_type(fst_type),
far_type(far_type), generate_keys(generate_keys), fet(fet),
tt(tt), symbols_fname(symbols_fname), unknown_symbol(unknown_symbol),
+ keep_symbols(keep_symbols), initial_symbols(initial_symbols),
allow_negative_labels(allow_negative_labels),
file_list_input(file_list_input), key_prefix(key_prefix),
key_suffix(key_suffix) { }
@@ -82,7 +88,8 @@
fst::FarCompileStrings<Arc>(
args->in_fnames, args->out_fname, args->fst_type, args->far_type,
args->generate_keys, args->fet, args->tt, args->symbols_fname,
- args->unknown_symbol, args->allow_negative_labels, args->file_list_input,
+ args->unknown_symbol, args->keep_symbols, args->initial_symbols,
+ args->allow_negative_labels, args->file_list_input,
args->key_prefix, args->key_suffix);
}
@@ -97,6 +104,8 @@
FarTokenType tt,
const string &symbols_fname,
const string &unknown_symbol,
+ bool keep_symbols,
+ bool initial_symbols,
bool allow_negative_labels,
bool file_list_input,
const string &key_prefix,
@@ -143,6 +152,25 @@
const string &key_suffix);
+typedef args::Package<const string &, const string &, float,
+ const string &, const string &> FarEqualInnerArgs;
+typedef args::WithReturnValue<bool, FarEqualInnerArgs> FarEqualArgs;
+
+template <class Arc>
+void FarEqual(FarEqualArgs *args) {
+ args->retval = fst::FarEqual<Arc>(
+ args->args.arg1, args->args.arg2, args->args.arg3,
+ args->args.arg4, args->args.arg5);
+}
+
+bool FarEqual(const string &filename1,
+ const string &filename2,
+ const string &arc_type,
+ float delta = kDelta,
+ const string &begin_key = string(),
+ const string &end_key = string());
+
+
typedef args::Package<const vector<string> &, int32,
const string&, const string&, const string&,
const string&> FarExtractArgs;
@@ -180,7 +208,9 @@
const string &begin_key;
const string &end_key;
const bool print_key;
+ const bool print_weight;
const string &symbols_fname;
+ const bool initial_symbols;
const int32 generate_filenames;
const string &filename_prefix;
const string &filename_suffix;
@@ -188,12 +218,14 @@
FarPrintStringsArgs(
const vector<string> &ifilenames, const FarEntryType entry_type,
const FarTokenType token_type, const string &begin_key,
- const string &end_key, const bool print_key,
- const string &symbols_fname, const int32 generate_filenames,
+ const string &end_key, const bool print_key, const bool print_weight,
+ const string &symbols_fname, const bool initial_symbols,
+ const int32 generate_filenames,
const string &filename_prefix, const string &filename_suffix) :
ifilenames(ifilenames), entry_type(entry_type), token_type(token_type),
- begin_key(begin_key), end_key(end_key), print_key(print_key),
- symbols_fname(symbols_fname),
+ begin_key(begin_key), end_key(end_key),
+ print_key(print_key), print_weight(print_weight),
+ symbols_fname(symbols_fname), initial_symbols(initial_symbols),
generate_filenames(generate_filenames), filename_prefix(filename_prefix),
filename_suffix(filename_suffix) { }
};
@@ -202,9 +234,9 @@
void FarPrintStrings(FarPrintStringsArgs *args) {
fst::FarPrintStrings<Arc>(
args->ifilenames, args->entry_type, args->token_type,
- args->begin_key, args->end_key, args->print_key,
- args->symbols_fname, args->generate_filenames, args->filename_prefix,
- args->filename_suffix);
+ args->begin_key, args->end_key, args->print_key, args->print_weight,
+ args->symbols_fname, args->initial_symbols, args->generate_filenames,
+ args->filename_prefix, args->filename_suffix);
}
@@ -215,7 +247,9 @@
const string &begin_key,
const string &end_key,
const bool print_key,
+ const bool print_weight,
const string &symbols_fname,
+ const bool initial_symbols,
const int32 generate_filenames,
const string &filename_prefix,
const string &filename_suffix);
@@ -227,6 +261,7 @@
#define REGISTER_FST_FAR_OPERATIONS(ArcType) \
REGISTER_FST_OPERATION(FarCompileStrings, ArcType, FarCompileStringsArgs); \
REGISTER_FST_OPERATION(FarCreate, ArcType, FarCreateArgs); \
+ REGISTER_FST_OPERATION(FarEqual, ArcType, FarEqualArgs); \
REGISTER_FST_OPERATION(FarExtract, ArcType, FarExtractArgs); \
REGISTER_FST_OPERATION(FarInfo, ArcType, FarInfoArgs); \
REGISTER_FST_OPERATION(FarPrintStrings, ArcType, FarPrintStringsArgs)
diff --git a/src/include/fst/extensions/far/info.h b/src/include/fst/extensions/far/info.h
index f010546..100fe68 100644
--- a/src/include/fst/extensions/far/info.h
+++ b/src/include/fst/extensions/far/info.h
@@ -34,7 +34,7 @@
StateIterator<Fst<Arc> > siter(fst);
for (; !siter.Done(); siter.Next(), ++(*nstate)) {
ArcIterator<Fst<Arc> > aiter(fst, siter.Value());
- for (; !aiter.Done(); aiter.Next(), ++(*narc));
+ for (; !aiter.Done(); aiter.Next(), ++(*narc)) {}
}
}
diff --git a/src/include/fst/extensions/far/print-strings.h b/src/include/fst/extensions/far/print-strings.h
index aff1e51..dcc7351 100644
--- a/src/include/fst/extensions/far/print-strings.h
+++ b/src/include/fst/extensions/far/print-strings.h
@@ -27,17 +27,21 @@
using std::vector;
#include <fst/extensions/far/far.h>
+#include <fst/shortest-distance.h>
#include <fst/string.h>
+DECLARE_string(far_field_separator);
+
namespace fst {
template <class Arc>
void FarPrintStrings(
const vector<string> &ifilenames, const FarEntryType entry_type,
const FarTokenType far_token_type, const string &begin_key,
- const string &end_key, const bool print_key, const string &symbols_fname,
- const int32 generate_filenames, const string &filename_prefix,
- const string &filename_suffix) {
+ const string &end_key, const bool print_key, const bool print_weight,
+ const string &symbols_fname, const bool initial_symbols,
+ const int32 generate_filenames,
+ const string &filename_prefix, const string &filename_suffix) {
typename StringPrinter<Arc>::TokenType token_type;
if (far_token_type == FTT_SYMBOL) {
@@ -54,7 +58,9 @@
const SymbolTable *syms = 0;
if (!symbols_fname.empty()) {
// allow negative flag?
- syms = SymbolTable::ReadText(symbols_fname, true);
+ SymbolTableTextOptions opts;
+ opts.allow_negative = true;
+ syms = SymbolTable::ReadText(symbols_fname, opts);
if (!syms) {
FSTERROR() << "FarPrintStrings: error reading symbol table: "
<< symbols_fname;
@@ -62,8 +68,6 @@
}
}
- StringPrinter<Arc> string_printer(token_type, syms);
-
FarReader<Arc> *far_reader = FarReader<Arc>::Open(ifilenames);
if (!far_reader) return;
@@ -83,14 +87,21 @@
okey = key;
const Fst<Arc> &fst = far_reader->GetFst();
+ if (i == 1 && initial_symbols && syms == 0 && fst.InputSymbols() != 0)
+ syms = fst.InputSymbols()->Copy();
string str;
VLOG(2) << "Handling key: " << key;
+ StringPrinter<Arc> string_printer(
+ token_type, syms ? syms : fst.InputSymbols());
string_printer(fst, &str);
if (entry_type == FET_LINE) {
if (print_key)
- cout << key << "\t";
- cout << str << endl;
+ cout << key << FLAGS_far_field_separator[0];
+ cout << str;
+ if (print_weight)
+ cout << FLAGS_far_field_separator[0] << ShortestDistance(fst);
+ cout << endl;
} else if (entry_type == FET_FILE) {
stringstream sstrm;
if (generate_filenames) {
@@ -117,6 +128,7 @@
ostrm << "\n";
}
}
+ delete syms;
}
diff --git a/src/include/fst/extensions/far/stlist.h b/src/include/fst/extensions/far/stlist.h
index 4738181..1cdc80c 100644
--- a/src/include/fst/extensions/far/stlist.h
+++ b/src/include/fst/extensions/far/stlist.h
@@ -26,6 +26,7 @@
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/util.h>
#include <algorithm>
@@ -58,7 +59,7 @@
explicit STListWriter(const string filename)
: stream_(
- filename.empty() ? &std::cout :
+ filename.empty() ? &cout :
new ofstream(filename.c_str(), ofstream::out | ofstream::binary)),
error_(false) {
WriteType(*stream_, kSTListMagicNumber);
@@ -92,7 +93,7 @@
~STListWriter() {
WriteType(*stream_, string());
- if (stream_ != &std::cout)
+ if (stream_ != &cout)
delete stream_;
}
@@ -127,7 +128,7 @@
for (size_t i = 0; i < filenames.size(); ++i) {
if (filenames[i].empty()) {
if (!has_stdin) {
- streams_[i] = &std::cin;
+ streams_[i] = &cin;
sources_[i] = "stdin";
has_stdin = true;
} else {
@@ -177,7 +178,7 @@
~STListReader() {
for (size_t i = 0; i < streams_.size(); ++i) {
- if (streams_[i] != &std::cin)
+ if (streams_[i] != &cin)
delete streams_[i];
}
if (entry_)
diff --git a/src/include/fst/extensions/far/sttable.h b/src/include/fst/extensions/far/sttable.h
index 3a03133..3ce0a4b 100644
--- a/src/include/fst/extensions/far/sttable.h
+++ b/src/include/fst/extensions/far/sttable.h
@@ -29,6 +29,7 @@
#include <algorithm>
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/util.h>
namespace fst {
diff --git a/src/include/fst/extensions/ngram/bitmap-index.h b/src/include/fst/extensions/ngram/bitmap-index.h
new file mode 100644
index 0000000..f5a5ba7
--- /dev/null
+++ b/src/include/fst/extensions/ngram/bitmap-index.h
@@ -0,0 +1,183 @@
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: sorenj@google.com (Jeffrey Sorensen)
+
+#ifndef FST_EXTENSIONS_NGRAM_BITMAP_INDEX_H_
+#define FST_EXTENSIONS_NGRAM_BITMAP_INDEX_H_
+
+#include <vector>
+using std::vector;
+
+#include <fst/compat.h>
+
+// This class is a bitstring storage class with an index that allows
+// seeking to the Nth set or clear bit in time O(Log(N)) where N is
+// the length of the bit vector. In addition, it allows counting set or
+// clear bits over ranges in constant time.
+//
+// This is accomplished by maintaining an "secondary" index of limited
+// size in bits that maintains a running count of the number of bits set
+// in each block of bitmap data. A block is defined as the number of
+// uint64 values that can fit in the secondary index before an overflow
+// occurs.
+//
+// To handle overflows, a "primary" index containing a running count of
+// bits set in each block is created using the type uint64.
+
+namespace fst {
+
+class BitmapIndex {
+ public:
+ static size_t StorageSize(size_t size) {
+ return ((size + kStorageBlockMask) >> kStorageLogBitSize);
+ }
+
+ BitmapIndex() : bits_(NULL), size_(0) { }
+
+ bool Get(size_t index) const {
+ return (bits_[index >> kStorageLogBitSize] &
+ (kOne << (index & kStorageBlockMask))) != 0;
+ }
+
+ static void Set(uint64* bits, size_t index) {
+ bits[index >> kStorageLogBitSize] |= (kOne << (index & kStorageBlockMask));
+ }
+
+ static void Clear(uint64* bits, size_t index) {
+ bits[index >> kStorageLogBitSize] &= ~(kOne << (index & kStorageBlockMask));
+ }
+
+ size_t Bits() const {
+ return size_;
+ }
+
+ size_t ArraySize() const {
+ return StorageSize(size_);
+ }
+
+ // Returns the number of one bits in the bitmap
+ size_t GetOnesCount() const {
+ return primary_index_[primary_index_size() - 1];
+ }
+
+ // Returns the number of one bits in positions 0 to limit - 1.
+ // REQUIRES: limit <= Bits()
+ size_t Rank1(size_t end) const;
+
+ // Returns the number of one bits in the range start to end - 1.
+ // REQUIRES: limit <= Bits()
+ size_t GetOnesCountInRange(size_t start, size_t end) const {
+ return Rank1(end) - Rank1(start);
+ }
+
+ // Returns the number of zero bits in positions 0 to limit - 1.
+ // REQUIRES: limit <= Bits()
+ size_t Rank0(size_t end) const {
+ return end - Rank1(end);
+ }
+
+ // Returns the number of zero bits in the range start to end - 1.
+ // REQUIRES: limit <= Bits()
+ size_t GetZeroesCountInRange(size_t start, size_t end) const {
+ return end - start - GetOnesCountInRange(start, end);
+ }
+
+ // Return true if any bit between begin inclusive and end exclusive
+ // is set. 0 <= begin <= end <= Bits() is required.
+ //
+ bool TestRange(size_t start, size_t end) const {
+ return Rank1(end) > Rank1(start);
+ }
+
+ // Returns the offset to the nth set bit (zero based)
+ // or Bits() if index >= number of ones
+ size_t Select1(size_t bit_index) const;
+
+ // Returns the offset to the nth clear bit (zero based)
+ // or Bits() if index > number of
+ size_t Select0(size_t bit_index) const;
+
+ // Rebuilds from index for the associated Bitmap, should be called
+ // whenever changes have been made to the Bitmap or else behavior
+ // of the indexed bitmap methods will be undefined.
+ void BuildIndex(const uint64 *bits, size_t size);
+
+ // the secondary index accumulates counts until it can possibly overflow
+ // this constant computes the number of uint64 units that can fit into
+ // units the size of uint16.
+ static const uint64 kOne = 1;
+ static const uint32 kStorageBitSize = 64;
+ static const uint32 kStorageLogBitSize = 6;
+ static const uint32 kSecondaryBlockSize = ((1 << 16) - 1)
+ >> kStorageLogBitSize;
+
+ private:
+ static const uint32 kStorageBlockMask = kStorageBitSize - 1;
+
+ // returns, from the index, the count of ones up to array_index
+ size_t get_index_ones_count(size_t array_index) const;
+
+ // because the indexes, both primary and secondary, contain a running
+ // count of the population of one bits contained in [0,i), there is
+ // no reason to have an element in the zeroth position as this value would
+ // necessarily be zero. (The bits are indexed in a zero based way.) Thus
+ // we don't store the 0th element in either index. Both of the following
+ // functions, if greater than 0, must be decremented by one before retreiving
+ // the value from the corresponding array.
+ // returns the 1 + the block that contains the bitindex in question
+ // the inverted version works the same but looks for zeros using an inverted
+ // view of the index
+ size_t find_primary_block(size_t bit_index) const;
+
+ size_t find_inverted_primary_block(size_t bit_index) const;
+
+ // similarly, the secondary index (which resets its count to zero at
+ // the end of every kSecondaryBlockSize entries) does not store the element
+ // at 0. Note that the rem_bit_index parameter is the number of bits
+ // within the secondary block, after the bits accounted for by the primary
+ // block have been removed (i.e. the remaining bits) And, because we
+ // reset to zero with each new block, there is no need to store those
+ // actual zeros.
+ // returns 1 + the secondary block that contains the bitindex in question
+ size_t find_secondary_block(size_t block, size_t rem_bit_index) const;
+
+ size_t find_inverted_secondary_block(size_t block, size_t rem_bit_index)
+ const;
+
+ // We create a primary index based upon the number of secondary index
+ // blocks. The primary index uses fields wide enough to accomodate any
+ // index of the bitarray so cannot overflow
+ // The primary index is the actual running
+ // count of one bits set for all blocks (and, thus, all uint64s).
+ size_t primary_index_size() const {
+ return (ArraySize() + kSecondaryBlockSize - 1) / kSecondaryBlockSize;
+ }
+
+ const uint64* bits_;
+ size_t size_;
+
+ // The primary index contains the running popcount of all blocks
+ // which means the nth value contains the popcounts of
+ // [0,n*kSecondaryBlockSize], however, the 0th element is omitted.
+ vector<uint32> primary_index_;
+ // The secondary index contains the running popcount of the associated
+ // bitmap. It is the same length (in units of uint16) as the
+ // bitmap's map is in units of uint64s.
+ vector<uint16> secondary_index_;
+};
+
+} // end namespace fst
+
+#endif // FST_EXTENSIONS_NGRAM_BITMAP_INDEX_H_
diff --git a/src/include/fst/extensions/ngram/ngram-fst.h b/src/include/fst/extensions/ngram/ngram-fst.h
new file mode 100644
index 0000000..eee664a
--- /dev/null
+++ b/src/include/fst/extensions/ngram/ngram-fst.h
@@ -0,0 +1,912 @@
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: sorenj@google.com (Jeffrey Sorensen)
+//
+#ifndef FST_EXTENSIONS_NGRAM_NGRAM_FST_H_
+#define FST_EXTENSIONS_NGRAM_NGRAM_FST_H_
+
+#include <stddef.h>
+#include <string.h>
+#include <algorithm>
+#include <string>
+#include <vector>
+using std::vector;
+
+#include <fst/compat.h>
+#include <fst/fstlib.h>
+#include <fst/extensions/ngram/bitmap-index.h>
+
+// NgramFst implements a n-gram language model based upon the LOUDS data
+// structure. Please refer to "Unary Data Strucutres for Language Models"
+// http://research.google.com/pubs/archive/37218.pdf
+
+namespace fst {
+template <class A> class NGramFst;
+template <class A> class NGramFstMatcher;
+
+// Instance data containing mutable state for bookkeeping repeated access to
+// the same state.
+template <class A>
+struct NGramFstInst {
+ typedef typename A::Label Label;
+ typedef typename A::StateId StateId;
+ typedef typename A::Weight Weight;
+ StateId state_;
+ size_t num_futures_;
+ size_t offset_;
+ size_t node_;
+ StateId node_state_;
+ vector<Label> context_;
+ StateId context_state_;
+ NGramFstInst()
+ : state_(kNoStateId), node_state_(kNoStateId),
+ context_state_(kNoStateId) { }
+};
+
+// Implementation class for LOUDS based NgramFst interface
+template <class A>
+class NGramFstImpl : public FstImpl<A> {
+ using FstImpl<A>::SetInputSymbols;
+ using FstImpl<A>::SetOutputSymbols;
+ using FstImpl<A>::SetType;
+ using FstImpl<A>::WriteHeader;
+
+ friend class ArcIterator<NGramFst<A> >;
+ friend class NGramFstMatcher<A>;
+
+ public:
+ using FstImpl<A>::InputSymbols;
+ using FstImpl<A>::SetProperties;
+ using FstImpl<A>::Properties;
+
+ typedef A Arc;
+ typedef typename A::Label Label;
+ typedef typename A::StateId StateId;
+ typedef typename A::Weight Weight;
+
+ NGramFstImpl() : data_(0), owned_(false) {
+ SetType("ngram");
+ SetInputSymbols(NULL);
+ SetOutputSymbols(NULL);
+ SetProperties(kStaticProperties);
+ }
+
+ NGramFstImpl(const Fst<A> &fst, vector<StateId>* order_out);
+
+ ~NGramFstImpl() {
+ if (owned_) {
+ delete [] data_;
+ }
+ }
+
+ static NGramFstImpl<A>* Read(istream &strm, // NOLINT
+ const FstReadOptions &opts) {
+ NGramFstImpl<A>* impl = new NGramFstImpl();
+ FstHeader hdr;
+ if (!impl->ReadHeader(strm, opts, kMinFileVersion, &hdr)) return 0;
+ uint64 num_states, num_futures, num_final;
+ const size_t offset = sizeof(num_states) + sizeof(num_futures) +
+ sizeof(num_final);
+ // Peek at num_states and num_futures to see how much more needs to be read.
+ strm.read(reinterpret_cast<char *>(&num_states), sizeof(num_states));
+ strm.read(reinterpret_cast<char *>(&num_futures), sizeof(num_futures));
+ strm.read(reinterpret_cast<char *>(&num_final), sizeof(num_final));
+ size_t size = Storage(num_states, num_futures, num_final);
+ char* data = new char[size];
+ // Copy num_states, num_futures and num_final back into data.
+ memcpy(data, reinterpret_cast<char *>(&num_states), sizeof(num_states));
+ memcpy(data + sizeof(num_states), reinterpret_cast<char *>(&num_futures),
+ sizeof(num_futures));
+ memcpy(data + sizeof(num_states) + sizeof(num_futures),
+ reinterpret_cast<char *>(&num_final), sizeof(num_final));
+ strm.read(data + offset, size - offset);
+ if (!strm) {
+ delete impl;
+ return NULL;
+ }
+ impl->Init(data, true /* owned */);
+ return impl;
+ }
+
+ bool Write(ostream &strm, // NOLINT
+ const FstWriteOptions &opts) const {
+ FstHeader hdr;
+ hdr.SetStart(Start());
+ hdr.SetNumStates(num_states_);
+ WriteHeader(strm, opts, kFileVersion, &hdr);
+ strm.write(data_, Storage(num_states_, num_futures_, num_final_));
+ return strm;
+ }
+
+ StateId Start() const {
+ return 1;
+ }
+
+ Weight Final(StateId state) const {
+ if (final_index_.Get(state)) {
+ return final_probs_[final_index_.Rank1(state)];
+ } else {
+ return Weight::Zero();
+ }
+ }
+
+ size_t NumArcs(StateId state, NGramFstInst<A> *inst = NULL) const {
+ if (inst == NULL) {
+ const size_t next_zero = future_index_.Select0(state + 1);
+ const size_t this_zero = future_index_.Select0(state);
+ return next_zero - this_zero - 1;
+ }
+ SetInstFuture(state, inst);
+ return inst->num_futures_ + ((state == 0) ? 0 : 1);
+ }
+
+ size_t NumInputEpsilons(StateId state) const {
+ // State 0 has no parent, thus no backoff.
+ if (state == 0) return 0;
+ return 1;
+ }
+
+ size_t NumOutputEpsilons(StateId state) const {
+ return NumInputEpsilons(state);
+ }
+
+ StateId NumStates() const {
+ return num_states_;
+ }
+
+ void InitStateIterator(StateIteratorData<A>* data) const {
+ data->base = 0;
+ data->nstates = num_states_;
+ }
+
+ static size_t Storage(uint64 num_states, uint64 num_futures,
+ uint64 num_final) {
+ uint64 b64;
+ Weight weight;
+ Label label;
+ size_t offset = sizeof(num_states) + sizeof(num_futures) +
+ sizeof(num_final);
+ offset += sizeof(b64) * (
+ BitmapIndex::StorageSize(num_states * 2 + 1) +
+ BitmapIndex::StorageSize(num_futures + num_states + 1) +
+ BitmapIndex::StorageSize(num_states));
+ offset += (num_states + 1) * sizeof(label) + num_futures * sizeof(label);
+ // Pad for alignemnt, see
+ // http://en.wikipedia.org/wiki/Data_structure_alignment#Computing_padding
+ offset = (offset + sizeof(weight) - 1) & ~(sizeof(weight) - 1);
+ offset += (num_states + 1) * sizeof(weight) + num_final * sizeof(weight) +
+ (num_futures + 1) * sizeof(weight);
+ return offset;
+ }
+
+ void SetInstFuture(StateId state, NGramFstInst<A> *inst) const {
+ if (inst->state_ != state) {
+ inst->state_ = state;
+ const size_t next_zero = future_index_.Select0(state + 1);
+ const size_t this_zero = future_index_.Select0(state);
+ inst->num_futures_ = next_zero - this_zero - 1;
+ inst->offset_ = future_index_.Rank1(future_index_.Select0(state) + 1);
+ }
+ }
+
+ void SetInstNode(NGramFstInst<A> *inst) const {
+ if (inst->node_state_ != inst->state_) {
+ inst->node_state_ = inst->state_;
+ inst->node_ = context_index_.Select1(inst->state_);
+ }
+ }
+
+ void SetInstContext(NGramFstInst<A> *inst) const {
+ SetInstNode(inst);
+ if (inst->context_state_ != inst->state_) {
+ inst->context_state_ = inst->state_;
+ inst->context_.clear();
+ size_t node = inst->node_;
+ while (node != 0) {
+ inst->context_.push_back(context_words_[context_index_.Rank1(node)]);
+ node = context_index_.Select1(context_index_.Rank0(node) - 1);
+ }
+ }
+ }
+
+ // Access to the underlying representation
+ const char* GetData(size_t* data_size) const {
+ *data_size = Storage(num_states_, num_futures_, num_final_);
+ return data_;
+ }
+
+ void Init(const char* data, bool owned);
+
+ private:
+ StateId Transition(const vector<Label> &context, Label future) const;
+
+ // Properties always true for this Fst class.
+ static const uint64 kStaticProperties = kAcceptor | kIDeterministic |
+ kODeterministic | kEpsilons | kIEpsilons | kOEpsilons | kILabelSorted |
+ kOLabelSorted | kWeighted | kCyclic | kInitialAcyclic | kNotTopSorted |
+ kAccessible | kCoAccessible | kNotString | kExpanded;
+ // Current file format version.
+ static const int kFileVersion = 4;
+ // Minimum file format version supported.
+ static const int kMinFileVersion = 4;
+
+ const char* data_;
+ bool owned_; // True if we own data_
+ uint64 num_states_, num_futures_, num_final_;
+ size_t root_num_children_;
+ const Label *root_children_;
+ size_t root_first_child_;
+ // borrowed references
+ const uint64 *context_, *future_, *final_;
+ const Label *context_words_, *future_words_;
+ const Weight *backoff_, *final_probs_, *future_probs_;
+ BitmapIndex context_index_;
+ BitmapIndex future_index_;
+ BitmapIndex final_index_;
+
+ void operator=(const NGramFstImpl<A> &); // Disallow
+};
+
+template<typename A>
+NGramFstImpl<A>::NGramFstImpl(const Fst<A> &fst, vector<StateId>* order_out)
+ : data_(0), owned_(false) {
+ typedef A Arc;
+ typedef typename Arc::Label Label;
+ typedef typename Arc::Weight Weight;
+ typedef typename Arc::StateId StateId;
+ SetType("ngram");
+ SetInputSymbols(fst.InputSymbols());
+ SetOutputSymbols(fst.OutputSymbols());
+ SetProperties(kStaticProperties);
+
+ // Check basic requirements for an OpenGRM language model Fst.
+ int64 props = kAcceptor | kIDeterministic | kIEpsilons | kILabelSorted;
+ if (fst.Properties(props, true) != props) {
+ FSTERROR() << "NGramFst only accepts OpenGRM langauge models as input";
+ SetProperties(kError, kError);
+ return;
+ }
+
+ int64 num_states = CountStates(fst);
+ Label* context = new Label[num_states];
+
+ // Find the unigram state by starting from the start state, following
+ // epsilons.
+ StateId unigram = fst.Start();
+ while (1) {
+ ArcIterator<Fst<A> > aiter(fst, unigram);
+ if (aiter.Done()) {
+ FSTERROR() << "Start state has no arcs";
+ SetProperties(kError, kError);
+ return;
+ }
+ if (aiter.Value().ilabel != 0) break;
+ unigram = aiter.Value().nextstate;
+ }
+
+ // Each state's context is determined by the subtree it is under from the
+ // unigram state.
+ queue<pair<StateId, Label> > label_queue;
+ vector<bool> visited(num_states);
+ // Force an epsilon link to the start state.
+ label_queue.push(make_pair(fst.Start(), 0));
+ for (ArcIterator<Fst<A> > aiter(fst, unigram);
+ !aiter.Done(); aiter.Next()) {
+ label_queue.push(make_pair(aiter.Value().nextstate, aiter.Value().ilabel));
+ }
+ // investigate states in breadth first fashion to assign context words.
+ while (!label_queue.empty()) {
+ pair<StateId, Label> &now = label_queue.front();
+ if (!visited[now.first]) {
+ context[now.first] = now.second;
+ visited[now.first] = true;
+ for (ArcIterator<Fst<A> > aiter(fst, now.first);
+ !aiter.Done(); aiter.Next()) {
+ const Arc &arc = aiter.Value();
+ if (arc.ilabel != 0) {
+ label_queue.push(make_pair(arc.nextstate, now.second));
+ }
+ }
+ }
+ label_queue.pop();
+ }
+ visited.clear();
+
+ // The arc from the start state should be assigned an epsilon to put it
+ // in front of the all other labels (which makes Start state 1 after
+ // unigram which is state 0).
+ context[fst.Start()] = 0;
+
+ // Build the tree of contexts fst by reversing the epsilon arcs from fst.
+ VectorFst<Arc> context_fst;
+ uint64 num_final = 0;
+ for (int i = 0; i < num_states; ++i) {
+ if (fst.Final(i) != Weight::Zero()) {
+ ++num_final;
+ }
+ context_fst.SetFinal(context_fst.AddState(), fst.Final(i));
+ }
+ context_fst.SetStart(unigram);
+ context_fst.SetInputSymbols(fst.InputSymbols());
+ context_fst.SetOutputSymbols(fst.OutputSymbols());
+ int64 num_context_arcs = 0;
+ int64 num_futures = 0;
+ for (StateIterator<Fst<A> > siter(fst); !siter.Done(); siter.Next()) {
+ const StateId &state = siter.Value();
+ num_futures += fst.NumArcs(state) - fst.NumInputEpsilons(state);
+ ArcIterator<Fst<A> > aiter(fst, state);
+ if (!aiter.Done()) {
+ const Arc &arc = aiter.Value();
+ // this arc goes from state to arc.nextstate, so create an arc from
+ // arc.nextstate to state to reverse it.
+ if (arc.ilabel == 0) {
+ context_fst.AddArc(arc.nextstate, Arc(context[state], context[state],
+ arc.weight, state));
+ num_context_arcs++;
+ }
+ }
+ }
+ if (num_context_arcs != context_fst.NumStates() - 1) {
+ FSTERROR() << "Number of contexts arcs != number of states - 1";
+ SetProperties(kError, kError);
+ return;
+ }
+ if (context_fst.NumStates() != num_states) {
+ FSTERROR() << "Number of contexts != number of states";
+ SetProperties(kError, kError);
+ return;
+ }
+ int64 context_props = context_fst.Properties(kIDeterministic |
+ kILabelSorted, true);
+ if (!(context_props & kIDeterministic)) {
+ FSTERROR() << "Input fst is not structured properly";
+ SetProperties(kError, kError);
+ return;
+ }
+ if (!(context_props & kILabelSorted)) {
+ ArcSort(&context_fst, ILabelCompare<Arc>());
+ }
+
+ delete [] context;
+
+ uint64 b64;
+ Weight weight;
+ Label label = kNoLabel;
+ const size_t storage = Storage(num_states, num_futures, num_final);
+ char* data = new char[storage];
+ memset(data, 0, storage);
+ size_t offset = 0;
+ memcpy(data + offset, reinterpret_cast<char *>(&num_states),
+ sizeof(num_states));
+ offset += sizeof(num_states);
+ memcpy(data + offset, reinterpret_cast<char *>(&num_futures),
+ sizeof(num_futures));
+ offset += sizeof(num_futures);
+ memcpy(data + offset, reinterpret_cast<char *>(&num_final),
+ sizeof(num_final));
+ offset += sizeof(num_final);
+ uint64* context_bits = reinterpret_cast<uint64*>(data + offset);
+ offset += BitmapIndex::StorageSize(num_states * 2 + 1) * sizeof(b64);
+ uint64* future_bits = reinterpret_cast<uint64*>(data + offset);
+ offset +=
+ BitmapIndex::StorageSize(num_futures + num_states + 1) * sizeof(b64);
+ uint64* final_bits = reinterpret_cast<uint64*>(data + offset);
+ offset += BitmapIndex::StorageSize(num_states) * sizeof(b64);
+ Label* context_words = reinterpret_cast<Label*>(data + offset);
+ offset += (num_states + 1) * sizeof(label);
+ Label* future_words = reinterpret_cast<Label*>(data + offset);
+ offset += num_futures * sizeof(label);
+ offset = (offset + sizeof(weight) - 1) & ~(sizeof(weight) - 1);
+ Weight* backoff = reinterpret_cast<Weight*>(data + offset);
+ offset += (num_states + 1) * sizeof(weight);
+ Weight* final_probs = reinterpret_cast<Weight*>(data + offset);
+ offset += num_final * sizeof(weight);
+ Weight* future_probs = reinterpret_cast<Weight*>(data + offset);
+ int64 context_arc = 0, future_arc = 0, context_bit = 0, future_bit = 0,
+ final_bit = 0;
+
+ // pseudo-root bits
+ BitmapIndex::Set(context_bits, context_bit++);
+ ++context_bit;
+ context_words[context_arc] = label;
+ backoff[context_arc] = Weight::Zero();
+ context_arc++;
+
+ ++future_bit;
+ if (order_out) {
+ order_out->clear();
+ order_out->resize(num_states);
+ }
+
+ queue<StateId> context_q;
+ context_q.push(context_fst.Start());
+ StateId state_number = 0;
+ while (!context_q.empty()) {
+ const StateId &state = context_q.front();
+ if (order_out) {
+ (*order_out)[state] = state_number;
+ }
+
+ const Weight &final = context_fst.Final(state);
+ if (final != Weight::Zero()) {
+ BitmapIndex::Set(final_bits, state_number);
+ final_probs[final_bit] = final;
+ ++final_bit;
+ }
+
+ for (ArcIterator<VectorFst<A> > aiter(context_fst, state);
+ !aiter.Done(); aiter.Next()) {
+ const Arc &arc = aiter.Value();
+ context_words[context_arc] = arc.ilabel;
+ backoff[context_arc] = arc.weight;
+ ++context_arc;
+ BitmapIndex::Set(context_bits, context_bit++);
+ context_q.push(arc.nextstate);
+ }
+ ++context_bit;
+
+ for (ArcIterator<Fst<A> > aiter(fst, state); !aiter.Done(); aiter.Next()) {
+ const Arc &arc = aiter.Value();
+ if (arc.ilabel != 0) {
+ future_words[future_arc] = arc.ilabel;
+ future_probs[future_arc] = arc.weight;
+ ++future_arc;
+ BitmapIndex::Set(future_bits, future_bit++);
+ }
+ }
+ ++future_bit;
+ ++state_number;
+ context_q.pop();
+ }
+
+ if ((state_number != num_states) ||
+ (context_bit != num_states * 2 + 1) ||
+ (context_arc != num_states) ||
+ (future_arc != num_futures) ||
+ (future_bit != num_futures + num_states + 1) ||
+ (final_bit != num_final)) {
+ FSTERROR() << "Structure problems detected during construction";
+ SetProperties(kError, kError);
+ return;
+ }
+
+ Init(data, true /* owned */);
+}
+
+template<typename A>
+inline void NGramFstImpl<A>::Init(const char* data, bool owned) {
+ if (owned_) {
+ delete [] data_;
+ }
+ owned_ = owned;
+ data_ = data;
+ size_t offset = 0;
+ num_states_ = *(reinterpret_cast<const uint64*>(data_ + offset));
+ offset += sizeof(num_states_);
+ num_futures_ = *(reinterpret_cast<const uint64*>(data_ + offset));
+ offset += sizeof(num_futures_);
+ num_final_ = *(reinterpret_cast<const uint64*>(data_ + offset));
+ offset += sizeof(num_final_);
+ uint64 bits;
+ size_t context_bits = num_states_ * 2 + 1;
+ size_t future_bits = num_futures_ + num_states_ + 1;
+ context_ = reinterpret_cast<const uint64*>(data_ + offset);
+ offset += BitmapIndex::StorageSize(context_bits) * sizeof(bits);
+ future_ = reinterpret_cast<const uint64*>(data_ + offset);
+ offset += BitmapIndex::StorageSize(future_bits) * sizeof(bits);
+ final_ = reinterpret_cast<const uint64*>(data_ + offset);
+ offset += BitmapIndex::StorageSize(num_states_ + 1) * sizeof(bits);
+ context_words_ = reinterpret_cast<const Label*>(data_ + offset);
+ offset += (num_states_ + 1) * sizeof(*context_words_);
+ future_words_ = reinterpret_cast<const Label*>(data_ + offset);
+ offset += num_futures_ * sizeof(*future_words_);
+ offset = (offset + sizeof(*backoff_) - 1) & ~(sizeof(*backoff_) - 1);
+ backoff_ = reinterpret_cast<const Weight*>(data_ + offset);
+ offset += (num_states_ + 1) * sizeof(*backoff_);
+ final_probs_ = reinterpret_cast<const Weight*>(data_ + offset);
+ offset += num_final_ * sizeof(*final_probs_);
+ future_probs_ = reinterpret_cast<const Weight*>(data_ + offset);
+
+ context_index_.BuildIndex(context_, context_bits);
+ future_index_.BuildIndex(future_, future_bits);
+ final_index_.BuildIndex(final_, num_states_);
+
+ const size_t node_rank = context_index_.Rank1(0);
+ root_first_child_ = context_index_.Select0(node_rank) + 1;
+ if (context_index_.Get(root_first_child_) == false) {
+ FSTERROR() << "Missing unigrams";
+ SetProperties(kError, kError);
+ return;
+ }
+ const size_t last_child = context_index_.Select0(node_rank + 1) - 1;
+ root_num_children_ = last_child - root_first_child_ + 1;
+ root_children_ = context_words_ + context_index_.Rank1(root_first_child_);
+}
+
+template<typename A>
+inline typename A::StateId NGramFstImpl<A>::Transition(
+ const vector<Label> &context, Label future) const {
+ size_t num_children = root_num_children_;
+ const Label *children = root_children_;
+ const Label *loc = lower_bound(children, children + num_children, future);
+ if (loc == children + num_children || *loc != future) {
+ return context_index_.Rank1(0);
+ }
+ size_t node = root_first_child_ + loc - children;
+ size_t node_rank = context_index_.Rank1(node);
+ size_t first_child = context_index_.Select0(node_rank) + 1;
+ if (context_index_.Get(first_child) == false) {
+ return context_index_.Rank1(node);
+ }
+ size_t last_child = context_index_.Select0(node_rank + 1) - 1;
+ num_children = last_child - first_child + 1;
+ for (int word = context.size() - 1; word >= 0; --word) {
+ children = context_words_ + context_index_.Rank1(first_child);
+ loc = lower_bound(children, children + last_child - first_child + 1,
+ context[word]);
+ if (loc == children + last_child - first_child + 1 ||
+ *loc != context[word]) {
+ break;
+ }
+ node = first_child + loc - children;
+ node_rank = context_index_.Rank1(node);
+ first_child = context_index_.Select0(node_rank) + 1;
+ if (context_index_.Get(first_child) == false) break;
+ last_child = context_index_.Select0(node_rank + 1) - 1;
+ }
+ return context_index_.Rank1(node);
+}
+
+/*****************************************************************************/
+template<class A>
+class NGramFst : public ImplToExpandedFst<NGramFstImpl<A> > {
+ friend class ArcIterator<NGramFst<A> >;
+ friend class NGramFstMatcher<A>;
+
+ public:
+ typedef A Arc;
+ typedef typename A::StateId StateId;
+ typedef typename A::Label Label;
+ typedef typename A::Weight Weight;
+ typedef NGramFstImpl<A> Impl;
+
+ explicit NGramFst(const Fst<A> &dst)
+ : ImplToExpandedFst<Impl>(new Impl(dst, NULL)) {}
+
+ NGramFst(const Fst<A> &fst, vector<StateId>* order_out)
+ : ImplToExpandedFst<Impl>(new Impl(fst, order_out)) {}
+
+ // Because the NGramFstImpl is a const stateless data structure, there
+ // is never a need to do anything beside copy the reference.
+ NGramFst(const NGramFst<A> &fst, bool safe = false)
+ : ImplToExpandedFst<Impl>(fst, false) {}
+
+ NGramFst() : ImplToExpandedFst<Impl>(new Impl()) {}
+
+ // Non-standard constructor to initialize NGramFst directly from data.
+ NGramFst(const char* data, bool owned) : ImplToExpandedFst<Impl>(new Impl()) {
+ GetImpl()->Init(data, owned);
+ }
+
+ // Get method that gets the data associated with Init().
+ const char* GetData(size_t* data_size) const {
+ return GetImpl()->GetData(data_size);
+ }
+
+ virtual size_t NumArcs(StateId s) const {
+ return GetImpl()->NumArcs(s, &inst_);
+ }
+
+ virtual NGramFst<A>* Copy(bool safe = false) const {
+ return new NGramFst(*this, safe);
+ }
+
+ static NGramFst<A>* Read(istream &strm, const FstReadOptions &opts) {
+ Impl* impl = Impl::Read(strm, opts);
+ return impl ? new NGramFst<A>(impl) : 0;
+ }
+
+ static NGramFst<A>* Read(const string &filename) {
+ if (!filename.empty()) {
+ ifstream strm(filename.c_str(), ifstream::in | ifstream::binary);
+ if (!strm) {
+ LOG(ERROR) << "NGramFst::Read: Can't open file: " << filename;
+ return 0;
+ }
+ return Read(strm, FstReadOptions(filename));
+ } else {
+ return Read(cin, FstReadOptions("standard input"));
+ }
+ }
+
+ virtual bool Write(ostream &strm, const FstWriteOptions &opts) const {
+ return GetImpl()->Write(strm, opts);
+ }
+
+ virtual bool Write(const string &filename) const {
+ return Fst<A>::WriteFile(filename);
+ }
+
+ virtual inline void InitStateIterator(StateIteratorData<A>* data) const {
+ GetImpl()->InitStateIterator(data);
+ }
+
+ virtual inline void InitArcIterator(
+ StateId s, ArcIteratorData<A>* data) const;
+
+ virtual MatcherBase<A>* InitMatcher(MatchType match_type) const {
+ return new NGramFstMatcher<A>(*this, match_type);
+ }
+
+ private:
+ explicit NGramFst(Impl* impl) : ImplToExpandedFst<Impl>(impl) {}
+
+ Impl* GetImpl() const {
+ return
+ ImplToExpandedFst<Impl, ExpandedFst<A> >::GetImpl();
+ }
+
+ void SetImpl(Impl* impl, bool own_impl = true) {
+ ImplToExpandedFst<Impl, Fst<A> >::SetImpl(impl, own_impl);
+ }
+
+ mutable NGramFstInst<A> inst_;
+};
+
+template <class A> inline void
+NGramFst<A>::InitArcIterator(StateId s, ArcIteratorData<A>* data) const {
+ GetImpl()->SetInstFuture(s, &inst_);
+ GetImpl()->SetInstNode(&inst_);
+ data->base = new ArcIterator<NGramFst<A> >(*this, s);
+}
+
+/*****************************************************************************/
+template <class A>
+class NGramFstMatcher : public MatcherBase<A> {
+ public:
+ typedef A Arc;
+ typedef typename A::Label Label;
+ typedef typename A::StateId StateId;
+ typedef typename A::Weight Weight;
+
+ NGramFstMatcher(const NGramFst<A> &fst, MatchType match_type)
+ : fst_(fst), inst_(fst.inst_), match_type_(match_type),
+ current_loop_(false),
+ loop_(kNoLabel, 0, A::Weight::One(), kNoStateId) {
+ if (match_type_ == MATCH_OUTPUT) {
+ swap(loop_.ilabel, loop_.olabel);
+ }
+ }
+
+ NGramFstMatcher(const NGramFstMatcher<A> &matcher, bool safe = false)
+ : fst_(matcher.fst_), inst_(matcher.inst_),
+ match_type_(matcher.match_type_), current_loop_(false),
+ loop_(kNoLabel, 0, A::Weight::One(), kNoStateId) {
+ if (match_type_ == MATCH_OUTPUT) {
+ swap(loop_.ilabel, loop_.olabel);
+ }
+ }
+
+ virtual NGramFstMatcher<A>* Copy(bool safe = false) const {
+ return new NGramFstMatcher<A>(*this, safe);
+ }
+
+ virtual MatchType Type(bool test) const {
+ return match_type_;
+ }
+
+ virtual const Fst<A> &GetFst() const {
+ return fst_;
+ }
+
+ virtual uint64 Properties(uint64 props) const {
+ return props;
+ }
+
+ private:
+ virtual void SetState_(StateId s) {
+ fst_.GetImpl()->SetInstFuture(s, &inst_);
+ current_loop_ = false;
+ }
+
+ virtual bool Find_(Label label) {
+ const Label nolabel = kNoLabel;
+ done_ = true;
+ if (label == 0 || label == nolabel) {
+ if (label == 0) {
+ current_loop_ = true;
+ loop_.nextstate = inst_.state_;
+ }
+ // The unigram state has no epsilon arc.
+ if (inst_.state_ != 0) {
+ arc_.ilabel = arc_.olabel = 0;
+ fst_.GetImpl()->SetInstNode(&inst_);
+ arc_.nextstate = fst_.GetImpl()->context_index_.Rank1(
+ fst_.GetImpl()->context_index_.Select1(
+ fst_.GetImpl()->context_index_.Rank0(inst_.node_) - 1));
+ arc_.weight = fst_.GetImpl()->backoff_[inst_.state_];
+ done_ = false;
+ }
+ } else {
+ const Label *start = fst_.GetImpl()->future_words_ + inst_.offset_;
+ const Label *end = start + inst_.num_futures_;
+ const Label* search = lower_bound(start, end, label);
+ if (search != end && *search == label) {
+ size_t state = search - start;
+ arc_.ilabel = arc_.olabel = label;
+ arc_.weight = fst_.GetImpl()->future_probs_[inst_.offset_ + state];
+ fst_.GetImpl()->SetInstContext(&inst_);
+ arc_.nextstate = fst_.GetImpl()->Transition(inst_.context_, label);
+ done_ = false;
+ }
+ }
+ return !Done_();
+ }
+
+ virtual bool Done_() const {
+ return !current_loop_ && done_;
+ }
+
+ virtual const Arc& Value_() const {
+ return (current_loop_) ? loop_ : arc_;
+ }
+
+ virtual void Next_() {
+ if (current_loop_) {
+ current_loop_ = false;
+ } else {
+ done_ = true;
+ }
+ }
+
+ const NGramFst<A>& fst_;
+ NGramFstInst<A> inst_;
+ MatchType match_type_; // Supplied by caller
+ bool done_;
+ Arc arc_;
+ bool current_loop_; // Current arc is the implicit loop
+ Arc loop_;
+};
+
+/*****************************************************************************/
+template<class A>
+class ArcIterator<NGramFst<A> > : public ArcIteratorBase<A> {
+ public:
+ typedef A Arc;
+ typedef typename A::Label Label;
+ typedef typename A::StateId StateId;
+ typedef typename A::Weight Weight;
+
+ ArcIterator(const NGramFst<A> &fst, StateId state)
+ : lazy_(~0), impl_(fst.GetImpl()), i_(0), flags_(kArcValueFlags) {
+ inst_ = fst.inst_;
+ impl_->SetInstFuture(state, &inst_);
+ impl_->SetInstNode(&inst_);
+ }
+
+ bool Done() const {
+ return i_ >= ((inst_.node_ == 0) ? inst_.num_futures_ :
+ inst_.num_futures_ + 1);
+ }
+
+ const Arc &Value() const {
+ bool eps = (inst_.node_ != 0 && i_ == 0);
+ StateId state = (inst_.node_ == 0) ? i_ : i_ - 1;
+ if (flags_ & lazy_ & (kArcILabelValue | kArcOLabelValue)) {
+ arc_.ilabel =
+ arc_.olabel = eps ? 0 : impl_->future_words_[inst_.offset_ + state];
+ lazy_ &= ~(kArcILabelValue | kArcOLabelValue);
+ }
+ if (flags_ & lazy_ & kArcNextStateValue) {
+ if (eps) {
+ arc_.nextstate = impl_->context_index_.Rank1(
+ impl_->context_index_.Select1(
+ impl_->context_index_.Rank0(inst_.node_) - 1));
+ } else {
+ if (lazy_ & kArcNextStateValue) {
+ impl_->SetInstContext(&inst_); // first time only.
+ }
+ arc_.nextstate =
+ impl_->Transition(inst_.context_,
+ impl_->future_words_[inst_.offset_ + state]);
+ }
+ lazy_ &= ~kArcNextStateValue;
+ }
+ if (flags_ & lazy_ & kArcWeightValue) {
+ arc_.weight = eps ? impl_->backoff_[inst_.state_] :
+ impl_->future_probs_[inst_.offset_ + state];
+ lazy_ &= ~kArcWeightValue;
+ }
+ return arc_;
+ }
+
+ void Next() {
+ ++i_;
+ lazy_ = ~0;
+ }
+
+ size_t Position() const { return i_; }
+
+ void Reset() {
+ i_ = 0;
+ lazy_ = ~0;
+ }
+
+ void Seek(size_t a) {
+ if (i_ != a) {
+ i_ = a;
+ lazy_ = ~0;
+ }
+ }
+
+ uint32 Flags() const {
+ return flags_;
+ }
+
+ void SetFlags(uint32 f, uint32 m) {
+ flags_ &= ~m;
+ flags_ |= (f & kArcValueFlags);
+ }
+
+ private:
+ virtual bool Done_() const { return Done(); }
+ virtual const Arc& Value_() const { return Value(); }
+ virtual void Next_() { Next(); }
+ virtual size_t Position_() const { return Position(); }
+ virtual void Reset_() { Reset(); }
+ virtual void Seek_(size_t a) { Seek(a); }
+ uint32 Flags_() const { return Flags(); }
+ void SetFlags_(uint32 f, uint32 m) { SetFlags(f, m); }
+
+ mutable Arc arc_;
+ mutable uint32 lazy_;
+ const NGramFstImpl<A> *impl_;
+ mutable NGramFstInst<A> inst_;
+
+ size_t i_;
+ uint32 flags_;
+
+ DISALLOW_COPY_AND_ASSIGN(ArcIterator);
+};
+
+/*****************************************************************************/
+// Specialization for NGramFst; see generic version in fst.h
+// for sample usage (but use the ProdLmFst type!). This version
+// should inline.
+template <class A>
+class StateIterator<NGramFst<A> > : public StateIteratorBase<A> {
+ public:
+ typedef typename A::StateId StateId;
+
+ explicit StateIterator(const NGramFst<A> &fst)
+ : s_(0), num_states_(fst.NumStates()) { }
+
+ bool Done() const { return s_ >= num_states_; }
+ StateId Value() const { return s_; }
+ void Next() { ++s_; }
+ void Reset() { s_ = 0; }
+
+ private:
+ virtual bool Done_() const { return Done(); }
+ virtual StateId Value_() const { return Value(); }
+ virtual void Next_() { Next(); }
+ virtual void Reset_() { Reset(); }
+
+ StateId s_, num_states_;
+
+ DISALLOW_COPY_AND_ASSIGN(StateIterator);
+};
+} // namespace fst
+#endif // FST_EXTENSIONS_NGRAM_NGRAM_FST_H_
diff --git a/src/include/fst/extensions/ngram/nthbit.h b/src/include/fst/extensions/ngram/nthbit.h
new file mode 100644
index 0000000..d4a9a5a
--- /dev/null
+++ b/src/include/fst/extensions/ngram/nthbit.h
@@ -0,0 +1,46 @@
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright 2005-2010 Google, Inc.
+// Author: sorenj@google.com (Jeffrey Sorensen)
+// dr@google.com (Doug Rohde)
+
+#ifndef FST_EXTENSIONS_NGRAM_NTHBIT_H_
+#define FST_EXTENSIONS_NGRAM_NTHBIT_H_
+
+#include <fst/types.h>
+
+extern uint32 nth_bit_bit_offset[];
+
+inline uint32 nth_bit(uint64 v, uint32 r) {
+ uint32 shift = 0;
+ uint32 c = __builtin_popcount(v & 0xffffffff);
+ uint32 mask = -(r > c);
+ r -= c & mask;
+ shift += (32 & mask);
+
+ c = __builtin_popcount((v >> shift) & 0xffff);
+ mask = -(r > c);
+ r -= c & mask;
+ shift += (16 & mask);
+
+ c = __builtin_popcount((v >> shift) & 0xff);
+ mask = -(r > c);
+ r -= c & mask;
+ shift += (8 & mask);
+
+ return shift + ((nth_bit_bit_offset[(v >> shift) & 0xff] >>
+ ((r - 1) << 2)) & 0xf);
+}
+
+#endif // FST_EXTENSIONS_NGRAM_NTHBIT_H_
diff --git a/src/include/fst/extensions/pdt/collection.h b/src/include/fst/extensions/pdt/collection.h
index 26be504..24a443f 100644
--- a/src/include/fst/extensions/pdt/collection.h
+++ b/src/include/fst/extensions/pdt/collection.h
@@ -16,7 +16,7 @@
// Author: riley@google.com (Michael Riley)
//
// \file
-// Class to store a collection of sets with elements of type T.
+// Class to store a collection of ordered (multi-)sets with elements of type T.
#ifndef FST_EXTENSIONS_PDT_COLLECTION_H__
#define FST_EXTENSIONS_PDT_COLLECTION_H__
@@ -29,11 +29,11 @@
namespace fst {
-// Stores a collection of non-empty sets with elements of type T. A
-// default constructor, equality ==, a total order <, and an STL-style
-// hash class must be defined on the elements. Provides signed
-// integer ID (of type I) of each unique set. The IDs are allocated
-// starting from 0 in order.
+// Stores a collection of non-empty, ordered (multi-)sets with elements
+// of type T. A default constructor, equality ==, and an STL-style
+// hash class must be defined on the elements. Provides signed integer
+// ID (of type I) of each unique set. The IDs are allocated starting
+// from 0 in order.
template <class I, class T>
class Collection {
public:
@@ -80,31 +80,34 @@
Collection() {}
- // Lookups integer ID from set. If it doesn't exist, then adds it.
- // Set elements should be in strict order (and therefore unique).
- I FindId(const vector<T> &set) {
+ // Lookups integer ID from ordered multi-set. If it doesn't exist
+ // and 'insert' is true, then adds it. Otherwise returns -1.
+ I FindId(const vector<T> &set, bool insert = true) {
I node_id = kNoNodeId;
for (ssize_t i = set.size() - 1; i >= 0; --i) {
Node node(node_id, set[i]);
- node_id = node_table_.FindId(node);
+ node_id = node_table_.FindId(node, insert);
+ if (node_id == -1) break;
}
return node_id;
}
- // Finds set given integer ID. Returns true if ID corresponds
- // to set. Use iterators below to traverse result.
+ // Finds ordered (multi-)set given integer ID. Returns set iterator
+ // to traverse result.
SetIterator FindSet(I id) {
- if (id < 0 && id >= node_table_.Size()) {
+ if (id < 0 || id >= node_table_.Size()) {
return SetIterator(kNoNodeId, Node(kNoNodeId, T()), &node_table_);
} else {
return SetIterator(id, node_table_.FindEntry(id), &node_table_);
}
}
+ I Size() const { return node_table_.Size(); }
+
private:
static const I kNoNodeId;
static const size_t kPrime;
- static std::tr1::hash<T> hash_;
+ static std::hash<T> hash_;
NodeTable node_table_;
@@ -115,7 +118,7 @@
template <class I, class T> const size_t Collection<I, T>::kPrime = 7853;
-template <class I, class T> std::tr1::hash<T> Collection<I, T>::hash_;
+template <class I, class T> std::hash<T> Collection<I, T>::hash_;
} // namespace fst
diff --git a/src/include/fst/extensions/pdt/info.h b/src/include/fst/extensions/pdt/info.h
index ef9a860..55e76c4 100644
--- a/src/include/fst/extensions/pdt/info.h
+++ b/src/include/fst/extensions/pdt/info.h
@@ -24,7 +24,7 @@
#include <unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
-#include <tr1/unordered_set>
+#include <unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
#include <vector>
diff --git a/src/include/fst/extensions/pdt/paren.h b/src/include/fst/extensions/pdt/paren.h
index 7b9887f..a9d30c5 100644
--- a/src/include/fst/extensions/pdt/paren.h
+++ b/src/include/fst/extensions/pdt/paren.h
@@ -26,7 +26,7 @@
#include <unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
-#include <tr1/unordered_set>
+#include <unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
#include <set>
@@ -144,7 +144,8 @@
const vector<pair<Label, Label> > &parens, bool close)
: fst_(fst),
parens_(parens),
- close_(close) {
+ close_(close),
+ error_(false) {
for (Label i = 0; i < parens.size(); ++i) {
const pair<Label, Label> &p = parens[i];
paren_id_map_[p.first] = i;
@@ -155,12 +156,18 @@
StateId start = fst.Start();
if (start == kNoStateId)
return;
- DFSearch(start, start);
+ if (!DFSearch(start)) {
+ FSTERROR() << "PdtReachable: Underlying cyclicity not supported";
+ error_ = true;
+ }
} else {
FSTERROR() << "PdtParenReachable: open paren info not implemented";
+ error_ = true;
}
}
+ bool const Error() { return error_; }
+
// Given a state ID, returns an iterator over paren IDs
// for close (open) parens reachable from that state along balanced
// paths.
@@ -194,7 +201,7 @@
private:
// DFS that gathers paren and state set information.
// Bool returns false when cycle detected.
- bool DFSearch(StateId s, StateId start);
+ bool DFSearch(StateId s);
// Unions state sets together gathered by the DFS.
void ComputeStateSet(StateId s);
@@ -212,12 +219,13 @@
vector<char> state_color_; // DFS state
mutable Collection<ssize_t, StateId> state_sets_; // Reachable states -> ID
StateSetMap set_map_; // ID -> Reachable states
+ bool error_;
DISALLOW_COPY_AND_ASSIGN(PdtParenReachable);
};
// DFS that gathers paren and state set information.
template <class A>
-bool PdtParenReachable<A>::DFSearch(StateId s, StateId start) {
+bool PdtParenReachable<A>::DFSearch(StateId s) {
if (s >= state_color_.size())
state_color_.resize(s + 1, kDfsWhite);
@@ -239,7 +247,8 @@
if (pit != paren_id_map_.end()) { // paren?
Label paren_id = pit->second;
if (arc.ilabel == parens_[paren_id].first) { // open paren
- DFSearch(arc.nextstate, arc.nextstate);
+ if (!DFSearch(arc.nextstate))
+ return false;
for (SetIterator set_iter = FindStates(paren_id, arc.nextstate);
!set_iter.Done(); set_iter.Next()) {
for (ParenArcIterator paren_arc_iter =
@@ -247,15 +256,14 @@
!paren_arc_iter.Done();
paren_arc_iter.Next()) {
const A &cparc = paren_arc_iter.Value();
- DFSearch(cparc.nextstate, start);
+ if (!DFSearch(cparc.nextstate))
+ return false;
}
}
}
} else { // non-paren
- if(!DFSearch(arc.nextstate, start)) {
- FSTERROR() << "PdtReachable: Underlying cyclicity not supported";
- return true;
- }
+ if(!DFSearch(arc.nextstate))
+ return false;
}
}
ComputeStateSet(s);
diff --git a/src/include/fst/extensions/pdt/shortest-path.h b/src/include/fst/extensions/pdt/shortest-path.h
index e90471b..85f94b8 100644
--- a/src/include/fst/extensions/pdt/shortest-path.h
+++ b/src/include/fst/extensions/pdt/shortest-path.h
@@ -28,7 +28,7 @@
#include <unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
-#include <tr1/unordered_set>
+#include <unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
#include <stack>
@@ -387,7 +387,6 @@
typedef typename SpData::SearchState SearchState;
typedef typename SpData::ParenSpec ParenSpec;
- typedef typename PdtParenReachable<Arc>::SetIterator StateSetIterator;
typedef typename PdtBalanceData<Arc>::SetIterator CloseSourceIterator;
PdtShortestPath(const Fst<Arc> &ifst,
@@ -403,7 +402,7 @@
if ((Weight::Properties() & (kPath | kRightSemiring))
!= (kPath | kRightSemiring)) {
- FSTERROR() << "SingleShortestPath: Weight needs to have the path"
+ FSTERROR() << "PdtShortestPath: Weight needs to have the path"
<< " property and be right distributive: " << Weight::Type();
error_ = true;
}
@@ -440,6 +439,7 @@
static const Arc kNoArc;
static const uint8 kEnqueued;
static const uint8 kExpanded;
+ static const uint8 kFinished;
const uint8 kFinal;
public:
@@ -543,6 +543,7 @@
ProcArcs(s);
sp_data_.SetFlags(s, kExpanded, kExpanded);
}
+ sp_data_.SetFlags(q, kFinished, kFinished);
balance_data_.FinishInsert(start);
sp_data_.GC(start);
}
@@ -607,7 +608,11 @@
Queue *state_queue = state_queue_;
GetDistance(d.start);
state_queue_ = state_queue;
+ } else if (!(sp_data_.Flags(d) & kFinished)) {
+ FSTERROR() << "PdtShortestPath: open parenthesis recursion: not bounded stack";
+ error_ = true;
}
+
for (CloseSourceIterator set_iter =
balance_data_.Find(paren_id, arc.nextstate);
!set_iter.Done(); set_iter.Next()) {
@@ -765,6 +770,9 @@
const uint8 PdtShortestPath<Arc, Queue>::kExpanded = 0x20;
template<class Arc, class Queue>
+const uint8 PdtShortestPath<Arc, Queue>::kFinished = 0x40;
+
+template<class Arc, class Queue>
void ShortestPath(const Fst<Arc> &ifst,
const vector<pair<typename Arc::Label,
typename Arc::Label> > &parens,
diff --git a/src/include/fst/flags.h b/src/include/fst/flags.h
index ec3d301..b3bb66c 100644
--- a/src/include/fst/flags.h
+++ b/src/include/fst/flags.h
@@ -22,6 +22,8 @@
#include <iostream>
#include <map>
+#include <set>
+#include <sstream>
#include <string>
#include <fst/types.h>
@@ -42,7 +44,7 @@
//
// DECLARE_int32(length);
//
-// SetFlags() can be used to set flags from the command line
+// SET_FLAGS() can be used to set flags from the command line
// using, for example, '--length=2'.
//
// ShowUsage() can be used to print out command and flag usage.
@@ -56,12 +58,18 @@
template <typename T>
struct FlagDescription {
- FlagDescription(T *addr, const char *doc, const char *type, const T val)
- : address(addr), doc_string(doc), type_name(type), default_value(val) {}
+ FlagDescription(T *addr, const char *doc, const char *type,
+ const char *file, const T val)
+ : address(addr),
+ doc_string(doc),
+ type_name(type),
+ file_name(file),
+ default_value(val) {}
T *address;
const char *doc_string;
const char *type_name;
+ const char *file_name;
const T default_value;
};
@@ -118,8 +126,7 @@
}
bool SetFlag(const string &arg, const string &val) const {
- for (typename std::map< string,
- FlagDescription<T> >::const_iterator it =
+ for (typename std::map< string, FlagDescription<T> >::const_iterator it =
flag_table_.begin();
it != flag_table_.end();
++it) {
@@ -131,19 +138,7 @@
return false;
}
- void ShowDefault(bool default_value) const {
- std::cout << ", default = ";
- std::cout << (default_value ? "true" : "false");
- }
- void ShowDefault(const string &default_value) const {
- std::cout << ", default = ";
- std::cout << "\"" << default_value << "\"";
- }
- template<typename V> void ShowDefault(const V& default_value) const {
- std::cout << ", default = ";
- std::cout << default_value;
- }
- void ShowUsage() const {
+ void GetUsage(std::set< std::pair<string, string> > *usage_set) const {
for (typename std::map< string,
FlagDescription<T> >::const_iterator it =
flag_table_.begin();
@@ -151,10 +146,13 @@
++it) {
const string &name = it->first;
const FlagDescription<T> &desc = it->second;
- std::cout << " --" << name
- << ": type = " << desc.type_name;
- ShowDefault(desc.default_value);
- std::cout << "\n " << desc.doc_string << "\n";
+ string usage = " --" + name;
+ usage += ": type = ";
+ usage += desc.type_name;
+ usage += ", default = ";
+ usage += GetDefault(desc.default_value) + "\n ";
+ usage += desc.doc_string;
+ usage_set->insert(make_pair(desc.file_name, usage));
}
}
@@ -163,11 +161,26 @@
register_lock_ = new fst::Mutex;
register_ = new FlagRegister<T>;
}
+
+ std::map< string, FlagDescription<T> > flag_table_;
+
+ string GetDefault(bool default_value) const {
+ return default_value ? "true" : "false";
+ }
+
+ string GetDefault(const string &default_value) const {
+ return "\"" + default_value + "\"";
+ }
+
+ template<typename V> string GetDefault(const V& default_value) const {
+ std::ostringstream strm;
+ strm << default_value;
+ return strm.str();
+ }
+
static fst::FstOnceType register_init_; // ensures only called once
static fst::Mutex* register_lock_; // multithreading lock
static FlagRegister<T> *register_;
-
- std::map< string, FlagDescription<T> > flag_table_;
};
template <class T>
@@ -199,6 +212,7 @@
name ## _flags_registerer(#name, FlagDescription<type>(&FLAGS_ ## name, \
doc, \
#type, \
+ __FILE__, \
value))
#define DEFINE_bool(name, value, doc) DEFINE_VAR(bool, name, value, doc)
@@ -212,13 +226,17 @@
// Temporary directory
DECLARE_string(tmpdir);
-void SetFlags(const char *usage, int *argc, char ***argv, bool remove_flags);
+void SetFlags(const char *usage, int *argc, char ***argv, bool remove_flags,
+ const char *src = "");
+
+#define SET_FLAGS(usage, argc, argv, rmflags) \
+SetFlags(usage, argc, argv, rmflags, __FILE__)
// Deprecated - for backward compatibility
inline void InitFst(const char *usage, int *argc, char ***argv, bool rmflags) {
return SetFlags(usage, argc, argv, rmflags);
}
-void ShowUsage();
+void ShowUsage(bool long_usage = true);
#endif // FST_LIB_FLAGS_H__
diff --git a/src/include/fst/float-weight.h b/src/include/fst/float-weight.h
index 530cbdd..eb22638 100644
--- a/src/include/fst/float-weight.h
+++ b/src/include/fst/float-weight.h
@@ -37,20 +37,23 @@
template <class T>
class FloatLimits {
public:
- static const T kPosInfinity;
- static const T kNegInfinity;
- static const T kNumberBad;
+ static const T PosInfinity() {
+ static const T pos_infinity = numeric_limits<T>::infinity();
+ return pos_infinity;
+ }
+
+ static const T NegInfinity() {
+ static const T neg_infinity = -PosInfinity();
+ return neg_infinity;
+ }
+
+ static const T NumberBad() {
+ static const T number_bad = numeric_limits<T>::quiet_NaN();
+ return number_bad;
+ }
+
};
-template <class T>
-const T FloatLimits<T>::kPosInfinity = numeric_limits<T>::infinity();
-
-template <class T>
-const T FloatLimits<T>::kNegInfinity = -FloatLimits<T>::kPosInfinity;
-
-template <class T>
-const T FloatLimits<T>::kNumberBad = numeric_limits<T>::quiet_NaN();
-
// weight class to be templated on floating-points types
template <class T = float>
class FloatWeightTpl {
@@ -151,9 +154,9 @@
template <class T>
inline ostream &operator<<(ostream &strm, const FloatWeightTpl<T> &w) {
- if (w.Value() == FloatLimits<T>::kPosInfinity)
+ if (w.Value() == FloatLimits<T>::PosInfinity())
return strm << "Infinity";
- else if (w.Value() == FloatLimits<T>::kNegInfinity)
+ else if (w.Value() == FloatLimits<T>::NegInfinity())
return strm << "-Infinity";
else if (w.Value() != w.Value()) // Fails for NaN
return strm << "BadNumber";
@@ -166,9 +169,9 @@
string s;
strm >> s;
if (s == "Infinity") {
- w = FloatWeightTpl<T>(FloatLimits<T>::kPosInfinity);
+ w = FloatWeightTpl<T>(FloatLimits<T>::PosInfinity());
} else if (s == "-Infinity") {
- w = FloatWeightTpl<T>(FloatLimits<T>::kNegInfinity);
+ w = FloatWeightTpl<T>(FloatLimits<T>::NegInfinity());
} else {
char *p;
T f = strtod(s.c_str(), &p);
@@ -196,13 +199,13 @@
TropicalWeightTpl(const TropicalWeightTpl<T> &w) : FloatWeightTpl<T>(w) {}
static const TropicalWeightTpl<T> Zero() {
- return TropicalWeightTpl<T>(FloatLimits<T>::kPosInfinity); }
+ return TropicalWeightTpl<T>(FloatLimits<T>::PosInfinity()); }
static const TropicalWeightTpl<T> One() {
return TropicalWeightTpl<T>(0.0F); }
static const TropicalWeightTpl<T> NoWeight() {
- return TropicalWeightTpl<T>(FloatLimits<T>::kNumberBad); }
+ return TropicalWeightTpl<T>(FloatLimits<T>::NumberBad()); }
static const string &Type() {
static const string type = "tropical" +
@@ -212,12 +215,12 @@
bool Member() const {
// First part fails for IEEE NaN
- return Value() == Value() && Value() != FloatLimits<T>::kNegInfinity;
+ return Value() == Value() && Value() != FloatLimits<T>::NegInfinity();
}
TropicalWeightTpl<T> Quantize(float delta = kDelta) const {
- if (Value() == FloatLimits<T>::kNegInfinity ||
- Value() == FloatLimits<T>::kPosInfinity ||
+ if (Value() == FloatLimits<T>::NegInfinity() ||
+ Value() == FloatLimits<T>::PosInfinity() ||
Value() != Value())
return *this;
else
@@ -259,9 +262,9 @@
if (!w1.Member() || !w2.Member())
return TropicalWeightTpl<T>::NoWeight();
T f1 = w1.Value(), f2 = w2.Value();
- if (f1 == FloatLimits<T>::kPosInfinity)
+ if (f1 == FloatLimits<T>::PosInfinity())
return w1;
- else if (f2 == FloatLimits<T>::kPosInfinity)
+ else if (f2 == FloatLimits<T>::PosInfinity())
return w2;
else
return TropicalWeightTpl<T>(f1 + f2);
@@ -284,10 +287,10 @@
if (!w1.Member() || !w2.Member())
return TropicalWeightTpl<T>::NoWeight();
T f1 = w1.Value(), f2 = w2.Value();
- if (f2 == FloatLimits<T>::kPosInfinity)
- return FloatLimits<T>::kNumberBad;
- else if (f1 == FloatLimits<T>::kPosInfinity)
- return FloatLimits<T>::kPosInfinity;
+ if (f2 == FloatLimits<T>::PosInfinity())
+ return FloatLimits<T>::NumberBad();
+ else if (f1 == FloatLimits<T>::PosInfinity())
+ return FloatLimits<T>::PosInfinity();
else
return TropicalWeightTpl<T>(f1 - f2);
}
@@ -320,7 +323,7 @@
LogWeightTpl(const LogWeightTpl<T> &w) : FloatWeightTpl<T>(w) {}
static const LogWeightTpl<T> Zero() {
- return LogWeightTpl<T>(FloatLimits<T>::kPosInfinity);
+ return LogWeightTpl<T>(FloatLimits<T>::PosInfinity());
}
static const LogWeightTpl<T> One() {
@@ -328,7 +331,7 @@
}
static const LogWeightTpl<T> NoWeight() {
- return LogWeightTpl<T>(FloatLimits<T>::kNumberBad); }
+ return LogWeightTpl<T>(FloatLimits<T>::NumberBad()); }
static const string &Type() {
static const string type = "log" + FloatWeightTpl<T>::GetPrecisionString();
@@ -337,12 +340,12 @@
bool Member() const {
// First part fails for IEEE NaN
- return Value() == Value() && Value() != FloatLimits<T>::kNegInfinity;
+ return Value() == Value() && Value() != FloatLimits<T>::NegInfinity();
}
LogWeightTpl<T> Quantize(float delta = kDelta) const {
- if (Value() == FloatLimits<T>::kNegInfinity ||
- Value() == FloatLimits<T>::kPosInfinity ||
+ if (Value() == FloatLimits<T>::NegInfinity() ||
+ Value() == FloatLimits<T>::PosInfinity() ||
Value() != Value())
return *this;
else
@@ -368,9 +371,9 @@
inline LogWeightTpl<T> Plus(const LogWeightTpl<T> &w1,
const LogWeightTpl<T> &w2) {
T f1 = w1.Value(), f2 = w2.Value();
- if (f1 == FloatLimits<T>::kPosInfinity)
+ if (f1 == FloatLimits<T>::PosInfinity())
return w2;
- else if (f2 == FloatLimits<T>::kPosInfinity)
+ else if (f2 == FloatLimits<T>::PosInfinity())
return w1;
else if (f1 > f2)
return LogWeightTpl<T>(f2 - LogExp(f1 - f2));
@@ -394,9 +397,9 @@
if (!w1.Member() || !w2.Member())
return LogWeightTpl<T>::NoWeight();
T f1 = w1.Value(), f2 = w2.Value();
- if (f1 == FloatLimits<T>::kPosInfinity)
+ if (f1 == FloatLimits<T>::PosInfinity())
return w1;
- else if (f2 == FloatLimits<T>::kPosInfinity)
+ else if (f2 == FloatLimits<T>::PosInfinity())
return w2;
else
return LogWeightTpl<T>(f1 + f2);
@@ -419,10 +422,10 @@
if (!w1.Member() || !w2.Member())
return LogWeightTpl<T>::NoWeight();
T f1 = w1.Value(), f2 = w2.Value();
- if (f2 == FloatLimits<T>::kPosInfinity)
- return FloatLimits<T>::kNumberBad;
- else if (f1 == FloatLimits<T>::kPosInfinity)
- return FloatLimits<T>::kPosInfinity;
+ if (f2 == FloatLimits<T>::PosInfinity())
+ return FloatLimits<T>::NumberBad();
+ else if (f1 == FloatLimits<T>::PosInfinity())
+ return FloatLimits<T>::PosInfinity();
else
return LogWeightTpl<T>(f1 - f2);
}
@@ -454,15 +457,15 @@
MinMaxWeightTpl(const MinMaxWeightTpl<T> &w) : FloatWeightTpl<T>(w) {}
static const MinMaxWeightTpl<T> Zero() {
- return MinMaxWeightTpl<T>(FloatLimits<T>::kPosInfinity);
+ return MinMaxWeightTpl<T>(FloatLimits<T>::PosInfinity());
}
static const MinMaxWeightTpl<T> One() {
- return MinMaxWeightTpl<T>(FloatLimits<T>::kNegInfinity);
+ return MinMaxWeightTpl<T>(FloatLimits<T>::NegInfinity());
}
static const MinMaxWeightTpl<T> NoWeight() {
- return MinMaxWeightTpl<T>(FloatLimits<T>::kNumberBad); }
+ return MinMaxWeightTpl<T>(FloatLimits<T>::NumberBad()); }
static const string &Type() {
static const string type = "minmax" +
@@ -477,8 +480,8 @@
MinMaxWeightTpl<T> Quantize(float delta = kDelta) const {
// If one of infinities, or a NaN
- if (Value() == FloatLimits<T>::kNegInfinity ||
- Value() == FloatLimits<T>::kPosInfinity ||
+ if (Value() == FloatLimits<T>::NegInfinity() ||
+ Value() == FloatLimits<T>::PosInfinity() ||
Value() != Value())
return *this;
else
@@ -541,7 +544,7 @@
if (!w1.Member() || !w2.Member())
return MinMaxWeightTpl<T>::NoWeight();
// min(w1, x) = w2, w1 >= w2 => min(w1, x) = w2, x = w2
- return w1.Value() >= w2.Value() ? w1 : FloatLimits<T>::kNumberBad;
+ return w1.Value() >= w2.Value() ? w1 : FloatLimits<T>::NumberBad();
}
inline MinMaxWeightTpl<float> Divide(const MinMaxWeightTpl<float> &w1,
diff --git a/src/include/fst/fst.h b/src/include/fst/fst.h
index 9c4d0db..dd11e4f 100644
--- a/src/include/fst/fst.h
+++ b/src/include/fst/fst.h
@@ -36,6 +36,7 @@
#include <fst/register.h>
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/symbol-table.h>
#include <fst/util.h>
@@ -229,7 +230,7 @@
}
return Read(strm, FstReadOptions(filename));
} else {
- return Read(std::cin, FstReadOptions("standard input"));
+ return Read(cin, FstReadOptions("standard input"));
}
}
@@ -267,7 +268,6 @@
virtual MatcherBase<A> *InitMatcher(MatchType match_type) const;
protected:
-
bool WriteFile(const string &filename) const {
if (!filename.empty()) {
ofstream strm(filename.c_str(), ofstream::out | ofstream::binary);
@@ -277,7 +277,7 @@
}
return Write(strm, FstWriteOptions(filename));
} else {
- return Write(std::cout, FstWriteOptions("standard output"));
+ return Write(cout, FstWriteOptions("standard output"));
}
}
};
@@ -706,12 +706,13 @@
// This method is used in the cross-type serialization methods Fst::WriteFst.
static void WriteFstHeader(const Fst<A> &fst, ostream &strm,
const FstWriteOptions& opts, int version,
- const string &type, FstHeader *hdr) {
+ const string &type, uint64 properties,
+ FstHeader *hdr) {
if (opts.write_header) {
hdr->SetFstType(type);
hdr->SetArcType(A::Type());
hdr->SetVersion(version);
- hdr->SetProperties(fst.Properties(kFstProperties, false));
+ hdr->SetProperties(properties);
int32 file_flags = 0;
if (fst.InputSymbols() && opts.write_isymbols)
file_flags |= FstHeader::HAS_ISYMBOLS;
@@ -737,14 +738,14 @@
// returns true on success, false on failure.
static bool UpdateFstHeader(const Fst<A> &fst, ostream &strm,
const FstWriteOptions& opts, int version,
- const string &type, FstHeader *hdr,
- size_t header_offset) {
+ const string &type, uint64 properties,
+ FstHeader *hdr, size_t header_offset) {
strm.seekp(header_offset);
if (!strm) {
LOG(ERROR) << "Fst::UpdateFstHeader: write failed: " << opts.source;
return false;
}
- WriteFstHeader(fst, strm, opts, version, type, hdr);
+ WriteFstHeader(fst, strm, opts, version, type, properties, hdr);
if (!strm) {
LOG(ERROR) << "Fst::UpdateFstHeader: write failed: " << opts.source;
return false;
diff --git a/src/include/fst/fstlib.h b/src/include/fst/fstlib.h
index c05c775..de5976d 100644
--- a/src/include/fst/fstlib.h
+++ b/src/include/fst/fstlib.h
@@ -142,6 +142,8 @@
#include <fst/state-reachable.h>
#include <iostream>
#include <fstream>
+#include <sstream>
+#include <fst/string.h>
#include <fst/symbol-table.h>
#include <fst/symbol-table-ops.h>
#include <fst/test-properties.h>
diff --git a/src/include/fst/icu.h b/src/include/fst/icu.h
index 6b74c2e..3947716 100644
--- a/src/include/fst/icu.h
+++ b/src/include/fst/icu.h
@@ -13,88 +13,101 @@
// limitations under the License.
//
// Copyright 2005-2010 Google, Inc.
-// Author: roubert@google.com (Fredrik Roubert)
-
-// Wrapper class for UErrorCode, with conversion operators for direct use in
-// ICU C and C++ APIs.
+// Author: sorenj@google.com (Jeffrey Sorensen)
+// roubert@google.com (Fredrik Roubert)
//
-// Features:
-// - The constructor initializes the internal UErrorCode to U_ZERO_ERROR,
-// removing one common source of errors.
-// - Same use in C APIs taking a UErrorCode* (pointer) and C++ taking
-// UErrorCode& (reference), via conversion operators.
-// - Automatic checking for success when it goes out of scope. On failure,
-// the destructor will FSTERROR() an error message.
-//
-// Most of ICU will handle errors gracefully and provide sensible fallbacks.
-// Using IcuErrorCode, it is therefore possible to write very compact code
-// that does sensible things on failure and provides logging for debugging.
-//
-// Example:
-//
-// IcuErrorCode icuerrorcode;
-// return collator.compareUTF8(a, b, icuerrorcode) == UCOL_EQUAL;
+// This library implements an unrestricted Thompson/Pike UTF-8 parser and
+// serializer. UTF-8 is a restricted subset of this byte stream encoding. See
+// http://en.wikipedia.org/wiki/UTF-8 for a good description of the encoding
+// details.
#ifndef FST_LIB_ICU_H_
#define FST_LIB_ICU_H_
-#include <unicode/errorcode.h>
-#include <unicode/unistr.h>
-#include <unicode/ustring.h>
-#include <unicode/utf8.h>
-
-class IcuErrorCode : public icu::ErrorCode {
- public:
- IcuErrorCode() {}
- virtual ~IcuErrorCode() { if (isFailure()) handleFailure(); }
-
- // Redefine 'errorName()' in order to be compatible with ICU version 4.2
- const char* errorName() const {
- return u_errorName(errorCode);
- }
-
- protected:
- virtual void handleFailure() const {
- FSTERROR() << errorName();
-}
-
- private:
- DISALLOW_COPY_AND_ASSIGN(IcuErrorCode);
-};
+#include <iostream>
+#include <fstream>
+#include <sstream>
namespace fst {
template <class Label>
bool UTF8StringToLabels(const string &str, vector<Label> *labels) {
- const char *c_str = str.c_str();
- int32_t length = str.size();
- UChar32 c;
- for (int32_t i = 0; i < length; /* no update */) {
- U8_NEXT(c_str, i, length, c);
- if (c < 0) {
- LOG(ERROR) << "UTF8StringToLabels: Invalid character found: " << c;
- return false;
+ const char *data = str.data();
+ size_t length = str.size();
+ for (int i = 0; i < length; /* no update */) {
+ int c = data[i++] & 0xff;
+ if ((c & 0x80) == 0) {
+ labels->push_back(c);
+ } else {
+ if ((c & 0xc0) == 0x80) {
+ LOG(ERROR) << "UTF8StringToLabels: continuation byte as lead byte";
+ return false;
+ }
+ int count = (c >= 0xc0) + (c >= 0xe0) + (c >= 0xf0) + (c >= 0xf8) +
+ (c >= 0xfc);
+ int code = c & ((1 << (6 - count)) - 1);
+ while (count != 0) {
+ if (i == length) {
+ LOG(ERROR) << "UTF8StringToLabels: truncated utf-8 byte sequence";
+ return false;
+ }
+ char cb = data[i++];
+ if ((cb & 0xc0) != 0x80) {
+ LOG(ERROR) << "UTF8StringToLabels: missing/invalid continuation byte";
+ return false;
+ }
+ code = (code << 6) | (cb & 0x3f);
+ count--;
+ }
+ if (code < 0) {
+ // This should not be able to happen.
+ LOG(ERROR) << "UTF8StringToLabels: Invalid character found: " << c;
+ return false;
+ }
+ labels->push_back(code);
}
- labels->push_back(c);
}
return true;
}
template <class Label>
bool LabelsToUTF8String(const vector<Label> &labels, string *str) {
- icu::UnicodeString u_str;
- char c_str[5];
+ ostringstream ostr;
for (size_t i = 0; i < labels.size(); ++i) {
- u_str.setTo(labels[i]);
- IcuErrorCode error;
- u_strToUTF8(c_str, 5, NULL, u_str.getTerminatedBuffer(), -1, error);
- if (error.isFailure()) {
- LOG(ERROR) << "LabelsToUTF8String: Bad encoding: "
- << error.errorName();
+ int32_t code = labels[i];
+ if (code < 0) {
+ LOG(ERROR) << "LabelsToUTF8String: Invalid character found: " << code;
return false;
+ } else if (code < 0x80) {
+ ostr << static_cast<char>(code);
+ } else if (code < 0x800) {
+ ostr << static_cast<char>((code >> 6) | 0xc0);
+ ostr << static_cast<char>((code & 0x3f) | 0x80);
+ } else if (code < 0x10000) {
+ ostr << static_cast<char>((code >> 12) | 0xe0);
+ ostr << static_cast<char>(((code >> 6) & 0x3f) | 0x80);
+ ostr << static_cast<char>((code & 0x3f) | 0x80);
+ } else if (code < 0x200000) {
+ ostr << static_cast<char>((code >> 18) | 0xf0);
+ ostr << static_cast<char>(((code >> 12) & 0x3f) | 0x80);
+ ostr << static_cast<char>(((code >> 6) & 0x3f) | 0x80);
+ ostr << static_cast<char>((code & 0x3f) | 0x80);
+ } else if (code < 0x4000000) {
+ ostr << static_cast<char>((code >> 24) | 0xf8);
+ ostr << static_cast<char>(((code >> 18) & 0x3f) | 0x80);
+ ostr << static_cast<char>(((code >> 12) & 0x3f) | 0x80);
+ ostr << static_cast<char>(((code >> 6) & 0x3f) | 0x80);
+ ostr << static_cast<char>((code & 0x3f) | 0x80);
+ } else {
+ ostr << static_cast<char>((code >> 30) | 0xfc);
+ ostr << static_cast<char>(((code >> 24) & 0x3f) | 0x80);
+ ostr << static_cast<char>(((code >> 18) & 0x3f) | 0x80);
+ ostr << static_cast<char>(((code >> 12) & 0x3f) | 0x80);
+ ostr << static_cast<char>(((code >> 6) & 0x3f) | 0x80);
+ ostr << static_cast<char>((code & 0x3f) | 0x80);
}
- *str += c_str;
}
+ *str = ostr.str();
return true;
}
diff --git a/src/include/fst/lock.h b/src/include/fst/lock.h
index 3adf7df..329015d 100644
--- a/src/include/fst/lock.h
+++ b/src/include/fst/lock.h
@@ -16,6 +16,9 @@
//
// \file
// Google-compatibility locking declarations and inline definitions
+//
+// Classes and functions here are no-ops (by design); proper locking requires
+// actual implementation.
#ifndef FST_LIB_LOCK_H__
#define FST_LIB_LOCK_H__
@@ -61,6 +64,14 @@
DISALLOW_COPY_AND_ASSIGN(MutexLock);
};
+class ReaderMutexLock {
+ public:
+ ReaderMutexLock(Mutex *) {}
+
+ private:
+ DISALLOW_COPY_AND_ASSIGN(ReaderMutexLock);
+};
+
// Reference counting - single-thread implementation
class RefCounter {
public:
diff --git a/src/include/fst/lookahead-matcher.h b/src/include/fst/lookahead-matcher.h
index 10d9c01..f927d65 100644
--- a/src/include/fst/lookahead-matcher.h
+++ b/src/include/fst/lookahead-matcher.h
@@ -96,35 +96,35 @@
// LOOK-AHEAD FLAGS (see also kMatcherFlags in matcher.h):
//
// Matcher is a lookahead matcher when 'match_type' is MATCH_INPUT.
-const uint32 kInputLookAheadMatcher = 0x00000001;
+const uint32 kInputLookAheadMatcher = 0x00000010;
// Matcher is a lookahead matcher when 'match_type' is MATCH_OUTPUT.
-const uint32 kOutputLookAheadMatcher = 0x00000002;
+const uint32 kOutputLookAheadMatcher = 0x00000020;
// A non-trivial implementation of LookAheadWeight() method defined and
// should be used?
-const uint32 kLookAheadWeight = 0x00000004;
+const uint32 kLookAheadWeight = 0x00000040;
// A non-trivial implementation of LookAheadPrefix() method defined and
// should be used?
-const uint32 kLookAheadPrefix = 0x00000008;
+const uint32 kLookAheadPrefix = 0x00000080;
// Look-ahead of matcher FST non-epsilon arcs?
-const uint32 kLookAheadNonEpsilons = 0x00000010;
+const uint32 kLookAheadNonEpsilons = 0x00000100;
// Look-ahead of matcher FST epsilon arcs?
-const uint32 kLookAheadEpsilons = 0x00000020;
+const uint32 kLookAheadEpsilons = 0x00000200;
// Ignore epsilon paths for the lookahead prefix? Note this gives
// correct results in composition only with an appropriate composition
// filter since it depends on the filter blocking the ignored paths.
-const uint32 kLookAheadNonEpsilonPrefix = 0x00000040;
+const uint32 kLookAheadNonEpsilonPrefix = 0x00000400;
// For LabelLookAheadMatcher, save relabeling data to file
-const uint32 kLookAheadKeepRelabelData = 0x00000080;
+const uint32 kLookAheadKeepRelabelData = 0x00000800;
// Flags used for lookahead matchers.
-const uint32 kLookAheadFlags = 0x000000ff;
+const uint32 kLookAheadFlags = 0x00000ff0;
// LookAhead Matcher interface, templated on the Arc definition; used
// for lookahead matcher specializations that are returned by the
@@ -601,10 +601,12 @@
bool reach_arc = label_reachable_->Reach(&aiter, 0,
internal::NumArcs(*lfst_, s),
reach_input, compute_weight);
+ Weight lfinal = internal::Final(*lfst_, s);
+ bool reach_final = lfinal != Weight::Zero() && label_reachable_->ReachFinal();
if (reach_arc) {
ssize_t begin = label_reachable_->ReachBegin();
ssize_t end = label_reachable_->ReachEnd();
- if (compute_prefix && end - begin == 1) {
+ if (compute_prefix && end - begin == 1 && !reach_final) {
aiter.Seek(begin);
SetLookAheadPrefix(aiter.Value());
compute_weight = false;
@@ -612,9 +614,6 @@
SetLookAheadWeight(label_reachable_->ReachWeight());
}
}
- Weight lfinal = internal::Final(*lfst_, s);
- bool reach_final = lfinal != Weight::Zero() &&
- label_reachable_->ReachFinal();
if (reach_final && compute_weight)
SetLookAheadWeight(reach_arc ?
Plus(LookAheadWeight(), lfinal) : lfinal);
diff --git a/src/include/fst/matcher.h b/src/include/fst/matcher.h
index a89325b..5ab3d26 100644
--- a/src/include/fst/matcher.h
+++ b/src/include/fst/matcher.h
@@ -83,8 +83,17 @@
// uint64 Properties(uint64 props) const;
// };
+//
+// MATCHER FLAGS (see also kLookAheadFlags in lookahead-matcher.h)
+//
+// Matcher prefers being used as the matching side in composition.
+const uint32 kPreferMatch = 0x00000001;
+
+// Matcher needs to be used as the matching side in composition.
+const uint32 kRequireMatch = 0x00000002;
+
// Flags used for basic matchers (see also lookahead.h).
-const uint32 kMatcherFlags = 0x00000000;
+const uint32 kMatcherFlags = kPreferMatch | kRequireMatch;
// Matcher interface, templated on the Arc definition; used
// for matcher specializations that are returned by the
@@ -452,6 +461,12 @@
virtual uint64 Properties(uint64 props) const;
+ virtual uint32 Flags() const {
+ if (rho_label_ == kNoLabel || match_type_ == MATCH_NONE)
+ return matcher_->Flags();
+ return matcher_->Flags() | kRequireMatch;
+ }
+
private:
virtual void SetState_(StateId s) { SetState(s); }
virtual bool Find_(Label label) { return Find(label); }
@@ -631,6 +646,15 @@
virtual uint64 Properties(uint64 props) const;
+ virtual uint32 Flags() const {
+ if (sigma_label_ == kNoLabel || match_type_ == MATCH_NONE)
+ return matcher_->Flags();
+ // kRequireMatch temporarily disabled until issues
+ // in //speech/gaudi/annotation/util/denorm are resolved.
+ // return matcher_->Flags() | kRequireMatch;
+ return matcher_->Flags();
+ }
+
private:
virtual void SetState_(StateId s) { SetState(s); }
virtual bool Find_(Label label) { return Find(label); }
@@ -722,11 +746,6 @@
match_type_ = MATCH_NONE;
error_ = true;
}
- if (phi_label == 0) {
- FSTERROR() << "PhiMatcher: 0 cannot be used as phi_label";
- phi_label_ = kNoLabel;
- error_ = true;
- }
if (rewrite_mode == MATCHER_REWRITE_AUTO)
rewrite_both_ = fst.Properties(kAcceptor, true);
@@ -768,10 +787,15 @@
const Arc& Value() const {
if ((phi_match_ == kNoLabel) && (phi_weight_ == Weight::One())) {
return matcher_->Value();
+ } else if (phi_match_ == 0) { // Virtual epsilon loop
+ phi_arc_ = Arc(kNoLabel, 0, Weight::One(), state_);
+ if (match_type_ == MATCH_OUTPUT)
+ swap(phi_arc_.ilabel, phi_arc_.olabel);
+ return phi_arc_;
} else {
phi_arc_ = matcher_->Value();
phi_arc_.weight = Times(phi_weight_, phi_arc_.weight);
- if (phi_match_ != kNoLabel) {
+ if (phi_match_ != kNoLabel) { // Phi loop match
if (rewrite_both_) {
if (phi_arc_.ilabel == phi_label_)
phi_arc_.ilabel = phi_match_;
@@ -793,6 +817,12 @@
virtual uint64 Properties(uint64 props) const;
+ virtual uint32 Flags() const {
+ if (phi_label_ == kNoLabel || match_type_ == MATCH_NONE)
+ return matcher_->Flags();
+ return matcher_->Flags() | kRequireMatch;
+ }
+
private:
virtual void SetState_(StateId s) { SetState(s); }
virtual bool Find_(Label label) { return Find(label); }
@@ -818,19 +848,33 @@
template <class M> inline
bool PhiMatcher<M>::Find(Label match_label) {
- if (match_label == phi_label_ && phi_label_ != kNoLabel) {
- FSTERROR() << "PhiMatcher::Find: bad label (phi)";
+ if (match_label == phi_label_ && phi_label_ != kNoLabel && phi_label_ != 0) {
+ FSTERROR() << "PhiMatcher::Find: bad label (phi): " << phi_label_;
error_ = true;
return false;
}
matcher_->SetState(state_);
phi_match_ = kNoLabel;
phi_weight_ = Weight::One();
+ if (phi_label_ == 0) { // When 'phi_label_ == 0',
+ if (match_label == kNoLabel) // there are no more true epsilon arcs,
+ return false;
+ if (match_label == 0) { // but virtual eps loop need to be returned
+ if (!matcher_->Find(kNoLabel)) {
+ return matcher_->Find(0);
+ } else {
+ phi_match_ = 0;
+ return true;
+ }
+ }
+ }
if (!has_phi_ || match_label == 0 || match_label == kNoLabel)
return matcher_->Find(match_label);
StateId state = state_;
while (!matcher_->Find(match_label)) {
- if (!matcher_->Find(phi_label_))
+ // Look for phi transition (if phi_label_ == 0, we need to look
+ // for -1 to avoid getting the virtual self-loop)
+ if (!matcher_->Find(phi_label_ == 0 ? -1 : phi_label_))
return false;
if (phi_loop_ && matcher_->Value().nextstate == state) {
phi_match_ = match_label;
@@ -856,6 +900,10 @@
if (match_type_ == MATCH_NONE) {
return outprops;
} else if (match_type_ == MATCH_INPUT) {
+ if (phi_label_ == 0) {
+ outprops &= ~kEpsilons | ~kIEpsilons | ~kOEpsilons;
+ outprops |= kNoEpsilons | kNoIEpsilons;
+ }
if (rewrite_both_) {
return outprops & ~(kODeterministic | kNonODeterministic | kString |
kILabelSorted | kNotILabelSorted |
@@ -866,6 +914,10 @@
kOLabelSorted | kNotOLabelSorted);
}
} else if (match_type_ == MATCH_OUTPUT) {
+ if (phi_label_ == 0) {
+ outprops &= ~kEpsilons | ~kIEpsilons | ~kOEpsilons;
+ outprops |= kNoEpsilons | kNoOEpsilons;
+ }
if (rewrite_both_) {
return outprops & ~(kIDeterministic | kNonIDeterministic | kString |
kILabelSorted | kNotILabelSorted |
diff --git a/src/include/fst/mutable-fst.h b/src/include/fst/mutable-fst.h
index 9afcab3..09eb237 100644
--- a/src/include/fst/mutable-fst.h
+++ b/src/include/fst/mutable-fst.h
@@ -128,7 +128,7 @@
}
return Read(strm, FstReadOptions(filename));
} else {
- return Read(std::cin, FstReadOptions("standard input"));
+ return Read(cin, FstReadOptions("standard input"));
}
} else { // Converts to 'convert_type' if not mutable.
Fst<A> *ifst = Fst<A>::Read(filename);
diff --git a/src/include/fst/queue.h b/src/include/fst/queue.h
index 707dffc..e31f087 100644
--- a/src/include/fst/queue.h
+++ b/src/include/fst/queue.h
@@ -451,7 +451,7 @@
while ((front_ <= back_) &&
(((*queue_)[front_] && (*queue_)[front_]->Empty())
|| (((*queue_)[front_] == 0) &&
- ((front_ > trivial_queue_.size())
+ ((front_ >= trivial_queue_.size())
|| (trivial_queue_[front_] == kNoStateId)))))
++front_;
if ((*queue_)[front_])
@@ -493,7 +493,7 @@
else if ((*queue_)[front_])
return (*queue_)[front_]->Empty();
else
- return (front_ > trivial_queue_.size())
+ return (front_ >= trivial_queue_.size())
|| (trivial_queue_[front_] == kNoStateId);
}
diff --git a/src/include/fst/register.h b/src/include/fst/register.h
index 55651cd..ea3f4d8 100644
--- a/src/include/fst/register.h
+++ b/src/include/fst/register.h
@@ -28,6 +28,7 @@
#include <fst/compat.h>
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/util.h>
#include <fst/generic-register.h>
diff --git a/src/include/fst/rmepsilon.h b/src/include/fst/rmepsilon.h
index ee9753e..32e64de 100644
--- a/src/include/fst/rmepsilon.h
+++ b/src/include/fst/rmepsilon.h
@@ -110,7 +110,6 @@
class ElementKey {
public:
size_t operator()(const Element& e) const {
- return static_cast<size_t>(e.nextstate);
return static_cast<size_t>(e.nextstate +
e.ilabel * kPrime0 +
e.olabel * kPrime1);
diff --git a/src/include/fst/script/compile-impl.h b/src/include/fst/script/compile-impl.h
index 4aab15b..1743452 100644
--- a/src/include/fst/script/compile-impl.h
+++ b/src/include/fst/script/compile-impl.h
@@ -31,6 +31,7 @@
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/fst.h>
#include <fst/util.h>
#include <fst/vector-fst.h>
diff --git a/src/include/fst/script/draw-impl.h b/src/include/fst/script/draw-impl.h
index e346649..893e258 100644
--- a/src/include/fst/script/draw-impl.h
+++ b/src/include/fst/script/draw-impl.h
@@ -139,9 +139,9 @@
EscapeChars(symbol, &nsymbol);
PrintString(nsymbol);
} else {
- ostringstream sid;
- sid << id;
- PrintString(sid.str());
+ string idstr;
+ Int64ToStr(id, &idstr);
+ PrintString(idstr);
}
}
diff --git a/src/include/fst/script/draw.h b/src/include/fst/script/draw.h
index 1611ad1..2b66373 100644
--- a/src/include/fst/script/draw.h
+++ b/src/include/fst/script/draw.h
@@ -22,6 +22,7 @@
#include <fst/script/draw-impl.h>
#include <iostream>
#include <fstream>
+#include <sstream>
namespace fst {
namespace script {
diff --git a/src/include/fst/script/fst-class.h b/src/include/fst/script/fst-class.h
index 3eacab4..a820c1c 100644
--- a/src/include/fst/script/fst-class.h
+++ b/src/include/fst/script/fst-class.h
@@ -24,6 +24,7 @@
#include <fst/vector-fst.h>
#include <iostream>
#include <fstream>
+#include <sstream>
// Classes to support "boxing" all existing types of FST arcs in a single
// FstClass which hides the arc types. This allows clients to load
@@ -52,6 +53,7 @@
virtual const SymbolTable *InputSymbols() const = 0;
virtual const SymbolTable *OutputSymbols() const = 0;
virtual void Write(const string& fname) const = 0;
+ virtual void Write(ostream &ostr, const FstWriteOptions &opts) const = 0;
virtual uint64 Properties(uint64 mask, bool test) const = 0;
virtual ~FstClassBase() { }
};
@@ -114,12 +116,18 @@
impl_->Write(fname);
}
+ virtual void Write(ostream &ostr, const FstWriteOptions &opts) const {
+ impl_->Write(ostr, opts);
+ }
+
virtual uint64 Properties(uint64 mask, bool test) const {
return impl_->Properties(mask, test);
}
virtual ~FstClassImpl() { delete impl_; }
+ Fst<Arc> *GetImpl() const { return impl_; }
+
Fst<Arc> *GetImpl() { return impl_; }
virtual FstClassImpl *Copy() {
@@ -154,13 +162,25 @@
}
}
+ FstClass() : impl_(NULL) {
+ }
+
template<class Arc>
- explicit FstClass(Fst<Arc> *fst) : impl_(new FstClassImpl<Arc>(fst)) { }
+ explicit FstClass(Fst<Arc> *fst) : impl_(new FstClassImpl<Arc>(fst)) {
+ }
explicit FstClass(const FstClass &other) : impl_(other.impl_->Copy()) { }
+ FstClass &operator=(const FstClass &other) {
+ delete impl_;
+ impl_ = other.impl_->Copy();
+ return *this;
+ }
+
static FstClass *Read(const string &fname);
+ static FstClass *Read(istream &istr, const string &source);
+
virtual const string &ArcType() const {
return impl_->ArcType();
}
@@ -185,6 +205,10 @@
impl_->Write(fname);
}
+ virtual void Write(ostream &ostr, const FstWriteOptions &opts) const {
+ impl_->Write(ostr, opts);
+ }
+
virtual uint64 Properties(uint64 mask, bool test) const {
return impl_->Properties(mask, test);
}
@@ -214,6 +238,8 @@
<< "particular arc type.";
return 0;
}
+
+
protected:
explicit FstClass(FstClassImplBase *impl) : impl_(impl) { }
@@ -233,7 +259,12 @@
}
}
+ FstClassImplBase *GetImpl() const { return impl_; }
+
FstClassImplBase *GetImpl() { return impl_; }
+
+// friend ostream &operator<<(ostream&, const FstClass&);
+
private:
FstClassImplBase *impl_;
};
@@ -269,6 +300,14 @@
}
}
+ virtual void Write(const string &fname) const {
+ GetImpl()->Write(fname);
+ }
+
+ virtual void Write(ostream &ostr, const FstWriteOptions &opts) const {
+ GetImpl()->Write(ostr, opts);
+ }
+
static MutableFstClass *Read(const string &fname, bool convert = false);
virtual void SetInputSymbols(SymbolTable *is) {
@@ -338,6 +377,4 @@
} // namespace script
} // namespace fst
-
-
#endif // FST_SCRIPT_FST_CLASS_H_
diff --git a/src/include/fst/script/text-io.h b/src/include/fst/script/text-io.h
index 95cc182..d97a007 100644
--- a/src/include/fst/script/text-io.h
+++ b/src/include/fst/script/text-io.h
@@ -32,6 +32,7 @@
#include <iostream>
#include <fstream>
+#include <sstream>
#include <fst/script/weight-class.h>
namespace fst {
diff --git a/src/include/fst/script/weight-class.h b/src/include/fst/script/weight-class.h
index 5a4890f..228216d 100644
--- a/src/include/fst/script/weight-class.h
+++ b/src/include/fst/script/weight-class.h
@@ -56,9 +56,9 @@
}
virtual string to_string() const {
- ostringstream s;
- s << weight;
- return s.str();
+ string str;
+ WeightToStr(weight, &str);
+ return str;
}
virtual bool operator == (const WeightImplBase &other) const {
diff --git a/src/include/fst/shortest-distance.h b/src/include/fst/shortest-distance.h
index 5d38409..9320c4c 100644
--- a/src/include/fst/shortest-distance.h
+++ b/src/include/fst/shortest-distance.h
@@ -178,7 +178,7 @@
!aiter.Done();
aiter.Next()) {
const Arc &arc = aiter.Value();
- if (!arc_filter_(arc) || arc.weight == Weight::Zero())
+ if (!arc_filter_(arc))
continue;
while (distance_->size() <= arc.nextstate) {
distance_->push_back(Weight::Zero());
diff --git a/src/include/fst/signed-log-weight.h b/src/include/fst/signed-log-weight.h
index da96479..61adefb 100644
--- a/src/include/fst/signed-log-weight.h
+++ b/src/include/fst/signed-log-weight.h
@@ -113,9 +113,9 @@
bool s2 = w2.Value1().Value() > 0.0;
T f1 = w1.Value2().Value();
T f2 = w2.Value2().Value();
- if (f1 == FloatLimits<T>::kPosInfinity)
+ if (f1 == FloatLimits<T>::PosInfinity())
return w2;
- else if (f2 == FloatLimits<T>::kPosInfinity)
+ else if (f2 == FloatLimits<T>::PosInfinity())
return w1;
else if (f1 == f2) {
if (s1 == s2)
@@ -173,12 +173,12 @@
bool s2 = w2.Value1().Value() > 0.0;
T f1 = w1.Value2().Value();
T f2 = w2.Value2().Value();
- if (f2 == FloatLimits<T>::kPosInfinity)
+ if (f2 == FloatLimits<T>::PosInfinity())
return SignedLogWeightTpl<T>(TropicalWeight(1.0),
- FloatLimits<T>::kNumberBad);
- else if (f1 == FloatLimits<T>::kPosInfinity)
+ FloatLimits<T>::NumberBad());
+ else if (f1 == FloatLimits<T>::PosInfinity())
return SignedLogWeightTpl<T>(TropicalWeight(1.0),
- FloatLimits<T>::kPosInfinity);
+ FloatLimits<T>::PosInfinity());
else if (s1 == s2)
return SignedLogWeightTpl<T>(TropicalWeight(1.0), (f1 - f2));
else
diff --git a/src/include/fst/slist.h b/src/include/fst/slist.h
index 9f94027..d061ebe 100644
--- a/src/include/fst/slist.h
+++ b/src/include/fst/slist.h
@@ -22,7 +22,7 @@
#include <fst/config.h>
-#if !defined(__ANDROID__) && defined(HAVE___GNU_CXX__SLIST_INT_)
+#if !defined(__ANDROID__) && HAVE___GNU_CXX__SLIST_INT_
#include <slist>
diff --git a/src/include/fst/state-map.h b/src/include/fst/state-map.h
index ace4a3c..454db24 100644
--- a/src/include/fst/state-map.h
+++ b/src/include/fst/state-map.h
@@ -295,6 +295,10 @@
SetArcs(s);
}
+ const Fst<A> &GetFst() const {
+ return *fst_;
+ }
+
private:
void Init() {
SetType("statemap");
@@ -364,10 +368,10 @@
GetImpl()->InitArcIterator(s, data);
}
- private:
- // Makes visible to friends.
+ protected:
Impl *GetImpl() const { return ImplToFst<Impl>::GetImpl(); }
+ private:
void operator=(const StateMapFst<A, B, C> &fst); // disallow
};
diff --git a/src/include/fst/string.h b/src/include/fst/string.h
index 3099b87..d51182e 100644
--- a/src/include/fst/string.h
+++ b/src/include/fst/string.h
@@ -24,6 +24,7 @@
#define FST_LIB_STRING_H_
#include <fst/compact-fst.h>
+#include <fst/icu.h>
#include <fst/mutable-fst.h>
DECLARE_string(fst_field_separator);
@@ -48,7 +49,7 @@
// Compile string 's' into FST 'fst'.
template <class F>
- bool operator()(const string &s, F *fst) {
+ bool operator()(const string &s, F *fst) const {
vector<Label> labels;
if (!ConvertStringToLabels(s, &labels))
return false;
diff --git a/src/include/fst/symbol-table.h b/src/include/fst/symbol-table.h
index 93ebe76..6eb6c2d 100644
--- a/src/include/fst/symbol-table.h
+++ b/src/include/fst/symbol-table.h
@@ -33,6 +33,7 @@
#include <fst/compat.h>
#include <iostream>
#include <fstream>
+#include <sstream>
#include <map>
@@ -56,6 +57,13 @@
string source;
};
+struct SymbolTableTextOptions {
+ SymbolTableTextOptions();
+
+ bool allow_negative;
+ string fst_field_separator;
+};
+
class SymbolTableImpl {
public:
SymbolTableImpl(const string &name)
@@ -88,9 +96,9 @@
return (key == -1) ? AddSymbol(symbol, available_key_++) : key;
}
- static SymbolTableImpl* ReadText(istream &strm,
- const string &name,
- bool allow_negative = false);
+ static SymbolTableImpl* ReadText(
+ istream &strm, const string &name,
+ const SymbolTableTextOptions &opts = SymbolTableTextOptions());
static SymbolTableImpl* Read(istream &strm,
const SymbolTableReadOptions& opts);
@@ -149,13 +157,11 @@
}
string CheckSum() const {
- MutexLock check_sum_lock(&check_sum_mutex_);
MaybeRecomputeCheckSum();
return check_sum_string_;
}
string LabeledCheckSum() const {
- MutexLock check_sum_lock(&check_sum_mutex_);
MaybeRecomputeCheckSum();
return labeled_check_sum_string_;
}
@@ -171,6 +177,8 @@
private:
// Recomputes the checksums (both of them) if we've had changes since the last
// computation (i.e., if check_sum_finalized_ is false).
+ // Takes ~2.5 microseconds (dbg) or ~230 nanoseconds (opt) on a 2.67GHz Xeon
+ // if the checksum is up-to-date (requiring no recomputation).
void MaybeRecomputeCheckSum() const;
struct StrCmp {
@@ -188,8 +196,6 @@
mutable RefCounter ref_count_;
mutable bool check_sum_finalized_;
- mutable CheckSummer check_sum_;
- mutable CheckSummer labeled_check_sum_;
mutable string check_sum_string_;
mutable string labeled_check_sum_string_;
mutable Mutex check_sum_mutex_;
@@ -212,6 +218,9 @@
public:
static const int64 kNoSymbol = -1;
+ // Construct symbol table with an unspecified name.
+ SymbolTable() : impl_(new SymbolTableImpl("<unspecified>")) {}
+
// Construct symbol table with a unique name.
SymbolTable(const string& name) : impl_(new SymbolTableImpl(name)) {}
@@ -226,14 +235,21 @@
if (!impl_->DecrRefCount()) delete impl_;
}
+ // Copys the implemenation from one symbol table to another.
+ void operator=(const SymbolTable &st) {
+ if (impl_ != st.impl_) {
+ st.impl_->IncrRefCount();
+ if (!impl_->DecrRefCount()) delete impl_;
+ impl_ = st.impl_;
+ }
+ }
+
// Read an ascii representation of the symbol table from an istream. Pass a
// name to give the resulting SymbolTable.
- static SymbolTable* ReadText(istream &strm,
- const string& name,
- bool allow_negative = false) {
- SymbolTableImpl* impl = SymbolTableImpl::ReadText(strm,
- name,
- allow_negative);
+ static SymbolTable* ReadText(
+ istream &strm, const string& name,
+ const SymbolTableTextOptions &opts = SymbolTableTextOptions()) {
+ SymbolTableImpl* impl = SymbolTableImpl::ReadText(strm, name, opts);
if (!impl)
return 0;
else
@@ -242,13 +258,13 @@
// read an ascii representation of the symbol table
static SymbolTable* ReadText(const string& filename,
- bool allow_negative = false) {
+ const SymbolTableTextOptions &opts = SymbolTableTextOptions()) {
ifstream strm(filename.c_str(), ifstream::in);
if (!strm) {
LOG(ERROR) << "SymbolTable::ReadText: Can't open file " << filename;
return 0;
}
- return ReadText(strm, filename, allow_negative);
+ return ReadText(strm, filename, opts);
}
@@ -341,7 +357,9 @@
}
// Dump an ascii text representation of the symbol table via a stream
- virtual bool WriteText(ostream &strm) const;
+ virtual bool WriteText(
+ ostream &strm,
+ const SymbolTableTextOptions &opts = SymbolTableTextOptions()) const;
// Dump an ascii text representation of the symbol table
bool WriteText(const string& filename) const {
@@ -404,8 +422,6 @@
private:
SymbolTableImpl* impl_;
-
- void operator=(const SymbolTable &table); // disallow
};
@@ -502,6 +518,20 @@
return new_table;
}
+// Symbol Table Serialization
+inline void SymbolTableToString(const SymbolTable *table, string *result) {
+ ostringstream ostrm;
+ table->Write(ostrm);
+ *result = ostrm.str();
+}
+
+inline SymbolTable *StringToSymbolTable(const string &s) {
+ istringstream istrm(s);
+ return SymbolTable::Read(istrm, SymbolTableReadOptions());
+}
+
+
+
} // namespace fst
#endif // FST_LIB_SYMBOL_TABLE_H__
diff --git a/src/include/fst/test-properties.h b/src/include/fst/test-properties.h
index db1ddcc..12bcba7 100644
--- a/src/include/fst/test-properties.h
+++ b/src/include/fst/test-properties.h
@@ -125,13 +125,14 @@
siter.Next()) {
StateId s = siter.Value();
- Arc prev_arc(kNoLabel, kNoLabel, Weight::One(), 0);
+ Arc prev_arc;
// Create these only if we need to
if (mask & (kIDeterministic | kNonIDeterministic))
ilabels = new unordered_set<Label>;
if (mask & (kODeterministic | kNonODeterministic))
olabels = new unordered_set<Label>;
+ bool first_arc = true;
for (ArcIterator< Fst<Arc> > aiter(fst, s);
!aiter.Done();
aiter.Next()) {
@@ -161,13 +162,15 @@
comp_props |= kOEpsilons;
comp_props &= ~kNoOEpsilons;
}
- if (prev_arc.ilabel != kNoLabel && arc.ilabel < prev_arc.ilabel) {
- comp_props |= kNotILabelSorted;
- comp_props &= ~kILabelSorted;
- }
- if (prev_arc.olabel != kNoLabel && arc.olabel < prev_arc.olabel) {
- comp_props |= kNotOLabelSorted;
- comp_props &= ~kOLabelSorted;
+ if (!first_arc) {
+ if (arc.ilabel < prev_arc.ilabel) {
+ comp_props |= kNotILabelSorted;
+ comp_props &= ~kILabelSorted;
+ }
+ if (arc.olabel < prev_arc.olabel) {
+ comp_props |= kNotOLabelSorted;
+ comp_props &= ~kOLabelSorted;
+ }
}
if (arc.weight != Weight::One() && arc.weight != Weight::Zero()) {
comp_props |= kWeighted;
@@ -182,6 +185,7 @@
comp_props &= ~kString;
}
prev_arc = arc;
+ first_arc = false;
if (ilabels)
ilabels->insert(arc.ilabel);
if (olabels)
diff --git a/src/include/fst/util.h b/src/include/fst/util.h
index 87231e1..a325beb 100644
--- a/src/include/fst/util.h
+++ b/src/include/fst/util.h
@@ -41,6 +41,7 @@
#include <iostream>
#include <fstream>
+#include <sstream>
//
// UTILITY FOR ERROR HANDLING
@@ -264,7 +265,7 @@
ostringstream strm;
strm.precision(9);
strm << w;
- *s += strm.str();
+ s->append(strm.str().data(), strm.str().size());
}
// Utilities for reading/writing label pairs
@@ -312,7 +313,7 @@
template <typename Label>
bool WriteLabelPairs(const string& filename,
const vector<pair<Label, Label> >& pairs) {
- ostream *strm = &std::cout;
+ ostream *strm = &cout;
if (!filename.empty()) {
strm = new ofstream(filename.c_str());
if (!*strm) {
@@ -329,7 +330,7 @@
<< (filename.empty() ? "standard output" : filename);
return false;
}
- if (strm != &std::cout)
+ if (strm != &cout)
delete strm;
return true;
}
diff --git a/src/include/fst/vector-fst.h b/src/include/fst/vector-fst.h
index f6d8a6d..8b80876 100644
--- a/src/include/fst/vector-fst.h
+++ b/src/include/fst/vector-fst.h
@@ -273,9 +273,10 @@
SetProperties(DeleteArcsProperties(Properties()));
}
- private:
// Properties always true of this Fst class
static const uint64 kStaticProperties = kExpanded | kMutable;
+
+ private:
// Current file format version
static const int kFileVersion = 2;
// Minimum file format version supported
@@ -542,7 +543,10 @@
hdr.SetNumStates(CountStates(fst));
update_header = false;
}
- FstImpl<A>::WriteFstHeader(fst, strm, opts, kFileVersion, "vector", &hdr);
+ uint64 properties = fst.Properties(kCopyProperties, false) |
+ VectorFstImpl<A>::kStaticProperties;
+ FstImpl<A>::WriteFstHeader(fst, strm, opts, kFileVersion, "vector",
+ properties, &hdr);
StateId num_states = 0;
for (StateIterator<F> siter(fst); !siter.Done(); siter.Next()) {
typename A::StateId s = siter.Value();
@@ -566,7 +570,7 @@
if (update_header) {
hdr.SetNumStates(num_states);
return FstImpl<A>::UpdateFstHeader(fst, strm, opts, kFileVersion, "vector",
- &hdr, start_offset);
+ properties, &hdr, start_offset);
} else {
if (num_states != hdr.NumStates()) {
LOG(ERROR) << "Inconsistent number of states observed during write";
diff --git a/src/include/fst/visit.h b/src/include/fst/visit.h
index 31a00a8..a02d86a 100644
--- a/src/include/fst/visit.h
+++ b/src/include/fst/visit.h
@@ -166,7 +166,8 @@
// Finds next tree root
for (root = root == start ? 0 : root + 1;
root < nstates && state_status[root] != kWhiteState;
- ++root);
+ ++root) {
+ }
// Check for a state beyond the largest known state
if (!expanded && root == nstates) {
diff --git a/src/include/fst/weight.h b/src/include/fst/weight.h
index 72f5a22..7eb4bb1 100644
--- a/src/include/fst/weight.h
+++ b/src/include/fst/weight.h
@@ -28,16 +28,18 @@
// A left semiring distributes on the left; a right semiring is
// similarly defined.
//
-// A Weight class is required to be (at least) a left or right semiring.
+// A Weight class must have binary functions =Plus= and =Times= and
+// static member functions =Zero()= and =One()= and these must form
+// (at least) a left or right semiring.
//
// In addition, the following should be defined for a Weight:
// Member: predicate on set membership.
-// NoWeight: returns an element that is not a member, should only be
-// used to signal an error.
-// >>: reads weight.
-// <<: prints weight.
-// Read(istream &strm): reads from an input stream.
-// Write(ostream &strm): writes to an output stream.
+// NoWeight: static member function that returns an element that is
+// not a set member; used to signal an error.
+// >>: reads textual representation of a weight.
+// <<: prints textual representation of a weight.
+// Read(istream &strm): reads binary representation of a weight.
+// Write(ostream &strm): writes binary representation of a weight.
// Hash: maps weight to size_t.
// ApproxEqual: approximate equality (for inexact weights)
// Quantize: quantizes wrt delta (for inexact weights)
@@ -46,11 +48,9 @@
// and Times(a, b') == c
// --> a' = Divide(c, b, DIVIDE_RIGHT) if a right semiring, a'.Member()
// and Times(a', b) == c
-// --> b' = Divide(c, a)
-// = Divide(c, a, DIVIDE_ANY)
-// = Divide(c, a, DIVIDE_LEFT)
-// = Divide(c, a, DIVIDE_RIGHT) if a commutative semiring,
-// b'.Member() and Times(a, b') == Times(b', a) == c
+// --> b' = Divide(c, a) = Divide(c, a, DIVIDE_ANY) =
+// Divide(c, a, DIVIDE_LEFT) = Divide(c, a, DIVIDE_RIGHT) if a
+// commutative semiring, b'.Member() and Times(a, b') = Times(b', a) = c
// ReverseWeight: the type of the corresponding reverse weight.
// Typically the same type as Weight for a (both left and right) semiring.
// For the left string semiring, it is the right string semiring.
@@ -66,7 +66,7 @@
// RightSemiring: indicates weights form a right semiring.
// Commutative: for all a,b: Times(a,b) == Times(b,a)
// Idempotent: for all a: Plus(a, a) == a.
-// Path Property: for all a, b: Plus(a, b) == a or Plus(a, b) == b.
+// Path: for all a, b: Plus(a, b) == a or Plus(a, b) == b.
#ifndef FST_LIB_WEIGHT_H__
diff --git a/src/lib/flags.cc b/src/lib/flags.cc
index d05fd73..222d66b 100644
--- a/src/lib/flags.cc
+++ b/src/lib/flags.cc
@@ -24,14 +24,18 @@
DEFINE_int32(v, 0, "verbose level");
DEFINE_bool(help, false, "show usage information");
+DEFINE_bool(helpshort, false, "show brief usage information");
DEFINE_string(tmpdir, "/tmp/", "temporary directory");
using namespace std;
static string flag_usage;
+static string prog_src;
-void SetFlags(const char *usage, int *argc, char ***argv, bool remove_flags) {
+void SetFlags(const char *usage, int *argc, char ***argv,
+ bool remove_flags, const char *src) {
flag_usage = usage;
+ prog_src = src;
int index = 1;
for (; index < *argc; ++index) {
string argval = (*argv)[index];
@@ -82,22 +86,75 @@
}
if (FLAGS_help) {
- ShowUsage();
+ ShowUsage(true);
+ exit(1);
+ }
+
+ if (FLAGS_helpshort) {
+ ShowUsage(false);
exit(1);
}
}
-void ShowUsage() {
+// If flag is defined in file 'src' and 'in_src' true or is not
+// defined in file 'src' and 'in_src' is false, then print usage.
+static void
+ShowUsageRestrict(const std::set< pair<string, string> > &usage_set,
+ const string &src, bool in_src, bool show_file) {
+ string old_file;
+ bool file_out = false;
+ bool usage_out = false;
+ for (std::set< pair<string, string> >::const_iterator it =
+ usage_set.begin();
+ it != usage_set.end();
+ ++it) {
+ const string &file = it->first;
+ const string &usage = it->second;
+
+ bool match = file == src;
+ if ((match && !in_src) || (!match && in_src))
+ continue;
+
+ if (file != old_file) {
+ if (show_file) {
+ if (file_out) cout << "\n";
+ cout << "Flags from: " << file << "\n";
+ file_out = true;
+ }
+ old_file = file;
+ }
+ cout << usage << "\n";
+ usage_out = true;
+ }
+ if (usage_out) cout << "\n";
+}
+
+void ShowUsage(bool long_usage) {
+ std::set< pair<string, string> > usage_set;
+
cout << flag_usage << "\n";
- cout << " Flags Description:\n";
+
FlagRegister<bool> *bool_register = FlagRegister<bool>::GetRegister();
- bool_register->ShowUsage();
+ bool_register->GetUsage(&usage_set);
FlagRegister<string> *string_register = FlagRegister<string>::GetRegister();
- string_register->ShowUsage();
+ string_register->GetUsage(&usage_set);
FlagRegister<int32> *int32_register = FlagRegister<int32>::GetRegister();
- int32_register->ShowUsage();
+ int32_register->GetUsage(&usage_set);
FlagRegister<int64> *int64_register = FlagRegister<int64>::GetRegister();
- int64_register->ShowUsage();
+ int64_register->GetUsage(&usage_set);
FlagRegister<double> *double_register = FlagRegister<double>::GetRegister();
- double_register->ShowUsage();
+ double_register->GetUsage(&usage_set);
+
+ if (!prog_src.empty()) {
+ cout << "PROGRAM FLAGS:\n\n";
+ ShowUsageRestrict(usage_set, prog_src, true, false);
+ }
+
+ if (!long_usage)
+ return;
+
+ if (!prog_src.empty())
+ cout << "LIBRARY FLAGS:\n\n";
+
+ ShowUsageRestrict(usage_set, prog_src, false, true);
}
diff --git a/src/lib/properties.cc b/src/lib/properties.cc
index db0e2c8..9e03d59 100644
--- a/src/lib/properties.cc
+++ b/src/lib/properties.cc
@@ -117,8 +117,10 @@
uint64 outprops = kAccessible;
if (((kAcceptor | kNoIEpsilons) & inprops) || has_subsequential_label)
outprops |= kIDeterministic;
- outprops |= (kError | kAcceptor | kNoEpsilons | kAcyclic |
+ outprops |= (kError | kAcceptor | kAcyclic |
kInitialAcyclic | kCoAccessible | kString) & inprops;
+ if (inprops & kNoIEpsilons)
+ outprops |= kNoEpsilons & inprops;
if (inprops & kAccessible)
outprops |= (kNotAcceptor | kEpsilons | kIEpsilons | kOEpsilons |
kCyclic) & inprops;
@@ -375,6 +377,7 @@
uint64 outprops = (kAcceptor | kUnweighted | kAcyclic | kAccessible)
& inprops1 & inprops2;
outprops |= kError & (inprops1 | inprops2);
+ outprops |= kInitialAcyclic;
bool empty1 = delayed; // Can fst1 be the empty machine?
bool empty2 = delayed; // Can fst2 be the empty machine?
diff --git a/src/lib/symbol-table.cc b/src/lib/symbol-table.cc
index 8b35cdf..a195a7c 100644
--- a/src/lib/symbol-table.cc
+++ b/src/lib/symbol-table.cc
@@ -20,6 +20,7 @@
// Classes to provide symbol-to-integer and integer-to-symbol mappings.
#include <fst/symbol-table.h>
+
#include <fst/util.h>
DEFINE_bool(fst_compat_symbols, true,
@@ -35,9 +36,12 @@
// Identifies stream data as a symbol table (and its endianity)
static const int32 kSymbolTableMagicNumber = 2125658996;
+SymbolTableTextOptions::SymbolTableTextOptions()
+ : allow_negative(false), fst_field_separator(FLAGS_fst_field_separator) { }
+
SymbolTableImpl* SymbolTableImpl::ReadText(istream &strm,
const string &filename,
- bool allow_negative) {
+ const SymbolTableTextOptions &opts) {
SymbolTableImpl* impl = new SymbolTableImpl(filename);
int64 nline = 0;
@@ -45,7 +49,7 @@
while (strm.getline(line, kLineLen)) {
++nline;
vector<char *> col;
- string separator = FLAGS_fst_field_separator + "\n";
+ string separator = opts.fst_field_separator + "\n";
SplitToVector(line, separator.c_str(), &col, true);
if (col.size() == 0) // empty line
continue;
@@ -61,7 +65,7 @@
char *p;
int64 key = strtoll(value, &p, 10);
if (p < value + strlen(value) ||
- (!allow_negative && key < 0) || key == -1) {
+ (!opts.allow_negative && key < 0) || key == -1) {
LOG(ERROR) << "SymbolTable::ReadText: Bad non-negative integer \""
<< value << "\" (skipping), "
<< "file = " << filename << ", line = " << nline;
@@ -74,21 +78,30 @@
}
void SymbolTableImpl::MaybeRecomputeCheckSum() const {
- if (check_sum_finalized_)
- return;
+ {
+ ReaderMutexLock check_sum_lock(&check_sum_mutex_);
+ if (check_sum_finalized_)
+ return;
+ }
+
+ // We'll aquire an exclusive lock to recompute the checksums.
+ MutexLock check_sum_lock(&check_sum_mutex_);
+ if (check_sum_finalized_) // Another thread (coming in around the same time
+ return; // might have done it already). So we recheck.
// Calculate the original label-agnostic check sum.
- check_sum_.Reset();
+ CheckSummer check_sum;
for (int64 i = 0; i < symbols_.size(); ++i)
- check_sum_.Update(symbols_[i], strlen(symbols_[i]) + 1);
- check_sum_string_ = check_sum_.Digest();
+ check_sum.Update(symbols_[i], strlen(symbols_[i]) + 1);
+ check_sum_string_ = check_sum.Digest();
// Calculate the safer, label-dependent check sum.
- labeled_check_sum_.Reset();
+ CheckSummer labeled_check_sum;
for (int64 key = 0; key < dense_key_limit_; ++key) {
ostringstream line;
line << symbols_[key] << '\t' << key;
- labeled_check_sum_.Update(line.str()); }
+ labeled_check_sum.Update(line.str().data(), line.str().size());
+ }
for (map<int64, const char*>::const_iterator it =
key_map_.begin();
it != key_map_.end();
@@ -96,10 +109,10 @@
if (it->first >= dense_key_limit_) {
ostringstream line;
line << it->second << '\t' << it->first;
- labeled_check_sum_.Update(line.str());
+ labeled_check_sum.Update(line.str().data(), line.str().size());
}
}
- labeled_check_sum_string_ = labeled_check_sum_.Digest();
+ labeled_check_sum_string_ = labeled_check_sum.Digest();
check_sum_finalized_ = true;
}
@@ -231,12 +244,22 @@
impl_->AddSymbol(iter.Symbol());
}
-bool SymbolTable::WriteText(ostream &strm) const {
+bool SymbolTable::WriteText(ostream &strm,
+ const SymbolTableTextOptions &opts) const {
+ if (opts.fst_field_separator.empty()) {
+ LOG(ERROR) << "Missing required field separator";
+ return false;
+ }
+ bool once_only = false;
for (SymbolTableIterator iter(*this); !iter.Done(); iter.Next()) {
ostringstream line;
- line << iter.Symbol() << FLAGS_fst_field_separator[0] << iter.Value()
+ if (iter.Value() < 0 && !opts.allow_negative && !once_only) {
+ LOG(WARNING) << "Negative symbol table entry when not allowed";
+ once_only = true;
+ }
+ line << iter.Symbol() << opts.fst_field_separator[0] << iter.Value()
<< '\n';
- strm.write(line.str().c_str(), line.str().length());
+ strm.write(line.str().data(), line.str().length());
}
return true;
}
diff --git a/src/lib/temp_Android.temp_mk b/src/lib/temp_Android.temp_mk
deleted file mode 100644
index 7a6936e..0000000
--- a/src/lib/temp_Android.temp_mk
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Copyright 2012 Google Inc. All Rights Reserved.
-# Author: npereira@google.com (Nicole Pereira)
-#
-# Android makefile for openfst library.
-#
-
-OPENFST_DIR := $(call my-dir)
-
-LOCAL_PATH := $(call my-dir)
-include $(CLEAR_VARS)
-
-LOCAL_MODULE := libopenfst_patts
-
-LOCAL_CPP_EXTENSION := .cc
-LOCAL_C_INCLUDES += $(OPENFST_DIR)/src/include/
-
-#LOCAL_SRC_FILES := $(call private-function-all-cpp-files-under, src)
-
-include $(BUILD_STATIC_LIBRARY)
diff --git a/src/lib/util.cc b/src/lib/util.cc
index eeba92b..f754da5 100644
--- a/src/lib/util.cc
+++ b/src/lib/util.cc
@@ -26,7 +26,7 @@
DEFINE_bool(fst_error_fatal, true,
"FST errors are fatal; o.w. return objects flagged as bad: "
- " e.g., FSTs - kError prop. true, FST weights - not a member()");
+ " e.g., FSTs - kError prop. true, FST weights - not a Member()");
namespace fst {
@@ -49,7 +49,7 @@
void Int64ToStr(int64 n, string *s) {
ostringstream nstr;
nstr << n;
- *s = nstr.str();
+ s->append(nstr.str().data(), nstr.str().size());
}
void ConvertToLegalCSymbol(string *s) {
diff --git a/src/script/fst-class.cc b/src/script/fst-class.cc
index a784b08..bf5d963 100644
--- a/src/script/fst-class.cc
+++ b/src/script/fst-class.cc
@@ -74,10 +74,14 @@
ifstream in(fname.c_str(), ifstream::in | ifstream::binary);
return ReadFst<FstClass>(in, fname);
} else {
- return ReadFst<FstClass>(std::cin, "standard input");
+ return ReadFst<FstClass>(cin, "standard input");
}
}
+FstClass *FstClass::Read(istream &istr, const string &source) {
+ return ReadFst<FstClass>(istr, source);
+}
+
//
// MUTABLE FST CLASS METHODS
//
@@ -88,7 +92,7 @@
ifstream in(fname.c_str(), ifstream::in | ifstream::binary);
return ReadFst<MutableFstClass>(in, fname);
} else {
- return ReadFst<MutableFstClass>(std::cin, "standard input");
+ return ReadFst<MutableFstClass>(cin, "standard input");
}
} else { // Converts to VectorFstClass if not mutable.
FstClass *ifst = FstClass::Read(fname);
@@ -133,7 +137,7 @@
ifstream in(fname.c_str(), ifstream::in | ifstream::binary);
return ReadFst<VectorFstClass>(in, fname);
} else {
- return ReadFst<VectorFstClass>(std::cin, "standard input");
+ return ReadFst<VectorFstClass>(cin, "standard input");
}
}
diff --git a/src/script/text-io.cc b/src/script/text-io.cc
index 2f5efab..65b4eed 100644
--- a/src/script/text-io.cc
+++ b/src/script/text-io.cc
@@ -67,7 +67,7 @@
// Writes vector of weights; returns true on success.
bool WritePotentials(const string& filename,
const vector<WeightClass>& potential) {
- ostream *strm = &std::cout;
+ ostream *strm = &cout;
if (!filename.empty()) {
strm = new ofstream(filename.c_str());
if (!*strm) {
@@ -85,7 +85,7 @@
LOG(ERROR) << "WritePotentials: Write failed: "
<< (filename.empty() ? "standard output" : filename);
bool ret = *strm;
- if (strm != &std::cout)
+ if (strm != &cout)
delete strm;
return ret;
}
diff --git a/src/test/algo_test.cc b/src/test/algo_test.cc
index 9538d54..27e3e95 100644
--- a/src/test/algo_test.cc
+++ b/src/test/algo_test.cc
@@ -66,7 +66,7 @@
int main(int argc, char **argv) {
FLAGS_fst_verify_properties = true;
std::set_new_handler(FailedNewHandler);
- SetFlags(argv[0], &argc, &argv, true);
+ SET_FLAGS(argv[0], &argc, &argv, true);
static const int kCacheGcLimit = 20;
diff --git a/src/test/fst_test.cc b/src/test/fst_test.cc
index 7d8d600..47899ba 100644
--- a/src/test/fst_test.cc
+++ b/src/test/fst_test.cc
@@ -123,7 +123,7 @@
int main(int argc, char **argv) {
FLAGS_fst_verify_properties = true;
std::set_new_handler(FailedNewHandler);
- SetFlags(argv[0], &argc, &argv, true);
+ SET_FLAGS(argv[0], &argc, &argv, true);
// VectorFst<StdArc> tests
{
diff --git a/src/test/fst_test.h b/src/test/fst_test.h
index da16476..8f9dca0 100644
--- a/src/test/fst_test.h
+++ b/src/test/fst_test.h
@@ -65,14 +65,14 @@
StateIterator<G> siter(fst);
Matcher<G> matcher(fst, MATCH_INPUT);
MatchType match_type = matcher.Type(true);
- for (; !siter.Done(); siter.Next());
+ for (; !siter.Done(); siter.Next()) {}
for (siter.Reset(); !siter.Done(); siter.Next()) {
StateId s = siter.Value();
matcher.SetState(s);
CHECK_EQ(fst.Final(s), NthWeight(s));
size_t na = 0;
ArcIterator<G> aiter(fst, s);
- for (; !aiter.Done(); aiter.Next());
+ for (; !aiter.Done(); aiter.Next()) {}
for (aiter.Reset(); !aiter.Done(); aiter.Next()) {
++na;
const Arc &arc = aiter.Value();
@@ -129,7 +129,7 @@
size_t na = 0;
size_t ni = fst->NumInputEpsilons(s);
MutableArcIterator<G> aiter(fst, s);
- for (; !aiter.Done(); aiter.Next());
+ for (; !aiter.Done(); aiter.Next()) {}
for (aiter.Reset(); !aiter.Done(); aiter.Next()) {
++na;
Arc arc = aiter.Value();
diff --git a/src/test/weight_test.cc b/src/test/weight_test.cc
index 54ba85d..3554456 100644
--- a/src/test/weight_test.cc
+++ b/src/test/weight_test.cc
@@ -97,7 +97,7 @@
int main(int argc, char **argv) {
std::set_new_handler(FailedNewHandler);
- SetFlags(argv[0], &argc, &argv, true);
+ SET_FLAGS(argv[0], &argc, &argv, true);
int seed = FLAGS_seed >= 0 ? FLAGS_seed : time(0);
LOG(INFO) << "Seed = " << seed;
