[Indic] Futher adjust ZWJ handling in Indic-like shapers
After the Ngapi hackfest work, we were assuming that fonts
won't use presentation features to choose specific forms
(eg. conjuncts). As such, we were using auto-joiner behavior
for such features. It proved to be troublesome as many fonts
used presentation forms ('pres') for example to form conjuncts,
which need to be disabled when a ZWJ is inserted.
Two examples:
U+0D2F,U+200D,U+0D4D,U+0D2F with kartika.ttf
U+0995,U+09CD,U+200D,U+09B7 with vrinda.ttf
What we do now is to never do magic to ZWJ during GSUB's main input
match for Indic-style shapers. Note that backtrack/lookahead are still
matched liberally, as is GPOS. This seems to be an acceptable
compromise.
As to the bug that initially started this work, that one needs to
be fixed differently:
Bug 58714 - Kannada u+0cb0 u+200d u+0ccd u+0c95 u+0cbe does not
provide same results as Windows8
https://bugs.freedesktop.org/show_bug.cgi?id=58714
New numbers:
BENGALI: 353689 out of 354188 tests passed. 499 failed (0.140886%)
DEVANAGARI: 707305 out of 707394 tests passed. 89 failed (0.0125814%)
GUJARATI: 366349 out of 366457 tests passed. 108 failed (0.0294714%)
GURMUKHI: 60706 out of 60747 tests passed. 41 failed (0.067493%)
KANNADA: 951030 out of 951913 tests passed. 883 failed (0.0927606%)
KHMER: 299070 out of 299124 tests passed. 54 failed (0.0180527%)
LAO: 53611 out of 53644 tests passed. 33 failed (0.0615167%)
MALAYALAM: 1048102 out of 1048334 tests passed. 232 failed (0.0221304%)
ORIYA: 42320 out of 42329 tests passed. 9 failed (0.021262%)
SINHALA: 271666 out of 271847 tests passed. 181 failed (0.0665816%)
TAMIL: 1091753 out of 1091754 tests passed. 1 failed (9.15957e-05%)
TELUGU: 970555 out of 970573 tests passed. 18 failed (0.00185457%)
TIBETAN: 208469 out of 208469 tests passed. 0 failed (0%)
diff --git a/src/hb-ot-layout-gsubgpos-private.hh b/src/hb-ot-layout-gsubgpos-private.hh
index d749efa..f46b378 100644
--- a/src/hb-ot-layout-gsubgpos-private.hh
+++ b/src/hb-ot-layout-gsubgpos-private.hh
@@ -261,7 +261,7 @@
hb_buffer_t *buffer;
hb_direction_t direction;
hb_mask_t lookup_mask;
- bool auto_joiners;
+ bool auto_zwj;
recurse_func_t recurse_func;
unsigned int nesting_level_left;
unsigned int lookup_props;
@@ -274,12 +274,12 @@
hb_font_t *font_,
hb_buffer_t *buffer_,
hb_mask_t lookup_mask_,
- bool auto_joiners_) :
+ bool auto_zwj_) :
table_index (table_index_),
font (font_), face (font->face), buffer (buffer_),
direction (buffer_->props.direction),
lookup_mask (lookup_mask_),
- auto_joiners (auto_joiners_),
+ auto_zwj (auto_zwj_),
recurse_func (NULL),
nesting_level_left (MAX_NESTING_LEVEL),
lookup_props (0),
@@ -383,16 +383,12 @@
end (c->buffer->len)
{
matcher.set_lookup_props (c->lookup_props);
- if (c->auto_joiners)
- {
- /* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
- matcher.set_ignore_zwnj (context_match || c->table_index == 1);
- matcher.set_ignore_zwj (true);
- }
+ /* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
+ matcher.set_ignore_zwnj (context_match || c->table_index == 1);
+ /* Ignore ZWJ if we are matching GSUB context, or matching GPOS, or if asked to. */
+ matcher.set_ignore_zwj (context_match || c->table_index == 1 || c->auto_zwj);
if (!context_match)
- {
matcher.set_mask (c->lookup_mask);
- }
matcher.set_syllable (start_index_ == c->buffer->idx ? c->buffer->cur().syllable () : 0);
}
inline void set_lookup_props (unsigned int lookup_props) { matcher.set_lookup_props (lookup_props); }
@@ -457,16 +453,12 @@
num_items (num_items_)
{
matcher.set_lookup_props (c->lookup_props);
- if (c->auto_joiners)
- {
- /* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
- matcher.set_ignore_zwnj (context_match || c->table_index == 1);
- matcher.set_ignore_zwj (true);
- }
+ /* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
+ matcher.set_ignore_zwnj (context_match || c->table_index == 1);
+ /* Ignore ZWJ if we are matching GSUB context, or matching GPOS, or if asked to. */
+ matcher.set_ignore_zwj (context_match || c->table_index == 1 || c->auto_zwj);
if (!context_match)
- {
matcher.set_mask (c->lookup_mask);
- }
matcher.set_syllable (start_index_ == c->buffer->idx ? c->buffer->cur().syllable () : 0);
}
inline void set_lookup_props (unsigned int lookup_props) { matcher.set_lookup_props (lookup_props); }
diff --git a/src/hb-ot-layout-private.hh b/src/hb-ot-layout-private.hh
index d826108..4866c41 100644
--- a/src/hb-ot-layout-private.hh
+++ b/src/hb-ot-layout-private.hh
@@ -207,7 +207,7 @@
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
- hb_bool_t auto_joiners);
+ hb_bool_t auto_zwj);
/* Should be called after all the substitute_lookup's are done */
HB_INTERNAL void
@@ -225,7 +225,7 @@
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
- hb_bool_t auto_joiners);
+ hb_bool_t auto_zwj);
/* Should be called after all the position_lookup's are done */
HB_INTERNAL void
diff --git a/src/hb-ot-layout.cc b/src/hb-ot-layout.cc
index 0349b32..8161ce3 100644
--- a/src/hb-ot-layout.cc
+++ b/src/hb-ot-layout.cc
@@ -667,11 +667,11 @@
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
- hb_bool_t auto_joiners)
+ hb_bool_t auto_zwj)
{
if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gsub_lookup_count)) return false;
- OT::hb_apply_context_t c (0, font, buffer, mask, auto_joiners);
+ OT::hb_apply_context_t c (0, font, buffer, mask, auto_zwj);
const OT::SubstLookup& l = hb_ot_layout_from_face (font->face)->gsub->get_lookup (lookup_index);
@@ -717,11 +717,11 @@
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
- hb_bool_t auto_joiners)
+ hb_bool_t auto_zwj)
{
if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gpos_lookup_count)) return false;
- OT::hb_apply_context_t c (1, font, buffer, mask, auto_joiners);
+ OT::hb_apply_context_t c (1, font, buffer, mask, auto_zwj);
const OT::PosLookup& l = hb_ot_layout_from_face (font->face)->gpos->get_lookup (lookup_index);
diff --git a/src/hb-ot-map-private.hh b/src/hb-ot-map-private.hh
index bc82557..a679fb5 100644
--- a/src/hb-ot-map-private.hh
+++ b/src/hb-ot-map-private.hh
@@ -50,7 +50,7 @@
hb_mask_t mask;
hb_mask_t _1_mask; /* mask for value=1, for quick access */
unsigned int needs_fallback : 1;
- unsigned int auto_joiners : 1;
+ unsigned int auto_zwj : 1;
static int cmp (const feature_map_t *a, const feature_map_t *b)
{ return a->tag < b->tag ? -1 : a->tag > b->tag ? 1 : 0; }
@@ -58,7 +58,7 @@
struct lookup_map_t {
unsigned short index;
- unsigned short auto_joiners : 1;
+ unsigned short auto_zwj : 1;
hb_mask_t mask;
static int cmp (const lookup_map_t *a, const lookup_map_t *b)
@@ -139,7 +139,7 @@
unsigned int table_index,
unsigned int feature_index,
hb_mask_t mask,
- bool auto_joiners);
+ bool auto_zwj);
hb_mask_t global_mask;
@@ -152,7 +152,7 @@
F_NONE = 0x0000,
F_GLOBAL = 0x0001,
F_HAS_FALLBACK = 0x0002,
- F_MANUAL_JOINERS = 0x0004
+ F_MANUAL_ZWJ = 0x0004
};
/* Macro version for where const is desired. */
#define F_COMBINE(l,r) (hb_ot_map_feature_flags_t ((unsigned int) (l) | (unsigned int) (r)))
diff --git a/src/hb-ot-map.cc b/src/hb-ot-map.cc
index 28a43a4..85e6e16 100644
--- a/src/hb-ot-map.cc
+++ b/src/hb-ot-map.cc
@@ -34,7 +34,7 @@
unsigned int table_index,
unsigned int feature_index,
hb_mask_t mask,
- bool auto_joiners)
+ bool auto_zwj)
{
unsigned int lookup_indices[32];
unsigned int offset, len;
@@ -54,7 +54,7 @@
return;
lookup->mask = mask;
lookup->index = lookup_indices[i];
- lookup->auto_joiners = auto_joiners;
+ lookup->auto_zwj = auto_zwj;
}
offset += len;
@@ -113,7 +113,7 @@
hb_ot_layout_substitute_lookup (font, buffer,
lookups[table_index][i].index,
lookups[table_index][i].mask,
- lookups[table_index][i].auto_joiners);
+ lookups[table_index][i].auto_zwj);
buffer->clear_output ();
@@ -124,7 +124,7 @@
for (; i < lookups[table_index].len; i++)
hb_ot_layout_substitute_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
- lookups[table_index][i].auto_joiners);
+ lookups[table_index][i].auto_zwj);
}
void hb_ot_map_t::position (const hb_ot_shape_plan_t *plan, hb_font_t *font, hb_buffer_t *buffer) const
@@ -137,7 +137,7 @@
for (; i < pause->num_lookups; i++)
hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
- lookups[table_index][i].auto_joiners);
+ lookups[table_index][i].auto_zwj);
if (pause->callback)
pause->callback (plan, font, buffer);
@@ -146,7 +146,7 @@
for (; i < lookups[table_index].len; i++)
hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
- lookups[table_index][i].auto_joiners);
+ lookups[table_index][i].auto_zwj);
}
void hb_ot_map_t::collect_lookups (unsigned int table_index, hb_set_t *lookups_out) const
@@ -243,7 +243,7 @@
map->index[1] = feature_index[1];
map->stage[0] = info->stage[0];
map->stage[1] = info->stage[1];
- map->auto_joiners = !(info->flags & F_MANUAL_JOINERS);
+ map->auto_zwj = !(info->flags & F_MANUAL_ZWJ);
if ((info->flags & F_GLOBAL) && info->max_value == 1) {
/* Uses the global bit */
map->shift = 0;
@@ -286,7 +286,7 @@
m.add_lookups (face, table_index,
m.features[i].index[table_index],
m.features[i].mask,
- m.features[i].auto_joiners);
+ m.features[i].auto_zwj);
/* Sort lookups and merge duplicates */
if (last_num_lookups < m.lookups[table_index].len)
@@ -300,7 +300,7 @@
else
{
m.lookups[table_index][j].mask |= m.lookups[table_index][i].mask;
- m.lookups[table_index][j].auto_joiners &= m.lookups[table_index][i].auto_joiners;
+ m.lookups[table_index][j].auto_zwj &= m.lookups[table_index][i].auto_zwj;
}
m.lookups[table_index].shrink (j + 1);
}
diff --git a/src/hb-ot-shape-complex-arabic-fallback.hh b/src/hb-ot-shape-complex-arabic-fallback.hh
index 6f8e951..5e151f7 100644
--- a/src/hb-ot-shape-complex-arabic-fallback.hh
+++ b/src/hb-ot-shape-complex-arabic-fallback.hh
@@ -244,7 +244,7 @@
{
for (unsigned int i = 0; i < ARABIC_NUM_FALLBACK_FEATURES; i++)
if (fallback_plan->lookup_array[i]) {
- OT::hb_apply_context_t c (0, font, buffer, fallback_plan->mask_array[i], true/*auto_joiners*/);
+ OT::hb_apply_context_t c (0, font, buffer, fallback_plan->mask_array[i], true/*auto_zwj*/);
fallback_plan->lookup_array[i]->apply_string (&c, &fallback_plan->digest_array[i]);
}
}
diff --git a/src/hb-ot-shape-complex-indic.cc b/src/hb-ot-shape-complex-indic.cc
index ddb1839..eb1e0be 100644
--- a/src/hb-ot-shape-complex-indic.cc
+++ b/src/hb-ot-shape-complex-indic.cc
@@ -339,18 +339,18 @@
* Basic features.
* These features are applied in order, one at a time, after initial_reordering.
*/
- {HB_TAG('n','u','k','t'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
- {HB_TAG('a','k','h','n'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
- {HB_TAG('r','p','h','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
- {HB_TAG('r','k','r','f'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
- {HB_TAG('p','r','e','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
- {HB_TAG('b','l','w','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
- {HB_TAG('h','a','l','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
- {HB_TAG('a','b','v','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
- {HB_TAG('p','s','t','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
- {HB_TAG('c','f','a','r'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
- {HB_TAG('v','a','t','u'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
- {HB_TAG('c','j','c','t'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
+ {HB_TAG('n','u','k','t'), F_GLOBAL},
+ {HB_TAG('a','k','h','n'), F_GLOBAL},
+ {HB_TAG('r','p','h','f'), F_NONE},
+ {HB_TAG('r','k','r','f'), F_GLOBAL},
+ {HB_TAG('p','r','e','f'), F_NONE},
+ {HB_TAG('b','l','w','f'), F_NONE},
+ {HB_TAG('h','a','l','f'), F_NONE},
+ {HB_TAG('a','b','v','f'), F_NONE},
+ {HB_TAG('p','s','t','f'), F_NONE},
+ {HB_TAG('c','f','a','r'), F_NONE},
+ {HB_TAG('v','a','t','u'), F_GLOBAL},
+ {HB_TAG('c','j','c','t'), F_GLOBAL},
/*
* Other features.
* These features are applied all at once, after final_reordering.
@@ -428,12 +428,12 @@
unsigned int i = 0;
map->add_gsub_pause (initial_reordering);
for (; i < INDIC_BASIC_FEATURES; i++) {
- map->add_feature (indic_features[i].tag, 1, indic_features[i].flags);
+ map->add_feature (indic_features[i].tag, 1, indic_features[i].flags | F_MANUAL_ZWJ);
map->add_gsub_pause (NULL);
}
map->add_gsub_pause (final_reordering);
for (; i < INDIC_NUM_FEATURES; i++) {
- map->add_feature (indic_features[i].tag, 1, indic_features[i].flags);
+ map->add_feature (indic_features[i].tag, 1, indic_features[i].flags | F_MANUAL_ZWJ);
}
}
@@ -1021,7 +1021,7 @@
/* ZWJ/ZWNJ should disable CJCT. They do that by simply
* being there, since we don't skip them for the CJCT
- * feature (ie. F_MANUAL_JOINERS) */
+ * feature (ie. F_MANUAL_ZWJ) */
/* A ZWNJ disables HALF. */
if (non_joiner)
diff --git a/src/hb-ot-shape-complex-myanmar.cc b/src/hb-ot-shape-complex-myanmar.cc
index 8491047..ff13bdd 100644
--- a/src/hb-ot-shape-complex-myanmar.cc
+++ b/src/hb-ot-shape-complex-myanmar.cc
@@ -92,12 +92,12 @@
map->add_gsub_pause (initial_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
{
- map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_JOINERS);
+ map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
map->add_gsub_pause (NULL);
}
map->add_gsub_pause (final_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
- map->add_feature (other_features[i], 1, F_GLOBAL);
+ map->add_feature (other_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
}
static void
diff --git a/src/hb-ot-shape-complex-sea.cc b/src/hb-ot-shape-complex-sea.cc
index 9438658..9c0c303 100644
--- a/src/hb-ot-shape-complex-sea.cc
+++ b/src/hb-ot-shape-complex-sea.cc
@@ -93,12 +93,12 @@
map->add_gsub_pause (initial_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
{
- map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_JOINERS);
+ map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
map->add_gsub_pause (NULL);
}
map->add_gsub_pause (final_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
- map->add_feature (other_features[i], 1, F_GLOBAL);
+ map->add_feature (other_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
}
static void
diff --git a/src/hb-ot-shape-fallback.cc b/src/hb-ot-shape-fallback.cc
index afe0f39..3341825 100644
--- a/src/hb-ot-shape-fallback.cc
+++ b/src/hb-ot-shape-fallback.cc
@@ -420,7 +420,7 @@
hb_mask_t kern_mask = plan->map.get_1_mask (HB_DIRECTION_IS_HORIZONTAL (buffer->props.direction) ?
HB_TAG ('k','e','r','n') : HB_TAG ('v','k','r','n'));
- OT::hb_apply_context_t c (1, font, buffer, kern_mask, true/*auto_joiners*/);
+ OT::hb_apply_context_t c (1, font, buffer, kern_mask, true/*auto_zwj*/);
c.set_lookup_props (OT::LookupFlag::IgnoreMarks);
for (buffer->idx = 0; buffer->idx < count;)
diff --git a/test/shaping/texts/in-tree/shaper-indic/indic/script-bengali/misc/misc.txt b/test/shaping/texts/in-tree/shaper-indic/indic/script-bengali/misc/misc.txt
index 35ce952..aa43590 100644
--- a/test/shaping/texts/in-tree/shaper-indic/indic/script-bengali/misc/misc.txt
+++ b/test/shaping/texts/in-tree/shaper-indic/indic/script-bengali/misc/misc.txt
@@ -50,3 +50,4 @@
ন্ত্র
ত্যু
চ্য্র
+ক্ষ
diff --git a/test/shaping/texts/in-tree/shaper-indic/indic/script-malayalam/misc/misc.txt b/test/shaping/texts/in-tree/shaper-indic/indic/script-malayalam/misc/misc.txt
index 78fdeb8..c5144f0 100644
--- a/test/shaping/texts/in-tree/shaper-indic/indic/script-malayalam/misc/misc.txt
+++ b/test/shaping/texts/in-tree/shaper-indic/indic/script-malayalam/misc/misc.txt
@@ -61,3 +61,4 @@
ല്പ്പേ
ശിം
കോം
+യ്യ
