From 36d900d6b824b97d98731fc8ef5fd3f918bb56ae Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 14:40:14 -0400 Subject: [PATCH 01/14] Decouple German ITN Cardinal in de from TN Signed-off-by: Adelina Dunina --- Jenkinsfile | 2 +- .../de/data/cardinal/denormalized.tsv | 17 + .../de/data/cardinal/digits.tsv | 13 + .../de/data/cardinal/flips.tsv | 72 ++++ .../de/data/cardinal/single_digit_years.tsv | 9 + .../de/data/cardinal/teens.tsv | 10 + .../de/data/cardinal/tens.tsv | 8 + .../de/data/cardinal/ties.tsv | 8 + .../measure/nouns_forcing_denormalization.tsv | 102 +++++ .../de/graph_utils.py | 334 +++++++++++++++ .../de/taggers/cardinal.py | 400 ++++++++++++++++-- .../de/taggers/tokenize_and_classify.py | 2 +- .../inverse_text_normalization/de/utils.py | 27 ++ .../de/verbalizers/cardinal.py | 86 +++- .../de/verbalizers/verbalize.py | 4 +- .../test_cases_cardinal.txt | 70 +-- 16 files changed, 1076 insertions(+), 88 deletions(-) create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/graph_utils.py create mode 100644 nemo_text_processing/inverse_text_normalization/de/utils.py diff --git a/Jenkinsfile b/Jenkinsfile index 39972c461..ba38def21 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,7 +11,7 @@ pipeline { } environment { AR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-0' - DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-23-24-0' + DE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-04-26-0' EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-11-26-1' ES_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-25-24-0' ES_EN_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-30-24-0' diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv new file mode 100644 index 000000000..89fa8ccff --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv @@ -0,0 +1,17 @@ +null 0 +eine 1 +eins 1 +ein 1 +zwei 2 +zwo 2 +zwö 2 +drei 3 +vier 4 +fünf 5 +sechs 6 +sieben 7 +acht 8 +neun 9 +zehn 10 +elf 11 +zwölf 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv new file mode 100644 index 000000000..d440115cf --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/digits.tsv @@ -0,0 +1,13 @@ +eine 1 +eins 1 +ein 1 +zwei 2 +zwo 2 +zwö 2 +drei 3 +vier 4 +fünf 5 +sechs 6 +sieben 7 +acht 8 +neun 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv new file mode 100644 index 000000000..16c4fd546 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv @@ -0,0 +1,72 @@ +12 21 +22 22 +32 23 +42 24 +52 25 +62 26 +72 27 +82 28 +92 29 +13 31 +23 32 +33 33 +43 34 +53 35 +63 36 +73 37 +83 38 +93 39 +14 41 +24 42 +34 43 +44 44 +54 45 +64 46 +74 47 +84 48 +94 49 +15 51 +25 52 +35 53 +45 54 +55 55 +65 56 +75 57 +85 58 +95 59 +16 61 +26 62 +36 63 +46 64 +56 65 +66 66 +76 67 +86 68 +96 69 +17 71 +27 72 +37 73 +47 74 +57 75 +67 76 +77 77 +87 78 +97 79 +18 81 +28 82 +38 83 +48 84 +58 85 +68 86 +78 87 +88 88 +98 89 +19 91 +29 92 +39 93 +49 94 +59 95 +69 96 +79 97 +89 98 +99 99 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv new file mode 100644 index 000000000..7d5893d20 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv @@ -0,0 +1,9 @@ +eins 01 +zwei 02 +drei 03 +vier 04 +fünf 05 +sechs 06 +sieben 07 +acht 08 +neun 09 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv new file mode 100644 index 000000000..854712ea6 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/teens.tsv @@ -0,0 +1,10 @@ +zehn 10 +elf 11 +zwölf 12 +dreizehn 13 +vierzehn 14 +fünfzehn 15 +sechzehn 16 +siebzehn 17 +achtzehn 18 +neunzehn 19 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv new file mode 100644 index 000000000..28a3f8032 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tens.tsv @@ -0,0 +1,8 @@ +zwanzig 2 +dreißig 3 +vierzig 4 +fünfzig 5 +sechzig 6 +siebzig 7 +achtzig 8 +neunzig 9 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv new file mode 100644 index 000000000..a0f669b86 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv @@ -0,0 +1,8 @@ +zwanzig 20 +dreißig 30 +vierzig 40 +fünfzig 50 +sechzig 60 +siebzig 70 +achtzig 80 +neunzig 90 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv b/nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv new file mode 100644 index 000000000..683972262 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv @@ -0,0 +1,102 @@ +Nummer +Ziffer +Zahl +Kapitel +Ausgabe +Folge +Szene +Figur +Artikel +Abschnitt +Saison +Seite +Band +Lektion +Aufgabe +Frage +Vers +Absatz +Tabelle +Abbildung +Anhang +Formular +Verordnung +Prüfungsteil +Teil +Staffel +Level +Track +Spieler +Runde +Lauf +Gruppe +Platz +Gleis +Gate +Reihe +Sitz +Wagen +Abschnitt +Zeile +Spalte +Ordner +Feld +Zeugnis +Karte +Kästchen +Modul +Punkt +Eintrag +Position +Blatt +Schritt +Bild +Stelle +Register +Bereich +Schlüssel +Version +Format +Typ +Kategorie +Code +Kurs +Thema +Schulfach +Klasse +Stufe +Jahrgang +Semester +Trimester +Übung +Projekt +Versuch +Experiment +Auflage +Druck +Schublade +Fach +Raum +Zimmer +Etage +Stockwerk +Hausnummer +Tür +Abteil +Sektor +Zone +Linie +Bus +Bahn +Flug +Flugnummer +Sitzplatz +Reiseziel +Lager +Palette +Container +Paket +Punktestand +Rang +Abzeichen +Autobahn \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py new file mode 100644 index 000000000..3068ebfeb --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py @@ -0,0 +1,334 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright 2015 and onwards Google, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import string +from pathlib import Path +from typing import Dict + +import pynini +from pynini import Far +from pynini.examples import plurals +from pynini.export import export +from pynini.lib import byte, pynutil, utf8 + +from nemo_text_processing.text_normalization.en.utils import get_abs_path, load_labels +from nemo_text_processing.utils.logging import logger + +NEMO_CHAR = utf8.VALID_UTF8_CHAR + +NEMO_DIGIT = byte.DIGIT +NEMO_LOWER = pynini.union(*string.ascii_lowercase).optimize() +NEMO_UPPER = pynini.union(*string.ascii_uppercase).optimize() +NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() +NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() +NEMO_HEX = pynini.union(*string.hexdigits).optimize() +NEMO_NON_BREAKING_SPACE = "\u00A0" +NEMO_SPACE = " " +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize() +NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() +NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() + +NEMO_PUNCT = pynini.union(*map(pynini.escape, string.punctuation)).optimize() +NEMO_GRAPH = pynini.union(NEMO_ALNUM, NEMO_PUNCT).optimize() + +NEMO_SIGMA = pynini.closure(NEMO_CHAR) +NEMO_LOWER_NOT_A = pynini.union( + "b", + "c", + "d", + "e", + "f", + "g", + "h", + "i", + "j", + "k", + "l", + "m", + "n", + "o", + "p", + "q", + "r", + "s", + "t", + "u", + "v", + "w", + "x", + "y", + "z", +).optimize() + +delete_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE)) +delete_zero_or_one_space = pynutil.delete(pynini.closure(NEMO_WHITE_SPACE, 0, 1)) +insert_space = pynutil.insert(" ") +delete_extra_space = pynini.cross(pynini.closure(NEMO_WHITE_SPACE, 1), " ") +delete_preserve_order = pynini.closure( + pynutil.delete(" preserve_order: true") + | (pynutil.delete(' field_order: "') + NEMO_NOT_QUOTE + pynutil.delete('"')) +) + +suppletive = pynini.string_file(get_abs_path("data/suppletive.tsv")) +# _v = pynini.union("a", "e", "i", "o", "u") +_c = pynini.union( + "b", + "c", + "d", + "f", + "g", + "h", + "j", + "k", + "l", + "m", + "n", + "p", + "q", + "r", + "s", + "t", + "v", + "w", + "x", + "y", + "z", +) +_ies = NEMO_SIGMA + _c + pynini.cross("y", "ies") +_es = NEMO_SIGMA + pynini.union("s", "sh", "ch", "x", "z") + pynutil.insert("es") +_s = NEMO_SIGMA + pynutil.insert("s") + +graph_plural = plurals._priority_union( + suppletive, + plurals._priority_union( + _ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA + ), + NEMO_SIGMA, +).optimize() + +SINGULAR_TO_PLURAL = graph_plural +PLURAL_TO_SINGULAR = pynini.invert(graph_plural) +TO_LOWER = pynini.union( + *[ + pynini.cross(x, y) + for x, y in zip(string.ascii_uppercase, string.ascii_lowercase) + ] +) +TO_UPPER = pynini.invert(TO_LOWER) +MIN_NEG_WEIGHT = -0.0001 +MIN_POS_WEIGHT = 0.0001 +INPUT_CASED = "cased" +INPUT_LOWER_CASED = "lower_cased" +MINUS = pynini.union("minus", "Minus").optimize() + + +def capitalized_input_graph( + graph: "pynini.FstLike", + original_graph_weight: float = None, + capitalized_graph_weight: float = None, +) -> "pynini.FstLike": + """ + Allow graph input to be capitalized, e.g. for ITN) + + Args: + graph: FstGraph + original_graph_weight: weight to add to the original `graph` + capitalized_graph_weight: weight to add to the capitalized graph + """ + capitalized_graph = pynini.compose(TO_LOWER + NEMO_SIGMA, graph).optimize() + + if original_graph_weight is not None: + graph = pynutil.add_weight(graph, weight=original_graph_weight) + + if capitalized_graph_weight is not None: + capitalized_graph = pynutil.add_weight( + capitalized_graph, weight=capitalized_graph_weight + ) + + graph |= capitalized_graph + return graph + + +def generator_main(file_name: str, graphs: Dict[str, "pynini.FstLike"]): + """ + Exports graph as OpenFst finite state archive (FAR) file with given file name and rule name. + + Args: + file_name: exported file name + graphs: Mapping of a rule name and Pynini WFST graph to be exported + """ + exporter = export.Exporter(file_name) + for rule, graph in graphs.items(): + exporter[rule] = graph.optimize() + exporter.close() + logger.info(f"Created {file_name}") + + +def get_plurals(fst): + """ + Given singular returns plurals + + Args: + fst: Fst + + Returns plurals to given singular forms + """ + return SINGULAR_TO_PLURAL @ fst + + +def get_singulars(fst): + """ + Given plural returns singulars + + Args: + fst: Fst + + Returns singulars to given plural forms + """ + return PLURAL_TO_SINGULAR @ fst + + +def convert_space(fst) -> "pynini.FstLike": + """ + Converts space to nonbreaking space. + Used only in tagger grammars for transducing token values within quotes, e.g. name: "hello kitty" + This is making transducer significantly slower, so only use when there could be potential spaces within quotes, otherwise leave it. + + Args: + fst: input fst + + Returns output fst where breaking spaces are converted to non breaking spaces + """ + return fst @ pynini.cdrewrite( + pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA + ) + + +def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): + labels = load_labels(input_file) + + if input_case == INPUT_CASED: + additional_labels = [] + for written, spoken, *weight in labels: + written_capitalized = written[0].upper() + written[1:] + additional_labels.extend( + [ + [ + written_capitalized, + spoken.capitalize(), + ], # first letter capitalized + [ + written_capitalized, + spoken.upper().replace(" AND ", " and "), + ], # # add pairs with the all letters capitalized + ] + ) + + spoken_no_space = spoken.replace(" ", "") + # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W" + if len(spoken) == (2 * len(spoken_no_space) - 1): + logger.debug(f"This is weight {weight}") + if len(weight) == 0: + additional_labels.extend( + [ + [written, spoken_no_space], + [written_capitalized, spoken_no_space.upper()], + ] + ) + else: + additional_labels.extend( + [ + [written, spoken_no_space, weight[0]], + [written_capitalized, spoken_no_space.upper(), weight[0]], + ] + ) + labels += additional_labels + + whitelist = pynini.string_map(labels).invert().optimize() + return whitelist + + +class GraphFst: + """ + Base class for all grammar fsts. + + Args: + name: name of grammar class + kind: either 'classify' or 'verbalize' + deterministic: if True will provide a single transduction option, + for False multiple transduction are generated (used for audio-based normalization) + """ + + def __init__(self, name: str, kind: str, deterministic: bool = True): + self.name = name + self.kind = kind + self._fst = None + self.deterministic = deterministic + + self.far_path = Path( + os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far" + ) + if self.far_exist(): + self._fst = Far( + self.far_path, mode="r", arc_type="standard", far_type="default" + ).get_fst() + + def far_exist(self) -> bool: + """ + Returns true if FAR can be loaded + """ + return self.far_path.exists() + + @property + def fst(self) -> "pynini.FstLike": + return self._fst + + @fst.setter + def fst(self, fst): + self._fst = fst + + def add_tokens(self, fst) -> "pynini.FstLike": + """ + Wraps class name around to given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + return pynutil.insert(f"{self.name} {{ ") + fst + pynutil.insert(" }") + + def delete_tokens(self, fst) -> "pynini.FstLike": + """ + Deletes class name wrap around output of given fst + + Args: + fst: input fst + + Returns: + Fst: fst + """ + res = ( + pynutil.delete(f"{self.name}") + + delete_space + + pynutil.delete("{") + + delete_space + + fst + + delete_space + + pynutil.delete("}") + ) + return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 46fdca4e3..4a42b4b0f 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,57 +12,381 @@ # See the License for the specific language governing permissions and # limitations under the License. + import pynini from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_SIGMA, GraphFst +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path +from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( + NEMO_DIGIT, + NEMO_SIGMA, + NEMO_SPACE, + GraphFst, +) class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinals. Numbers below ten are not converted. - Allows both compound numeral strings or separated by whitespace. - "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. - - e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } } - e.g. minus dreiundzwanzig -> cardinal { integer: "23" } } - e.g. dreizehn -> cardinal { integer: "13" } } - e.g. ein hundert -> cardinal { integer: "100" } } - e.g. einhundert -> cardinal { integer: "100" } } - e.g. ein tausend -> cardinal { integer: "1000" } } - e.g. eintausend -> cardinal { integer: "1000" } } - e.g. ein tausend zwanzig -> cardinal { integer: "1020" } } - - Args: - tn_cardinal_tagger: TN cardinal tagger + Finite state transducer for classifying cardinal numbers, e.g. + minus eine billion fünfundsechzig milliarden vier millionen sechs -> cardinal { negative: '-' integer: "1.065.004.000.006" } + The transducer implements a period separator every three digits by default. """ - def __init__(self, tn_cardinal_tagger: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="classify", deterministic=deterministic) + def __init__(self): + super().__init__(name="cardinal", kind="classify") + + # WFST mappings for numbers 0-99 + zero = pynini.string_map([("null", "0")]) + digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) + # Isolates single digit cardinals to pass to other graphs + self.digits = digits.optimize() + to_denormalize = pynini.string_file( + get_abs_path("data/cardinal/denormalized.tsv") + ) + # Isolates the first dozen + self.dozen = to_denormalize.optimize() + teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) + tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) + ties = pynini.string_file(get_abs_path("data/cardinal/ties.tsv")) + # German flips ones and tens in two-digit numbers. The WFST below handles these flips. + flips = pynini.string_file(get_abs_path("data/cardinal/flips.tsv")) + delete_space = pynutil.delete(NEMO_SPACE) + delete_und = pynutil.delete("und") + + # Accepts normalized digits+ties (ein+und+zwanzig) + digit_ties = digits + delete_space.ques + delete_und + delete_space.ques + tens + # Flips ties and digits for denormalization + ties_digit = digit_ties @ flips + + # WFST grammar for hundreds + graph_10_99 = teens | ties | ties_digit + self.graph_double_digits = graph_10_99 + # Isolates single and double-digit cardinals to pass to other graphs + graph_single_and_double_digits = digits | graph_10_99 + self.graph_single_and_double_digits = graph_single_and_double_digits.optimize() + + hundert = pynini.accep("hundert") | pynini.accep("ein hundert") + hundreds = (pynini.cross(hundert, "100")) | ( + ( + (digits | pynutil.insert("1")) + + delete_space.ques + + pynutil.delete("hundert") + + delete_space.ques + + delete_und.ques + + delete_space.ques + + graph_10_99 + ) + | ( + (digits | pynutil.insert("1")) + + delete_space.ques + + pynini.cross("hundert", "0") + + delete_space.ques + + delete_und.ques + + delete_space.ques + + digits + ) + | ( + (digits | pynutil.insert("1")) + + delete_space.ques + + pynini.cross("hundert", "00") + ) + ) + + # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. + # Clusters of three are separated by periods, applied right to left. + digit_cluster = ( + (hundreds) + | (pynutil.insert("0") + graph_10_99) + | (pynutil.insert("00") + digits) + | (pynutil.insert("000")) + ) + # The subgraph below introduces three-digit clusters containing at least one non-zero digit. + # It is mainly utilized by the "years" subgraph in the DATE class. + non_zero_digit_cluster = ( + (hundreds) + | (pynutil.insert("0") + graph_10_99) + | (pynutil.insert("00") + digits) + ) + + # WFST grammar for thousands + thousands = (pynini.cross("tausend", "1.000")) | ( + ( + (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) + | ( + digit_cluster + + delete_space.ques + + pynini.cross("tausend", ".") + + delete_und.ques + ) + | pynutil.insert("000.") + ) + + delete_space.ques + + digit_cluster + ) + + non_zero_thousands = (pynini.cross("tausend", "1.000")) | ( + ( + (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) + | ( + non_zero_digit_cluster + + delete_space.ques + + pynini.cross("tausend", ".") + + delete_und.ques + ) + # | pynutil.insert("000.") + ) + + delete_space.ques + + digit_cluster + ) + + # WFST grammar for millions + million = pynini.accep("million") | pynini.accep("millionen") + millions = (pynini.cross("million", "1.000.000")) | ( + ( + (pynini.cross("million", "1.") + delete_space.ques + delete_und.ques) + | ( + digit_cluster + + delete_space.ques + + pynini.cross(million, ".") + + delete_und.ques + ) + | pynutil.insert("000.") + ) + + delete_space.ques + + thousands + ) + + # WFST grammar for billions + billion = ( + pynini.accep("milliarde") + | pynini.accep("milliarden") + # include the consonant-final stem for ordinal declensions e.g "milliardste" + # "e" -> "" / _[ordinal morpheme] + | pynini.accep("milliard") + ) + billions = (pynini.cross("milliarde", "1.000.000.000")) | ( + ( + ( + pynini.cross( + (pynini.accep("milliarde") | pynini.accep("milliard")), "1." + ) + + delete_space.ques + + delete_und.ques + ) + | ( + digit_cluster + + delete_space.ques + + pynini.cross(billion, ".") + + delete_und.ques + ) + | pynutil.insert("000.") + ) + + delete_space.ques + + millions + ) + + # WFST grammar for trillions + trillion = pynini.accep("billion") | pynini.accep("billionen") + trillions = (pynini.cross("billion", "1.000.000.000.000")) | ( + ( + (pynini.cross("billion", "1.") + delete_space.ques + delete_und.ques) + | ( + digit_cluster + + delete_space.ques + + pynini.cross(trillion, ".") + + delete_und.ques + ) + | pynutil.insert("000.") + ) + + delete_space.ques + + billions + ) + + # WFST grammar for quadrillions + quadrillion = ( + pynini.accep("billiarde") + | pynini.accep("billiarden") + # include the consonant-final stem for ordinal declensions e.g "billiardste" + # "e" -> "" / _[ordinal morpheme] + | pynini.accep("billiard") + ) + quadrillions = (pynini.cross("billiarde", "1.000.000.000.000.000")) | ( + ( + (pynini.cross(quadrillion, "1.") + delete_space.ques + delete_und.ques) + | ( + digit_cluster + + delete_space.ques + + pynini.cross(quadrillion, ".") + + delete_und.ques + ) + | pynutil.insert("000.") + ) + + delete_space.ques + + trillions + ) + + # WFST grammar for quintillions + quintillion = pynini.accep("trillion") | pynini.accep("trillionen") + quintillions = (pynini.cross("trillion", "1.000.000.000.000.000.000")) | ( + ( + (pynini.cross("trillion", "1.") + delete_space.ques + delete_und.ques) + | ( + digit_cluster + + delete_space.ques + + pynini.cross(quintillion, ".") + + delete_und.ques + ) + | pynutil.insert("000.") + ) + + delete_space.ques + + quadrillions + ) - # add_space_between_chars = pynini.cdrewrite(pynini.closure(insert_space, 0, 1), NEMO_CHAR, NEMO_CHAR, NEMO_SIGMA) - optional_delete_space = pynini.closure(NEMO_SIGMA | pynutil.delete(" ")) + # WFST grammar for sextillions + sextillion = ( + pynini.accep("trilliarde") + | pynini.accep("trilliarden") + # include the consonant-final stem for ordinal declensions e.g "trilliardste" + # "e" -> "" / _[ordinal morpheme] + | pynini.accep("trilliard") + ) + sextillions = (pynini.cross("billiarde", "1.000.000.000.000.000.000.000")) | ( + ( + (pynini.cross(sextillion, "1.") + delete_space.ques + delete_und.ques) + | ( + digit_cluster + + delete_space.ques + + pynini.cross(sextillion, ".") + + delete_und.ques + ) + | pynutil.insert("000.") + ) + + delete_space.ques + + quintillions + ) - graph = (tn_cardinal_tagger.graph @ optional_delete_space).invert().optimize() - self.graph_hundred_component_at_least_one_none_zero_digit = ( - (tn_cardinal_tagger.graph_hundred_component_at_least_one_none_zero_digit @ optional_delete_space) - .invert() - .optimize() + # Remove the leading zeros + non_zero_digits = pynini.difference(NEMO_DIGIT, "0") + chars_to_remove = pynini.accep("0") | pynini.accep(".") + remove_chars = pynutil.delete(pynini.closure(chars_to_remove)) + remove_leading_zeros = pynini.cdrewrite( + remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA ) - self.graph_ties = (tn_cardinal_tagger.two_digit_non_zero @ optional_delete_space).invert().optimize() - # this is to make sure if there is an ambiguity with decimal, decimal is chosen, e.g. 1000000 vs. 1 million - graph = pynutil.add_weight(graph, weight=0.001) - self.graph_no_exception = graph - self.digit = pynini.arcmap(tn_cardinal_tagger.digit, map_type="rmweight").invert().optimize() - graph_exception = pynini.project(self.digit, 'input') - self.graph = (pynini.project(graph, "input") - graph_exception.arcsort()) @ graph + # All together now + grammars = [ + sextillions, + quintillions, + quadrillions, + trillions, + billions, + millions, + thousands, + digit_cluster, + zero, + ] - self.optional_minus_graph = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", "\"-\" "), 0, 1 + graph_cardinals = "" + for grammar in grammars: + graph_cardinals |= grammar + + # Generates a graph accepting all digits to be passed to other semiotic classes + graph_everything = graph_cardinals @ remove_leading_zeros + self.graph_all_cardinals = graph_everything.optimize() + + # Generates a graph denormalizing years from 0 to 9999 + # The graph will be passed into other semiotic classes + # Years 0 - 999 denormalize as regular cardinals + first_millenium = non_zero_digit_cluster # | zero + second_tenth_millenium = non_zero_thousands + # The graph below covers exceptions + # e.g. years 1100 - 1999 + # and all colloquial expresions (e.g. zwanzigvierundzwanzig -> 2024) + single_digit_years = pynini.string_file( + get_abs_path("data/cardinal/single_digit_years.tsv") + ) + ten = "zehn" + remove_ten = pynini.project(graph_10_99, "input") - ten + graph_11_99 = remove_ten @ graph_10_99 + + years_exceptions = ( + graph_11_99 + + pynutil.delete(NEMO_SPACE).ques + + pynutil.delete("hundert").ques + + pynutil.delete(NEMO_SPACE).ques + + (single_digit_years | graph_10_99 | pynutil.insert("00")) + ) + years = first_millenium | second_tenth_millenium | years_exceptions + remove_period_separators = pynini.cdrewrite( + pynutil.delete("."), "", "", NEMO_SIGMA ) + years = years @ remove_leading_zeros @ remove_period_separators + self.graph_years = years.optimize() - final_graph = self.optional_minus_graph + pynutil.insert("integer: \"") + self.graph + pynutil.insert("\"") + # The block below leaves numerals 1 - 12 canonically normalized + accept_denormalized_first_dozen = pynini.project( + to_denormalize, "input" + ) # acceptor for null - zwölf + accept_denormalized_everything = pynini.project( + self.graph_all_cardinals, "input" + ) # acceptor for all verbalized cardinals + accept_without_first_dozen = ( + accept_denormalized_everything - accept_denormalized_first_dozen + ) # acceptor for all verbalized cardinals greater than 12 + transduce_without_first_dozen = ( + accept_without_first_dozen @ self.graph_all_cardinals + ) # transducer for all verbalized cardinals greater than 12 + graph = accept_denormalized_first_dozen | transduce_without_first_dozen + self.graph = graph.optimize() - final_graph = self.add_tokens(final_graph) + self.optional_negative = pynini.closure( + pynutil.insert("negative: ") + + pynini.cross("minus ", '"-"') + + pynutil.insert(" "), + 0, + 1, + ) + + all_cardinals_graph = ( + self.optional_negative + + pynutil.insert('integer: "') + + self.graph_all_cardinals + + pynutil.insert('"') + ) + self.all_cardinals_graph = all_cardinals_graph.optimize() + + # The final graph for this semiotic class leaves the first dozen normalized + final_graph = ( + self.optional_negative + + pynutil.insert('integer: "') + + self.graph + + pynutil.insert('"') + ) + + # The block below handles noun + number combinations, where the noun forces full denormalization + # The nouns are implemented as a .tsv list + nouns_forcing_denormalization = pynini.string_file( + get_abs_path("data/measure/nouns_forcing_denormalization.tsv") + ) + graph_forced_denormalization = ( + pynutil.insert("morphosyntactic_features: ") + + pynutil.insert('"') + + nouns_forcing_denormalization + + pynutil.insert('"') + + pynini.accep(NEMO_SPACE) + + all_cardinals_graph + ) + + # Canonical representation with the first dozen normalized + self.canonical_cardinals_graph = final_graph.optimize() + + # Updated final graph to account for DPs with a denormalization-inducing noun in the D' position + updated_final_graph = (final_graph | graph_forced_denormalization).optimize() + + final_graph = self.add_tokens(updated_final_graph) self.fst = final_graph.optimize() + + self.graph_no_exception = self.graph_all_cardinals + self.optional_minus_graph = self.optional_negative + self.graph_hundred_component_at_least_one_none_zero_digit = self.graph_all_cardinals + self.digit = self.digits + self.graph_ties = self.graph_double_digits diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py index 1d60d071a..e0dbc4a42 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/tokenize_and_classify.py @@ -93,7 +93,7 @@ def __init__( tn_electronic_verbalizer = TNElectronicVerbalizer(deterministic=False) tn_whitelist_tagger = TNWhitelistTagger(input_case="cased", deterministic=False, input_file=whitelist) - cardinal = CardinalFst(tn_cardinal_tagger=tn_cardinal_tagger) + cardinal = CardinalFst() cardinal_graph = cardinal.fst ordinal = OrdinalFst(itn_cardinal_tagger=cardinal, tn_ordinal_verbalizer=tn_ordinal_verbalizer) diff --git a/nemo_text_processing/inverse_text_normalization/de/utils.py b/nemo_text_processing/inverse_text_normalization/de/utils.py new file mode 100644 index 000000000..78fdd87f2 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/utils.py @@ -0,0 +1,27 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def get_abs_path(rel_path): + """ + Get absolute path + + Args: + rel_path: relative path to this file + + Returns absolute path + """ + return os.path.dirname(os.path.abspath(__file__)) + "/" + rel_path diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index b13382a8e..7fedff33a 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,23 +14,81 @@ import pynini from pynini.lib import pynutil - -from nemo_text_processing.text_normalization.en.graph_utils import NEMO_NOT_QUOTE, GraphFst +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path +from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( + GraphFst, + delete_space, + NEMO_SPACE, + NEMO_DIGIT, + NEMO_ALPHA, +) class CardinalFst(GraphFst): """ - Finite state transducer for verbalizing cardinal - e.g. cardinal { integer: "23" negative: "-" } -> -23 - - Args: - tn_cardinal_verbalizer: TN cardinal verbalizer + Finite state transducer for verbalizing cardinal numbers. Note that the verbalizer retains period-separated formatting. + e.g. 'cardinal { negative: "-" integer: "1.234.512.102" }' -> -1.234.512.102 """ - def __init__(self, tn_cardinal_verbalizer: GraphFst, deterministic: bool = True): - super().__init__(name="cardinal", kind="verbalize", deterministic=deterministic) - self.numbers = tn_cardinal_verbalizer.numbers - optional_sign = pynini.closure(pynutil.delete("negative: \"") + NEMO_NOT_QUOTE + pynutil.delete("\" "), 0, 1) - graph = optional_sign + self.numbers - delete_tokens = self.delete_tokens(graph) + def __init__(self): + super().__init__(name="cardinal", kind="verbalize") + + DE_chars = pynini.union(*"äöüÄÖÜß").optimize() + + # removes the 'negative:' label and leaves the optional '-' sign in place + optional_minus = pynini.closure( + pynutil.delete("negative:") + + pynutil.delete(NEMO_SPACE) + + pynutil.delete('"') + + pynini.accep("-") + + pynutil.delete('"') + + pynutil.delete(NEMO_SPACE), + 0, + 1, + ) + + # handles all elements of a cardinal integer + integer_chars = NEMO_DIGIT | pynini.accep(".") + cardinal_components = NEMO_DIGIT | NEMO_ALPHA | DE_chars | pynini.accep(".") + + # removes the 'integer:' label + just_integers = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(integer_chars, 1) + + pynutil.delete('"') + + delete_space + ) + + # handles the canonical representation with the first dozen normalized + first_dozen_verbalized = ( + pynutil.delete("integer:") + + delete_space + + pynutil.delete('"') + + pynini.closure(cardinal_components, 1) + + pynutil.delete('"') + + delete_space + ) + + # Handles noun + number combinations, where the noun forces full denormalization + # The nouns are implemented as a .tsv list + nouns_forcing_denormalization = pynini.string_file( + get_abs_path("data/measure/nouns_forcing_denormalization.tsv") + ) + graph_forced_denormalization = ( + pynutil.delete("morphosyntactic_features: ") + + pynutil.delete('"') + + nouns_forcing_denormalization + + pynutil.delete('"') + + pynini.accep(NEMO_SPACE) + + just_integers + ) + + graph = optional_minus + just_integers + self.numbers = graph.optimize() + first_dozen = (optional_minus + first_dozen_verbalized).optimize() + self.first_dozen = first_dozen + updated_cardinals = (first_dozen | graph_forced_denormalization).optimize() + delete_tokens = self.delete_tokens(updated_cardinals) self.fst = delete_tokens.optimize() diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py index 9c921a63a..7aa670991 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/verbalize.py @@ -17,7 +17,6 @@ from nemo_text_processing.inverse_text_normalization.de.verbalizers.measure import MeasureFst from nemo_text_processing.inverse_text_normalization.de.verbalizers.money import MoneyFst from nemo_text_processing.inverse_text_normalization.de.verbalizers.time import TimeFst -from nemo_text_processing.text_normalization.de.verbalizers.cardinal import CardinalFst as TNCardinalVerbalizer from nemo_text_processing.text_normalization.de.verbalizers.decimal import DecimalFst as TNDecimalVerbalizer from nemo_text_processing.text_normalization.en.graph_utils import GraphFst @@ -31,10 +30,9 @@ class VerbalizeFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="verbalize", kind="verbalize", deterministic=deterministic) - tn_cardinal_verbalizer = TNCardinalVerbalizer(deterministic=False) tn_decimal_verbalizer = TNDecimalVerbalizer(deterministic=False) - cardinal = CardinalFst(tn_cardinal_verbalizer=tn_cardinal_verbalizer) + cardinal = CardinalFst() cardinal_graph = cardinal.fst decimal = DecimalFst(tn_decimal_verbalizer=tn_decimal_verbalizer) decimal_graph = decimal.fst diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index 0b2064296..d42c48339 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -4,26 +4,26 @@ ein hundert und zwei~102 einhundertzwei~102 ein hundert und zwanzig~120 ein hundert und elf~111 -ein tausend~1000 -eintausend~1000 +ein tausend~1.000 +eintausend~1.000 ein hundert zwanzig~120 -ein tausend zwanzig~1020 -eintausendzwanzig~1020 -neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9789382536130 +ein tausend zwanzig~1.020 +eintausendzwanzig~1.020 +neun billionen sieben hundert neun und achtzig milliarden drei hundert zwei und achtzig millionen fünf hundert sechs und dreißig tausend ein hundert dreißig~9.789.382.536.130 zwei hundert vier und fünfzig~254 -ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147451 -eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1156173 -eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1593072961 -sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97808264772792005 -zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10010000010100010 -zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10010000010100010 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünf und zwanzig tausend sieben und dreißig~-25037 -minus fünfundzwanzigtausendsiebenunddreißig~-25037 -eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1264301938000104 -eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1264301938000104 +ein hundert sieben und vierzig tausend vier hundert ein und fünfzig~147.451 +eine million ein hundert sechs und fünfzig tausend ein hundert drei und siebzig~1.156.173 +eine milliarde fünf hundert drei und neunzig millionen zwei und siebzig tausend neun hundert ein und sechzig~1.593.072.961 +sieben und neunzig billiarden acht hundert acht billionen zwei hundert vier und sechzig milliarden sieben hundert zwei und siebzig millionen sieben hundert zwei und neunzig tausend fünf~97.808.264.772.792.005 +zehn billiarden zehn billionen zehn millionen ein hundert tausend zehn~10.010.000.010.100.010 +zehn billiarden zehn billionen zehn millionen einhunderttausendzehn~10.010.000.010.100.010 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünf und zwanzig tausend sieben und dreißig~-25.037 +minus fünfundzwanzigtausendsiebenunddreißig~-25.037 +eine billiarde zwei hundert vier und sechzig billionen drei hundert eins milliarden neun hundert acht und dreißig millionen ein hundert vier~1.264.301.938.000.104 +eine billiarde zweihundertvierundsechzig billionen dreihunderteins milliarden neunhundertachtunddreißig millionen einhundertvier~1.264.301.938.000.104 minus sechzig~-60 -sechs und vierzig tausend sechs hundert vier und sechzig~46664 +sechsundvierzig tausend sechshundert vierundsechzig~46.664 sechzig~60 null~null eins~eins @@ -31,10 +31,12 @@ ein~ein eine~eine einer~einer zwei~zwei +zwö~zwö +zwo~zwo neun~neun -zehn~10 -elf~11 -zwölf~12 +zehn~zehn +elf~elf +zwölf~zwölf dreizehn~13 vierzehn~14 fünfzehn~15 @@ -49,14 +51,20 @@ sechzig~60 siebzig~70 achtzig~80 neunzig~90 -zwei millionen drei~2000003 -ein tausend dreizehn~1013 -ein tausend eins~1001 -ein tausend ein hundert~1100 -ein tausend sechs und zwanzig~1026 -ein tausend ein hundert sechs und zwanzig~1126 -achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18450990 -achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18940722 -achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18690916 -achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18690916 -achtzehn tausend acht hundert achtzig~18880 +zwei millionen drei~2.000.003 +ein tausend dreizehn~1.013 +ein tausend eins~1.001 +ein tausend ein hundert~1.100 +ein tausend sechs und zwanzig~1.026 +ein tausend ein hundert sechs und zwanzig~1.126 +achtzehn millionen vier hundert fünfzig tausend neun hundert neunzig~18.450.990 +achtzehn millionen neun hundert vierzig tausend sieben hundert zwei und zwanzig~18.940.722 +achtzehn millionen sechs hundert neunzig tausend neun hundert sechzehn~18.690.916 +achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18.690.916 +achtzehn tausend acht hundert achtzig~18.880 +einhunderteins~101 +ein tausend einhundert sechsundzwanzig~1.126 +Kapitel vier~Kapitel 4 +Kapitel vierzehn~Kapitel 14 +Nummer acht~Nummer 8 +Abteil sechs~Abteil 6 \ No newline at end of file From 9ced7324f16361231e0d97ee8d32a8b9b9107e81 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:03:56 +0000 Subject: [PATCH 02/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../de/graph_utils.py | 33 ++--- .../de/taggers/cardinal.py | 118 ++++-------------- .../de/verbalizers/cardinal.py | 9 +- 3 files changed, 36 insertions(+), 124 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py index 3068ebfeb..8dc9c6391 100644 --- a/nemo_text_processing/inverse_text_normalization/de/graph_utils.py +++ b/nemo_text_processing/inverse_text_normalization/de/graph_utils.py @@ -35,9 +35,9 @@ NEMO_ALPHA = pynini.union(NEMO_LOWER, NEMO_UPPER).optimize() NEMO_ALNUM = pynini.union(NEMO_DIGIT, NEMO_ALPHA).optimize() NEMO_HEX = pynini.union(*string.hexdigits).optimize() -NEMO_NON_BREAKING_SPACE = "\u00A0" +NEMO_NON_BREAKING_SPACE = "\u00a0" NEMO_SPACE = " " -NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00A0").optimize() +NEMO_WHITE_SPACE = pynini.union(" ", "\t", "\n", "\r", "\u00a0").optimize() NEMO_NOT_SPACE = pynini.difference(NEMO_CHAR, NEMO_WHITE_SPACE).optimize() NEMO_NOT_QUOTE = pynini.difference(NEMO_CHAR, r'"').optimize() @@ -113,20 +113,13 @@ graph_plural = plurals._priority_union( suppletive, - plurals._priority_union( - _ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA - ), + plurals._priority_union(_ies, plurals._priority_union(_es, _s, NEMO_SIGMA), NEMO_SIGMA), NEMO_SIGMA, ).optimize() SINGULAR_TO_PLURAL = graph_plural PLURAL_TO_SINGULAR = pynini.invert(graph_plural) -TO_LOWER = pynini.union( - *[ - pynini.cross(x, y) - for x, y in zip(string.ascii_uppercase, string.ascii_lowercase) - ] -) +TO_LOWER = pynini.union(*[pynini.cross(x, y) for x, y in zip(string.ascii_uppercase, string.ascii_lowercase)]) TO_UPPER = pynini.invert(TO_LOWER) MIN_NEG_WEIGHT = -0.0001 MIN_POS_WEIGHT = 0.0001 @@ -154,9 +147,7 @@ def capitalized_input_graph( graph = pynutil.add_weight(graph, weight=original_graph_weight) if capitalized_graph_weight is not None: - capitalized_graph = pynutil.add_weight( - capitalized_graph, weight=capitalized_graph_weight - ) + capitalized_graph = pynutil.add_weight(capitalized_graph, weight=capitalized_graph_weight) graph |= capitalized_graph return graph @@ -212,9 +203,7 @@ def convert_space(fst) -> "pynini.FstLike": Returns output fst where breaking spaces are converted to non breaking spaces """ - return fst @ pynini.cdrewrite( - pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA - ) + return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA) def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED): @@ -278,13 +267,9 @@ def __init__(self, name: str, kind: str, deterministic: bool = True): self._fst = None self.deterministic = deterministic - self.far_path = Path( - os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far" - ) + self.far_path = Path(os.path.dirname(__file__) + "/grammars/" + kind + "/" + name + ".far") if self.far_exist(): - self._fst = Far( - self.far_path, mode="r", arc_type="standard", far_type="default" - ).get_fst() + self._fst = Far(self.far_path, mode="r", arc_type="standard", far_type="default").get_fst() def far_exist(self) -> bool: """ @@ -331,4 +316,4 @@ def delete_tokens(self, fst) -> "pynini.FstLike": + delete_space + pynutil.delete("}") ) - return res @ pynini.cdrewrite(pynini.cross("\u00A0", " "), "", "", NEMO_SIGMA) + return res @ pynini.cdrewrite(pynini.cross("\u00a0", " "), "", "", NEMO_SIGMA) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 4a42b4b0f..d2244171c 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -15,13 +15,9 @@ import pynini from pynini.lib import pynutil + +from nemo_text_processing.inverse_text_normalization.de.graph_utils import NEMO_DIGIT, NEMO_SIGMA, NEMO_SPACE, GraphFst from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path -from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( - NEMO_DIGIT, - NEMO_SIGMA, - NEMO_SPACE, - GraphFst, -) class CardinalFst(GraphFst): @@ -39,9 +35,7 @@ def __init__(self): digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) # Isolates single digit cardinals to pass to other graphs self.digits = digits.optimize() - to_denormalize = pynini.string_file( - get_abs_path("data/cardinal/denormalized.tsv") - ) + to_denormalize = pynini.string_file(get_abs_path("data/cardinal/denormalized.tsv")) # Isolates the first dozen self.dozen = to_denormalize.optimize() teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) @@ -84,11 +78,7 @@ def __init__(self): + delete_space.ques + digits ) - | ( - (digits | pynutil.insert("1")) - + delete_space.ques - + pynini.cross("hundert", "00") - ) + | ((digits | pynutil.insert("1")) + delete_space.ques + pynini.cross("hundert", "00")) ) # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. @@ -101,22 +91,13 @@ def __init__(self): ) # The subgraph below introduces three-digit clusters containing at least one non-zero digit. # It is mainly utilized by the "years" subgraph in the DATE class. - non_zero_digit_cluster = ( - (hundreds) - | (pynutil.insert("0") + graph_10_99) - | (pynutil.insert("00") + digits) - ) + non_zero_digit_cluster = (hundreds) | (pynutil.insert("0") + graph_10_99) | (pynutil.insert("00") + digits) # WFST grammar for thousands thousands = (pynini.cross("tausend", "1.000")) | ( ( (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) - | ( - digit_cluster - + delete_space.ques - + pynini.cross("tausend", ".") - + delete_und.ques - ) + | (digit_cluster + delete_space.ques + pynini.cross("tausend", ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques @@ -126,12 +107,7 @@ def __init__(self): non_zero_thousands = (pynini.cross("tausend", "1.000")) | ( ( (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) - | ( - non_zero_digit_cluster - + delete_space.ques - + pynini.cross("tausend", ".") - + delete_und.ques - ) + | (non_zero_digit_cluster + delete_space.ques + pynini.cross("tausend", ".") + delete_und.ques) # | pynutil.insert("000.") ) + delete_space.ques @@ -143,12 +119,7 @@ def __init__(self): millions = (pynini.cross("million", "1.000.000")) | ( ( (pynini.cross("million", "1.") + delete_space.ques + delete_und.ques) - | ( - digit_cluster - + delete_space.ques - + pynini.cross(million, ".") - + delete_und.ques - ) + | (digit_cluster + delete_space.ques + pynini.cross(million, ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques @@ -166,18 +137,11 @@ def __init__(self): billions = (pynini.cross("milliarde", "1.000.000.000")) | ( ( ( - pynini.cross( - (pynini.accep("milliarde") | pynini.accep("milliard")), "1." - ) - + delete_space.ques - + delete_und.ques - ) - | ( - digit_cluster + pynini.cross((pynini.accep("milliarde") | pynini.accep("milliard")), "1.") + delete_space.ques - + pynini.cross(billion, ".") + delete_und.ques ) + | (digit_cluster + delete_space.ques + pynini.cross(billion, ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques @@ -189,12 +153,7 @@ def __init__(self): trillions = (pynini.cross("billion", "1.000.000.000.000")) | ( ( (pynini.cross("billion", "1.") + delete_space.ques + delete_und.ques) - | ( - digit_cluster - + delete_space.ques - + pynini.cross(trillion, ".") - + delete_und.ques - ) + | (digit_cluster + delete_space.ques + pynini.cross(trillion, ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques @@ -212,12 +171,7 @@ def __init__(self): quadrillions = (pynini.cross("billiarde", "1.000.000.000.000.000")) | ( ( (pynini.cross(quadrillion, "1.") + delete_space.ques + delete_und.ques) - | ( - digit_cluster - + delete_space.ques - + pynini.cross(quadrillion, ".") - + delete_und.ques - ) + | (digit_cluster + delete_space.ques + pynini.cross(quadrillion, ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques @@ -229,12 +183,7 @@ def __init__(self): quintillions = (pynini.cross("trillion", "1.000.000.000.000.000.000")) | ( ( (pynini.cross("trillion", "1.") + delete_space.ques + delete_und.ques) - | ( - digit_cluster - + delete_space.ques - + pynini.cross(quintillion, ".") - + delete_und.ques - ) + | (digit_cluster + delete_space.ques + pynini.cross(quintillion, ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques @@ -252,12 +201,7 @@ def __init__(self): sextillions = (pynini.cross("billiarde", "1.000.000.000.000.000.000.000")) | ( ( (pynini.cross(sextillion, "1.") + delete_space.ques + delete_und.ques) - | ( - digit_cluster - + delete_space.ques - + pynini.cross(sextillion, ".") - + delete_und.ques - ) + | (digit_cluster + delete_space.ques + pynini.cross(sextillion, ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques @@ -268,9 +212,7 @@ def __init__(self): non_zero_digits = pynini.difference(NEMO_DIGIT, "0") chars_to_remove = pynini.accep("0") | pynini.accep(".") remove_chars = pynutil.delete(pynini.closure(chars_to_remove)) - remove_leading_zeros = pynini.cdrewrite( - remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA - ) + remove_leading_zeros = pynini.cdrewrite(remove_chars, "[BOS]", non_zero_digits, NEMO_SIGMA) # All together now grammars = [ @@ -301,9 +243,7 @@ def __init__(self): # The graph below covers exceptions # e.g. years 1100 - 1999 # and all colloquial expresions (e.g. zwanzigvierundzwanzig -> 2024) - single_digit_years = pynini.string_file( - get_abs_path("data/cardinal/single_digit_years.tsv") - ) + single_digit_years = pynini.string_file(get_abs_path("data/cardinal/single_digit_years.tsv")) ten = "zehn" remove_ten = pynini.project(graph_10_99, "input") - ten graph_11_99 = remove_ten @ graph_10_99 @@ -316,16 +256,12 @@ def __init__(self): + (single_digit_years | graph_10_99 | pynutil.insert("00")) ) years = first_millenium | second_tenth_millenium | years_exceptions - remove_period_separators = pynini.cdrewrite( - pynutil.delete("."), "", "", NEMO_SIGMA - ) + remove_period_separators = pynini.cdrewrite(pynutil.delete("."), "", "", NEMO_SIGMA) years = years @ remove_leading_zeros @ remove_period_separators self.graph_years = years.optimize() # The block below leaves numerals 1 - 12 canonically normalized - accept_denormalized_first_dozen = pynini.project( - to_denormalize, "input" - ) # acceptor for null - zwölf + accept_denormalized_first_dozen = pynini.project(to_denormalize, "input") # acceptor for null - zwölf accept_denormalized_everything = pynini.project( self.graph_all_cardinals, "input" ) # acceptor for all verbalized cardinals @@ -339,28 +275,18 @@ def __init__(self): self.graph = graph.optimize() self.optional_negative = pynini.closure( - pynutil.insert("negative: ") - + pynini.cross("minus ", '"-"') - + pynutil.insert(" "), + pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" "), 0, 1, ) all_cardinals_graph = ( - self.optional_negative - + pynutil.insert('integer: "') - + self.graph_all_cardinals - + pynutil.insert('"') + self.optional_negative + pynutil.insert('integer: "') + self.graph_all_cardinals + pynutil.insert('"') ) self.all_cardinals_graph = all_cardinals_graph.optimize() # The final graph for this semiotic class leaves the first dozen normalized - final_graph = ( - self.optional_negative - + pynutil.insert('integer: "') - + self.graph - + pynutil.insert('"') - ) + final_graph = self.optional_negative + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') # The block below handles noun + number combinations, where the noun forces full denormalization # The nouns are implemented as a .tsv list @@ -389,4 +315,4 @@ def __init__(self): self.optional_minus_graph = self.optional_negative self.graph_hundred_component_at_least_one_none_zero_digit = self.graph_all_cardinals self.digit = self.digits - self.graph_ties = self.graph_double_digits + self.graph_ties = self.graph_double_digits diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index 7fedff33a..27dcfe27e 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -14,14 +14,15 @@ import pynini from pynini.lib import pynutil -from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path + from nemo_text_processing.inverse_text_normalization.de.graph_utils import ( + NEMO_ALPHA, + NEMO_DIGIT, + NEMO_SPACE, GraphFst, delete_space, - NEMO_SPACE, - NEMO_DIGIT, - NEMO_ALPHA, ) +from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path class CardinalFst(GraphFst): From ea1456a89d90ed12bc5c4d1847b00024e03c72bc Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 16:53:13 -0400 Subject: [PATCH 03/14] Split Cardinal denormalized.tsv into zero, digits, and irregular teens Signed-off-by: Adelina Dunina --- .../de/data/cardinal/denormalized.tsv | 17 ----------------- .../de/data/cardinal/irregular_teens.tsv | 3 +++ .../de/data/cardinal/zero.tsv | 1 + 3 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv deleted file mode 100644 index 89fa8ccff..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/denormalized.tsv +++ /dev/null @@ -1,17 +0,0 @@ -null 0 -eine 1 -eins 1 -ein 1 -zwei 2 -zwo 2 -zwö 2 -drei 3 -vier 4 -fünf 5 -sechs 6 -sieben 7 -acht 8 -neun 9 -zehn 10 -elf 11 -zwölf 12 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv new file mode 100644 index 000000000..520919081 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/irregular_teens.tsv @@ -0,0 +1,3 @@ +zehn 10 +elf 11 +zwölf 12 diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv new file mode 100644 index 000000000..973c00968 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/zero.tsv @@ -0,0 +1 @@ +null 0 From dceebc9998ed0fcbd014b7b84b745c7e1bc97f4e Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 17:01:14 -0400 Subject: [PATCH 04/14] Load Cardinal 0-12 from zero, digits, and irregular teens Signed-off-by: Adelina Dunina --- .../inverse_text_normalization/de/taggers/cardinal.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index d2244171c..2696ad861 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -31,11 +31,13 @@ def __init__(self): super().__init__(name="cardinal", kind="classify") # WFST mappings for numbers 0-99 - zero = pynini.string_map([("null", "0")]) + zero = pynini.string_file(get_abs_path("data/cardinal/zero.tsv")) digits = pynini.string_file(get_abs_path("data/cardinal/digits.tsv")) # Isolates single digit cardinals to pass to other graphs self.digits = digits.optimize() - to_denormalize = pynini.string_file(get_abs_path("data/cardinal/denormalized.tsv")) + irregular_teens = pynini.string_file(get_abs_path("data/cardinal/irregular_teens.tsv")) + # 0-12 stay as words: zero + digits (1-9) + irregular teens (10-12) + to_denormalize = zero | digits | irregular_teens # Isolates the first dozen self.dozen = to_denormalize.optimize() teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) From 17c642ca3df9d3d74118463fad95464b35991337 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 17:13:44 -0400 Subject: [PATCH 05/14] Replace Cardinal flips.tsv with a digit-tie flip function Signed-off-by: Adelina Dunina --- .../de/data/cardinal/flips.tsv | 72 ------------------- .../de/taggers/cardinal.py | 12 +++- 2 files changed, 10 insertions(+), 74 deletions(-) delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv deleted file mode 100644 index 16c4fd546..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/flips.tsv +++ /dev/null @@ -1,72 +0,0 @@ -12 21 -22 22 -32 23 -42 24 -52 25 -62 26 -72 27 -82 28 -92 29 -13 31 -23 32 -33 33 -43 34 -53 35 -63 36 -73 37 -83 38 -93 39 -14 41 -24 42 -34 43 -44 44 -54 45 -64 46 -74 47 -84 48 -94 49 -15 51 -25 52 -35 53 -45 54 -55 55 -65 56 -75 57 -85 58 -95 59 -16 61 -26 62 -36 63 -46 64 -56 65 -66 66 -76 67 -86 68 -96 69 -17 71 -27 72 -37 73 -47 74 -57 75 -67 76 -77 77 -87 78 -97 79 -18 81 -28 82 -38 83 -48 84 -58 85 -68 86 -78 87 -88 88 -98 89 -19 91 -29 92 -39 93 -49 94 -59 95 -69 96 -79 97 -89 98 -99 99 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 2696ad861..d4a28e522 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -20,6 +20,14 @@ from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path + +def _digit_tie_flips(): + """Map concatenated ones+tens (12 for einundzwanzig) to the written number (21).""" + return pynini.string_map( + [(f"{ones}{tens}", f"{tens}{ones}") for tens in range(2, 10) for ones in range(1, 10)] + ) + + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinal numbers, e.g. @@ -43,8 +51,8 @@ def __init__(self): teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) ties = pynini.string_file(get_abs_path("data/cardinal/ties.tsv")) - # German flips ones and tens in two-digit numbers. The WFST below handles these flips. - flips = pynini.string_file(get_abs_path("data/cardinal/flips.tsv")) + # German flips ones and tens in two-digit numbers (ein + zwanzig -> 21). + flips = _digit_tie_flips() delete_space = pynutil.delete(NEMO_SPACE) delete_und = pynutil.delete("und") From e7119b09b74ec2b1923bfb4a35188e18ee50a49c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:14:40 +0000 Subject: [PATCH 06/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../inverse_text_normalization/de/taggers/cardinal.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index d4a28e522..443bd5d0b 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -20,12 +20,9 @@ from nemo_text_processing.inverse_text_normalization.de.utils import get_abs_path - def _digit_tie_flips(): """Map concatenated ones+tens (12 for einundzwanzig) to the written number (21).""" - return pynini.string_map( - [(f"{ones}{tens}", f"{tens}{ones}") for tens in range(2, 10) for ones in range(1, 10)] - ) + return pynini.string_map([(f"{ones}{tens}", f"{tens}{ones}") for tens in range(2, 10) for ones in range(1, 10)]) class CardinalFst(GraphFst): From d51c2bb16afe2b56f41dc15eeb9777c976529e34 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 17:31:28 -0400 Subject: [PATCH 07/14] Remove Cardinal single_digit_years.tsv until Date Signed-off-by: Adelina Dunina --- .../de/data/cardinal/single_digit_years.tsv | 9 --------- .../inverse_text_normalization/de/taggers/cardinal.py | 3 +-- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv deleted file mode 100644 index 7d5893d20..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/single_digit_years.tsv +++ /dev/null @@ -1,9 +0,0 @@ -eins 01 -zwei 02 -drei 03 -vier 04 -fünf 05 -sechs 06 -sieben 07 -acht 08 -neun 09 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 443bd5d0b..7c4a29ca0 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -250,7 +250,6 @@ def __init__(self): # The graph below covers exceptions # e.g. years 1100 - 1999 # and all colloquial expresions (e.g. zwanzigvierundzwanzig -> 2024) - single_digit_years = pynini.string_file(get_abs_path("data/cardinal/single_digit_years.tsv")) ten = "zehn" remove_ten = pynini.project(graph_10_99, "input") - ten graph_11_99 = remove_ten @ graph_10_99 @@ -260,7 +259,7 @@ def __init__(self): + pynutil.delete(NEMO_SPACE).ques + pynutil.delete("hundert").ques + pynutil.delete(NEMO_SPACE).ques - + (single_digit_years | graph_10_99 | pynutil.insert("00")) + + (graph_10_99 | pynutil.insert("00")) ) years = first_millenium | second_tenth_millenium | years_exceptions remove_period_separators = pynini.cdrewrite(pynutil.delete("."), "", "", NEMO_SIGMA) From 4d529125e46b9eb68b38e23e7290ab8de7f2b125 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 17:35:19 -0400 Subject: [PATCH 08/14] Derive Cardinal ties from tens plus inserted 0 Signed-off-by: Adelina Dunina --- .../inverse_text_normalization/de/data/cardinal/ties.tsv | 8 -------- .../inverse_text_normalization/de/taggers/cardinal.py | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv deleted file mode 100644 index a0f669b86..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/ties.tsv +++ /dev/null @@ -1,8 +0,0 @@ -zwanzig 20 -dreißig 30 -vierzig 40 -fünfzig 50 -sechzig 60 -siebzig 70 -achtzig 80 -neunzig 90 \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 7c4a29ca0..70ac83d17 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -47,7 +47,8 @@ def __init__(self): self.dozen = to_denormalize.optimize() teens = pynini.string_file(get_abs_path("data/cardinal/teens.tsv")) tens = pynini.string_file(get_abs_path("data/cardinal/tens.tsv")) - ties = pynini.string_file(get_abs_path("data/cardinal/ties.tsv")) + # Standalone decades: tens digit (2) + 0 -> 20 + ties = tens + pynutil.insert("0") # German flips ones and tens in two-digit numbers (ein + zwanzig -> 21). flips = _digit_tie_flips() delete_space = pynutil.delete(NEMO_SPACE) From 1e72f3f7d93af7efd5ff5034c062ff1a7b6c9343 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 17:48:35 -0400 Subject: [PATCH 09/14] Drop Measure noun-forced denorm from Cardinal PR Signed-off-by: Adelina Dunina --- .../measure/nouns_forcing_denormalization.tsv | 102 ------------------ .../de/taggers/cardinal.py | 19 +--- .../test_cases_cardinal.txt | 2 - 3 files changed, 1 insertion(+), 122 deletions(-) delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv b/nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv deleted file mode 100644 index 683972262..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/measure/nouns_forcing_denormalization.tsv +++ /dev/null @@ -1,102 +0,0 @@ -Nummer -Ziffer -Zahl -Kapitel -Ausgabe -Folge -Szene -Figur -Artikel -Abschnitt -Saison -Seite -Band -Lektion -Aufgabe -Frage -Vers -Absatz -Tabelle -Abbildung -Anhang -Formular -Verordnung -Prüfungsteil -Teil -Staffel -Level -Track -Spieler -Runde -Lauf -Gruppe -Platz -Gleis -Gate -Reihe -Sitz -Wagen -Abschnitt -Zeile -Spalte -Ordner -Feld -Zeugnis -Karte -Kästchen -Modul -Punkt -Eintrag -Position -Blatt -Schritt -Bild -Stelle -Register -Bereich -Schlüssel -Version -Format -Typ -Kategorie -Code -Kurs -Thema -Schulfach -Klasse -Stufe -Jahrgang -Semester -Trimester -Übung -Projekt -Versuch -Experiment -Auflage -Druck -Schublade -Fach -Raum -Zimmer -Etage -Stockwerk -Hausnummer -Tür -Abteil -Sektor -Zone -Linie -Bus -Bahn -Flug -Flugnummer -Sitzplatz -Reiseziel -Lager -Palette -Container -Paket -Punktestand -Rang -Abzeichen -Autobahn \ No newline at end of file diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 70ac83d17..f610e2706 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -295,27 +295,10 @@ def __init__(self): # The final graph for this semiotic class leaves the first dozen normalized final_graph = self.optional_negative + pynutil.insert('integer: "') + self.graph + pynutil.insert('"') - # The block below handles noun + number combinations, where the noun forces full denormalization - # The nouns are implemented as a .tsv list - nouns_forcing_denormalization = pynini.string_file( - get_abs_path("data/measure/nouns_forcing_denormalization.tsv") - ) - graph_forced_denormalization = ( - pynutil.insert("morphosyntactic_features: ") - + pynutil.insert('"') - + nouns_forcing_denormalization - + pynutil.insert('"') - + pynini.accep(NEMO_SPACE) - + all_cardinals_graph - ) - # Canonical representation with the first dozen normalized self.canonical_cardinals_graph = final_graph.optimize() - # Updated final graph to account for DPs with a denormalization-inducing noun in the D' position - updated_final_graph = (final_graph | graph_forced_denormalization).optimize() - - final_graph = self.add_tokens(updated_final_graph) + final_graph = self.add_tokens(final_graph) self.fst = final_graph.optimize() self.graph_no_exception = self.graph_all_cardinals diff --git a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt index d42c48339..6572163db 100644 --- a/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt +++ b/tests/nemo_text_processing/de/data_inverse_text_normalization/test_cases_cardinal.txt @@ -64,7 +64,5 @@ achtzehn millionen sechshundertneunzigtausendneunhundertsechzehn~18.690.916 achtzehn tausend acht hundert achtzig~18.880 einhunderteins~101 ein tausend einhundert sechsundzwanzig~1.126 -Kapitel vier~Kapitel 4 -Kapitel vierzehn~Kapitel 14 Nummer acht~Nummer 8 Abteil sechs~Abteil 6 \ No newline at end of file From 90717ba171c4f6195e8d20c628f15141d7660668 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 19:06:44 -0400 Subject: [PATCH 10/14] Restore original CardinalFst tagger docstring and add examples Signed-off-by: Adelina Dunina --- .../de/taggers/cardinal.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index f610e2706..38d2491af 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -27,9 +27,19 @@ def _digit_tie_flips(): class CardinalFst(GraphFst): """ - Finite state transducer for classifying cardinal numbers, e.g. - minus eine billion fünfundsechzig milliarden vier millionen sechs -> cardinal { negative: '-' integer: "1.065.004.000.006" } - The transducer implements a period separator every three digits by default. + Finite state transducer for classifying cardinals. Numbers below ten are not converted. + Allows both compound numeral strings or separated by whitespace. + "und" (en: "and") can be inserted between "hundert" and following number or "tausend" and following single or double digit number. + + e.g. minus drei und zwanzig -> cardinal { negative: "-" integer: "23" } + e.g. minus dreiundzwanzig -> cardinal { negative: "-" integer: "23" } + e.g. dreizehn -> cardinal { integer: "13" } + e.g. ein hundert -> cardinal { integer: "100" } + e.g. einhundert -> cardinal { integer: "100" } + e.g. ein tausend -> cardinal { integer: "1.000" } + e.g. eintausend -> cardinal { integer: "1.000" } + e.g. ein tausend zwanzig -> cardinal { integer: "1.020" } + e.g. minus eine billion fünfundsechzig milliarden vier millionen sechs -> cardinal { negative: "-" integer: "1.065.004.000.006" } """ def __init__(self): From 4220d8273c828db0a4026125b9c521a2f7b8b7d1 Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 19:19:55 -0400 Subject: [PATCH 11/14] Load Cardinal German magnitude words from TSV files Signed-off-by: Adelina Dunina --- .../inverse_text_normalization/de/data/cardinal/billiarde.tsv | 3 +++ .../inverse_text_normalization/de/data/cardinal/billion.tsv | 2 ++ .../inverse_text_normalization/de/data/cardinal/hundert.tsv | 2 ++ .../inverse_text_normalization/de/data/cardinal/milliarde.tsv | 3 +++ .../inverse_text_normalization/de/data/cardinal/million.tsv | 2 ++ .../inverse_text_normalization/de/data/cardinal/minus.tsv | 1 + .../inverse_text_normalization/de/data/cardinal/tausend.tsv | 1 + .../inverse_text_normalization/de/data/cardinal/trilliarde.tsv | 3 +++ .../inverse_text_normalization/de/data/cardinal/trillion.tsv | 2 ++ .../inverse_text_normalization/de/data/cardinal/und.tsv | 1 + 10 files changed, 20 insertions(+) create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv new file mode 100644 index 000000000..fb1a150dc --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv @@ -0,0 +1,3 @@ +billiarde +billiarden +billiard diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv new file mode 100644 index 000000000..24bd342d0 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv @@ -0,0 +1,2 @@ +billion +billionen diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv new file mode 100644 index 000000000..1b9d5bc34 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv @@ -0,0 +1,2 @@ +hundert +ein hundert diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv new file mode 100644 index 000000000..b0be36378 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv @@ -0,0 +1,3 @@ +milliarde +milliarden +milliard diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv new file mode 100644 index 000000000..e3e031f7a --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv @@ -0,0 +1,2 @@ +million +millionen diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv new file mode 100644 index 000000000..fdf847317 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv @@ -0,0 +1 @@ +minus diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv new file mode 100644 index 000000000..7a25db756 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv @@ -0,0 +1 @@ +tausend diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv new file mode 100644 index 000000000..9da371de8 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv @@ -0,0 +1,3 @@ +trilliarde +trilliarden +trilliard diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv new file mode 100644 index 000000000..ed954c28c --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv @@ -0,0 +1,2 @@ +trillion +trillionen diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv new file mode 100644 index 000000000..ee61ac994 --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv @@ -0,0 +1 @@ +und From 27d3bdbc30e90f66a3d3191b41c7e5aa115c170e Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 19:28:54 -0400 Subject: [PATCH 12/14] Load Cardinal magnitude words from one lexicon.tsv Signed-off-by: Adelina Dunina --- .../de/data/cardinal/billiarde.tsv | 3 - .../de/data/cardinal/billion.tsv | 2 - .../de/data/cardinal/hundert.tsv | 2 - .../de/data/cardinal/lexicon.tsv | 20 ++++ .../de/data/cardinal/milliarde.tsv | 3 - .../de/data/cardinal/million.tsv | 2 - .../de/data/cardinal/minus.tsv | 1 - .../de/data/cardinal/tausend.tsv | 1 - .../de/data/cardinal/trilliarde.tsv | 3 - .../de/data/cardinal/trillion.tsv | 2 - .../de/data/cardinal/und.tsv | 1 - .../de/taggers/cardinal.py | 93 +++++++++---------- 12 files changed, 65 insertions(+), 68 deletions(-) delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv create mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/lexicon.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv delete mode 100644 nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv deleted file mode 100644 index fb1a150dc..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billiarde.tsv +++ /dev/null @@ -1,3 +0,0 @@ -billiarde -billiarden -billiard diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv deleted file mode 100644 index 24bd342d0..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/billion.tsv +++ /dev/null @@ -1,2 +0,0 @@ -billion -billionen diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv deleted file mode 100644 index 1b9d5bc34..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/hundert.tsv +++ /dev/null @@ -1,2 +0,0 @@ -hundert -ein hundert diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/lexicon.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/lexicon.tsv new file mode 100644 index 000000000..266e1038b --- /dev/null +++ b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/lexicon.tsv @@ -0,0 +1,20 @@ +und und +minus minus +hundert hundert +ein hundert hundert +tausend tausend +million million +millionen million +milliarde milliarde +milliarden milliarde +milliard milliarde +billion billion +billionen billion +billiarde billiarde +billiarden billiarde +billiard billiarde +trillion trillion +trillionen trillion +trilliarde trilliarde +trilliarden trilliarde +trilliard trilliarde diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv deleted file mode 100644 index b0be36378..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/milliarde.tsv +++ /dev/null @@ -1,3 +0,0 @@ -milliarde -milliarden -milliard diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv deleted file mode 100644 index e3e031f7a..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/million.tsv +++ /dev/null @@ -1,2 +0,0 @@ -million -millionen diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv deleted file mode 100644 index fdf847317..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/minus.tsv +++ /dev/null @@ -1 +0,0 @@ -minus diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv deleted file mode 100644 index 7a25db756..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/tausend.tsv +++ /dev/null @@ -1 +0,0 @@ -tausend diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv deleted file mode 100644 index 9da371de8..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trilliarde.tsv +++ /dev/null @@ -1,3 +0,0 @@ -trilliarde -trilliarden -trilliard diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv deleted file mode 100644 index ed954c28c..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/trillion.tsv +++ /dev/null @@ -1,2 +0,0 @@ -trillion -trillionen diff --git a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv b/nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv deleted file mode 100644 index ee61ac994..000000000 --- a/nemo_text_processing/inverse_text_normalization/de/data/cardinal/und.tsv +++ /dev/null @@ -1 +0,0 @@ -und diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index 38d2491af..fa1b8a419 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -25,6 +25,12 @@ def _digit_tie_flips(): return pynini.string_map([(f"{ones}{tens}", f"{tens}{ones}") for tens in range(2, 10) for ones in range(1, 10)]) + +def _forms(lexicon, lemma): + """Spoken forms for one lemma (e.g. million / millionen).""" + return pynini.project(lexicon @ pynini.accep(lemma), "input").optimize() + + class CardinalFst(GraphFst): """ Finite state transducer for classifying cardinals. Numbers below ten are not converted. @@ -61,8 +67,19 @@ def __init__(self): ties = tens + pynutil.insert("0") # German flips ones and tens in two-digit numbers (ein + zwanzig -> 21). flips = _digit_tie_flips() + lexicon = pynini.string_file(get_abs_path("data/cardinal/lexicon.tsv")) + und = _forms(lexicon, "und") + minus = _forms(lexicon, "minus") + hundert = _forms(lexicon, "hundert") + tausend = _forms(lexicon, "tausend") + million = _forms(lexicon, "million") + milliarde = _forms(lexicon, "milliarde") + billion_de = _forms(lexicon, "billion") + billiarde = _forms(lexicon, "billiarde") + trillion_de = _forms(lexicon, "trillion") + trilliarde = _forms(lexicon, "trilliarde") delete_space = pynutil.delete(NEMO_SPACE) - delete_und = pynutil.delete("und") + delete_und = pynutil.delete(und) # Accepts normalized digits+ties (ein+und+zwanzig) digit_ties = digits + delete_space.ques + delete_und + delete_space.ques + tens @@ -76,12 +93,11 @@ def __init__(self): graph_single_and_double_digits = digits | graph_10_99 self.graph_single_and_double_digits = graph_single_and_double_digits.optimize() - hundert = pynini.accep("hundert") | pynini.accep("ein hundert") hundreds = (pynini.cross(hundert, "100")) | ( ( (digits | pynutil.insert("1")) + delete_space.ques - + pynutil.delete("hundert") + + pynutil.delete(hundert) + delete_space.ques + delete_und.ques + delete_space.ques @@ -90,13 +106,13 @@ def __init__(self): | ( (digits | pynutil.insert("1")) + delete_space.ques - + pynini.cross("hundert", "0") + + pynini.cross(hundert, "0") + delete_space.ques + delete_und.ques + delete_space.ques + digits ) - | ((digits | pynutil.insert("1")) + delete_space.ques + pynini.cross("hundert", "00")) + | ((digits | pynutil.insert("1")) + delete_space.ques + pynini.cross(hundert, "00")) ) # Digits are grouped in clusters of three: {hundreds}{tens}{ones}. @@ -112,20 +128,20 @@ def __init__(self): non_zero_digit_cluster = (hundreds) | (pynutil.insert("0") + graph_10_99) | (pynutil.insert("00") + digits) # WFST grammar for thousands - thousands = (pynini.cross("tausend", "1.000")) | ( + thousands = (pynini.cross(tausend, "1.000")) | ( ( - (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) - | (digit_cluster + delete_space.ques + pynini.cross("tausend", ".") + delete_und.ques) + (pynini.cross(tausend, "1.") + delete_space.ques + delete_und.ques) + | (digit_cluster + delete_space.ques + pynini.cross(tausend, ".") + delete_und.ques) | pynutil.insert("000.") ) + delete_space.ques + digit_cluster ) - non_zero_thousands = (pynini.cross("tausend", "1.000")) | ( + non_zero_thousands = (pynini.cross(tausend, "1.000")) | ( ( - (pynini.cross("tausend", "1.") + delete_space.ques + delete_und.ques) - | (non_zero_digit_cluster + delete_space.ques + pynini.cross("tausend", ".") + delete_und.ques) + (pynini.cross(tausend, "1.") + delete_space.ques + delete_und.ques) + | (non_zero_digit_cluster + delete_space.ques + pynini.cross(tausend, ".") + delete_und.ques) # | pynutil.insert("000.") ) + delete_space.ques @@ -133,10 +149,9 @@ def __init__(self): ) # WFST grammar for millions - million = pynini.accep("million") | pynini.accep("millionen") - millions = (pynini.cross("million", "1.000.000")) | ( + millions = (pynini.cross(million, "1.000.000")) | ( ( - (pynini.cross("million", "1.") + delete_space.ques + delete_und.ques) + (pynini.cross(million, "1.") + delete_space.ques + delete_und.ques) | (digit_cluster + delete_space.ques + pynini.cross(million, ".") + delete_und.ques) | pynutil.insert("000.") ) @@ -145,17 +160,11 @@ def __init__(self): ) # WFST grammar for billions - billion = ( - pynini.accep("milliarde") - | pynini.accep("milliarden") - # include the consonant-final stem for ordinal declensions e.g "milliardste" - # "e" -> "" / _[ordinal morpheme] - | pynini.accep("milliard") - ) - billions = (pynini.cross("milliarde", "1.000.000.000")) | ( + billion = milliarde + billions = (pynini.cross(milliarde, "1.000.000.000")) | ( ( ( - pynini.cross((pynini.accep("milliarde") | pynini.accep("milliard")), "1.") + pynini.cross(milliarde, "1.") + delete_space.ques + delete_und.ques ) @@ -167,10 +176,10 @@ def __init__(self): ) # WFST grammar for trillions - trillion = pynini.accep("billion") | pynini.accep("billionen") - trillions = (pynini.cross("billion", "1.000.000.000.000")) | ( + trillion = billion_de + trillions = (pynini.cross(billion_de, "1.000.000.000.000")) | ( ( - (pynini.cross("billion", "1.") + delete_space.ques + delete_und.ques) + (pynini.cross(billion_de, "1.") + delete_space.ques + delete_und.ques) | (digit_cluster + delete_space.ques + pynini.cross(trillion, ".") + delete_und.ques) | pynutil.insert("000.") ) @@ -179,14 +188,8 @@ def __init__(self): ) # WFST grammar for quadrillions - quadrillion = ( - pynini.accep("billiarde") - | pynini.accep("billiarden") - # include the consonant-final stem for ordinal declensions e.g "billiardste" - # "e" -> "" / _[ordinal morpheme] - | pynini.accep("billiard") - ) - quadrillions = (pynini.cross("billiarde", "1.000.000.000.000.000")) | ( + quadrillion = billiarde + quadrillions = (pynini.cross(billiarde, "1.000.000.000.000.000")) | ( ( (pynini.cross(quadrillion, "1.") + delete_space.ques + delete_und.ques) | (digit_cluster + delete_space.ques + pynini.cross(quadrillion, ".") + delete_und.ques) @@ -197,10 +200,10 @@ def __init__(self): ) # WFST grammar for quintillions - quintillion = pynini.accep("trillion") | pynini.accep("trillionen") - quintillions = (pynini.cross("trillion", "1.000.000.000.000.000.000")) | ( + quintillion = trillion_de + quintillions = (pynini.cross(trillion_de, "1.000.000.000.000.000.000")) | ( ( - (pynini.cross("trillion", "1.") + delete_space.ques + delete_und.ques) + (pynini.cross(trillion_de, "1.") + delete_space.ques + delete_und.ques) | (digit_cluster + delete_space.ques + pynini.cross(quintillion, ".") + delete_und.ques) | pynutil.insert("000.") ) @@ -209,14 +212,8 @@ def __init__(self): ) # WFST grammar for sextillions - sextillion = ( - pynini.accep("trilliarde") - | pynini.accep("trilliarden") - # include the consonant-final stem for ordinal declensions e.g "trilliardste" - # "e" -> "" / _[ordinal morpheme] - | pynini.accep("trilliard") - ) - sextillions = (pynini.cross("billiarde", "1.000.000.000.000.000.000.000")) | ( + sextillion = trilliarde + sextillions = (pynini.cross(trilliarde, "1.000.000.000.000.000.000.000")) | ( ( (pynini.cross(sextillion, "1.") + delete_space.ques + delete_und.ques) | (digit_cluster + delete_space.ques + pynini.cross(sextillion, ".") + delete_und.ques) @@ -261,14 +258,14 @@ def __init__(self): # The graph below covers exceptions # e.g. years 1100 - 1999 # and all colloquial expresions (e.g. zwanzigvierundzwanzig -> 2024) - ten = "zehn" + ten = pynini.project(irregular_teens @ pynini.accep("10"), "input") remove_ten = pynini.project(graph_10_99, "input") - ten graph_11_99 = remove_ten @ graph_10_99 years_exceptions = ( graph_11_99 + pynutil.delete(NEMO_SPACE).ques - + pynutil.delete("hundert").ques + + pynutil.delete(hundert).ques + pynutil.delete(NEMO_SPACE).ques + (graph_10_99 | pynutil.insert("00")) ) @@ -292,7 +289,7 @@ def __init__(self): self.graph = graph.optimize() self.optional_negative = pynini.closure( - pynutil.insert("negative: ") + pynini.cross("minus ", '"-"') + pynutil.insert(" "), + pynutil.insert("negative: ") + pynini.cross(minus + pynini.accep(" "), '"-"') + pynutil.insert(" "), 0, 1, ) From 89063b261c56dd61f9ef2ccc51ea2243cdc13ddf Mon Sep 17 00:00:00 2001 From: Adelina Dunina Date: Fri, 4 Sep 2026 19:32:28 -0400 Subject: [PATCH 13/14] Restore 2021 copyright on existing de Cardinal files Signed-off-by: Adelina Dunina --- .../inverse_text_normalization/de/taggers/cardinal.py | 2 +- .../inverse_text_normalization/de/verbalizers/cardinal.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index fa1b8a419..a4a61f6d5 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py index 27dcfe27e..ba8cfc8b9 100644 --- a/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/verbalizers/cardinal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 06b449819964ea4121be4aedc03f5f6462aa59bd Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:37:53 +0000 Subject: [PATCH 14/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../inverse_text_normalization/de/taggers/cardinal.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py index a4a61f6d5..594ad1089 100644 --- a/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py +++ b/nemo_text_processing/inverse_text_normalization/de/taggers/cardinal.py @@ -25,7 +25,6 @@ def _digit_tie_flips(): return pynini.string_map([(f"{ones}{tens}", f"{tens}{ones}") for tens in range(2, 10) for ones in range(1, 10)]) - def _forms(lexicon, lemma): """Spoken forms for one lemma (e.g. million / millionen).""" return pynini.project(lexicon @ pynini.accep(lemma), "input").optimize() @@ -163,11 +162,7 @@ def __init__(self): billion = milliarde billions = (pynini.cross(milliarde, "1.000.000.000")) | ( ( - ( - pynini.cross(milliarde, "1.") - + delete_space.ques - + delete_und.ques - ) + (pynini.cross(milliarde, "1.") + delete_space.ques + delete_und.ques) | (digit_cluster + delete_space.ques + pynini.cross(billion, ".") + delete_und.ques) | pynutil.insert("000.") )