From d82f25c859c5606874f23191624f95a2e5665cb6 Mon Sep 17 00:00:00 2001 From: daniazie Date: Thu, 14 May 2026 13:44:29 +0900 Subject: [PATCH 01/13] update for transformers v5 --- comet/encoders/base.py | 12 ++++++++++-- comet/encoders/bert.py | 7 +++++++ comet/encoders/minilm.py | 8 ++++++++ comet/encoders/rembert.py | 7 +++++++ comet/encoders/xlmr.py | 11 ++++++++++- comet/encoders/xlmr_xl.py | 8 ++++++++ pyproject.toml | 26 +++++++++++++------------- 7 files changed, 63 insertions(+), 16 deletions(-) diff --git a/comet/encoders/base.py b/comet/encoders/base.py index 90ed5649..f4d31ab6 100644 --- a/comet/encoders/base.py +++ b/comet/encoders/base.py @@ -277,6 +277,7 @@ def concat_sequences( concatenated into a single input. """ concat_input_ids = [] + # Remove padding before concatenation for encoder_input in inputs: input_ids = encoder_input["input_ids"] @@ -295,7 +296,7 @@ def concat_sequences( # because that method adds them again lengths = tuple(len(x[i][1:-1]) for x in concat_input_ids) - # self.max_positions = 512 but we need to remove 4 aditional tokens + # self.max_positions = 512 but we need to remove 4 additional tokens # [CLS]...[SEP]...[SEP]...[SEP] special_tokens = 1 + len(inputs) * self.size_separator new_sequence = concat_input_ids[0][i] @@ -304,7 +305,7 @@ def concat_sequences( torch.zeros(len(new_sequence[1:-1]) + 2, dtype=torch.int) ) for j in range(1, len(inputs)): - new_sequence = self.tokenizer.build_inputs_with_special_tokens( + new_sequence = self.build_inputs_with_special_tokens( new_sequence[1:-1], concat_input_ids[j][i][1:-1] ) if sum(lengths) > self.max_positions - special_tokens: @@ -341,3 +342,10 @@ def concat_sequences( encoder_input["token_type_ids"] = token_type_ids return encoder_input, lengths, max_len + + def build_inputs_with_special_tokens( + self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None + ) -> list[int]: + if token_ids_1 is None: + return token_ids_0 + return token_ids_0 + token_ids_1 \ No newline at end of file diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index 753dc70d..b4a20da5 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -181,3 +181,10 @@ def forward( "all_layers": all_layers, "attention_mask": attention_mask, } + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep \ No newline at end of file diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index 35a460d9..252fc0c2 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -75,3 +75,11 @@ def from_pretrained( return MiniLMEncoder( pretrained_model, load_pretrained_weights, local_files_only ) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + sep + token_ids_1 + sep \ No newline at end of file diff --git a/comet/encoders/rembert.py b/comet/encoders/rembert.py index 003eb857..de76bcd6 100644 --- a/comet/encoders/rembert.py +++ b/comet/encoders/rembert.py @@ -84,3 +84,10 @@ def from_pretrained( return RemBERTEncoder( pretrained_model, load_pretrained_weights, local_files_only ) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep \ No newline at end of file diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index 13cc15af..a4b7f3ea 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -92,15 +92,24 @@ def from_pretrained( def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs ) -> Dict[str, torch.Tensor]: - last_hidden_states, _, all_layers = self.model( + last_hidden_states, all_layers = self.model( input_ids=input_ids, attention_mask=attention_mask, output_hidden_states=True, return_dict=False, ) + return { "sentemb": last_hidden_states[:, 0, :], "wordemb": last_hidden_states, "all_layers": all_layers, "attention_mask": attention_mask, } + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + sep + token_ids_1 + sep \ No newline at end of file diff --git a/comet/encoders/xlmr_xl.py b/comet/encoders/xlmr_xl.py index edb64ac4..f0f389b5 100644 --- a/comet/encoders/xlmr_xl.py +++ b/comet/encoders/xlmr_xl.py @@ -77,3 +77,11 @@ def from_pretrained( return XLMRXLEncoder( pretrained_model, load_pretrained_weights, local_files_only ) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/pyproject.toml b/pyproject.toml index 214a48c6..479ff363 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,24 +37,24 @@ comet-mbr = 'comet.cli.mbr:mbr_command' [tool.poetry.dependencies] python = "^3.8.0" -sentencepiece = "^0.2.0" +sentencepiece = ">=0.2.0" pandas = ">=1.4.1" -transformers = "^4.17" -pytorch-lightning = "^2.0.0" +transformers = ">=4.51.1" +pytorch-lightning = ">=2.0.0" jsonargparse = "3.13.1" torch = ">=1.6.0" -numpy = "^1.20.0" -torchmetrics = "^0.10.2" -sacrebleu = "^2.0.0" -scipy = "^1.5.4" -entmax = "^1.1" -huggingface-hub = ">=0.19.3,<1.0" -protobuf = "^4.24.4" +numpy = ">=1.20.0" +torchmetrics = ">=0.10.2" +sacrebleu = ">=2.0.0" +scipy = ">=1.5.4" +entmax = ">=1.1" +huggingface-hub = ">=0.30.0" +protobuf = ">=4.24.4" [tool.poetry.dev-dependencies] -sphinx-markdown-tables = "0.0.15" -coverage = "^5.5" -scikit-learn = "^1.0" +sphinx-markdown-tables = ">=0.0.15" +coverage = ">=5.5" +scikit-learn = ">=1.0" [build-system] requires = ["poetry-core>=1.0.0"] From 152cc72494421390a3ae2fa271a64ebd37d0ea8d Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Thu, 14 May 2026 14:34:41 +0900 Subject: [PATCH 02/13] Removed pkg_resources --- README.md | 5 + comet/__init__.py | 2 +- comet/cli/compare.py | 4 +- comet/cli/mbr.py | 30 +- comet/cli/score.py | 21 +- comet/cli/train.py | 2 +- comet/encoders/__init__.py | 2 +- comet/encoders/base.py | 4 +- comet/encoders/bert.py | 4 +- comet/encoders/minilm.py | 6 +- comet/encoders/rembert.py | 6 +- comet/encoders/xlmr.py | 7 +- comet/encoders/xlmr_xl.py | 7 +- comet/models/__init__.py | 20 +- comet/models/base.py | 24 +- comet/models/download_utils.py | 2 - comet/models/lru_cache.py | 4 +- comet/models/multitask/unified_metric.py | 10 +- comet/models/pooling_utils.py | 43 +-- comet/models/ranking/ranking_metric.py | 3 +- comet/models/regression/referenceless.py | 4 +- comet/models/regression/regression_metric.py | 3 +- comet/models/utils.py | 34 ++- comet/modules/feedforward.py | 4 +- comet/modules/layerwise_attention.py | 1 + .../integration/models/test_ranking_metric.py | 5 +- .../models/test_referenceless_regression.py | 5 +- .../models/test_regression_metric.py | 5 +- .../integration/models/test_unified_metric.py | 1 - tests/integration/modules/test_feedforward.py | 2 +- tests/unit/test_download_load.py | 5 +- tests/unit/test_models_predict.py | 284 +++++++++++++++--- 32 files changed, 404 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index b90bb73a..81b98876 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,11 @@ Please check all available models [here](https://github.com/Unbabel/COMET/blob/m # Quick Installation +> For this particular repo: +```bash + pip install https://github.com/daniazie/COMET.git +``` + COMET requires python 3.8 or above. Simple installation from PyPI ```bash diff --git a/comet/__init__.py b/comet/__init__.py index 5f8b64b6..e5dfe971 100644 --- a/comet/__init__.py +++ b/comet/__init__.py @@ -16,7 +16,7 @@ import logging -from .models import load_from_checkpoint, download_model +from .models import download_model, load_from_checkpoint logging.basicConfig(level=logging.INFO, format="%(message)s") logger = logging.getLogger(__name__) diff --git a/comet/cli/compare.py b/comet/cli/compare.py index d154dd8f..fd59dcd0 100644 --- a/comet/cli/compare.py +++ b/comet/cli/compare.py @@ -40,7 +40,7 @@ default: 0.4) --t_test_alternative T_TEST_ALTERNATIVE Alternative hypothesis from scipy.stats.ttest_rel. The - following options are available: 'two-sided', 'less', + following options are available: 'two-sided', 'less', 'greater'. Defaults to 'less' (type: str, default: two-sided) --to_json TO_JSON Exports results to a json file. (type: str, default: "") --model MODEL COMET model to be used. (type: str, default: wmt20-comet-da) @@ -73,7 +73,7 @@ from comet import download_model, load_from_checkpoint -torch.set_float32_matmul_precision('high') +torch.set_float32_matmul_precision("high") Statistical_test_info = Dict[str, Union[Path_fr, Dict[str, float]]] diff --git a/comet/cli/mbr.py b/comet/cli/mbr.py index f7b61eb0..6d018527 100644 --- a/comet/cli/mbr.py +++ b/comet/cli/mbr.py @@ -36,7 +36,7 @@ applying MBR. Disabled by default. (type: int, default: 0) --qe_model QE_MODEL Reference Free model used for reranking before MBR. (type: str, default: Unbabel/wmt23-cometkiwi-da-xl) - --model MODEL COMET model to be used. + --model MODEL COMET model to be used. (type: str, default: Unbabel/wmt23-comet-da-xl) --model_storage_path MODEL_STORAGE_PATH Path to the directory where models will be stored. By default @@ -55,7 +55,8 @@ from comet.models import RegressionMetric, download_model, load_from_checkpoint -torch.set_float32_matmul_precision('high') +torch.set_float32_matmul_precision("high") + def build_embeddings( sources: List[str], @@ -141,6 +142,7 @@ def mbr_decoding( return mbr_matrix + def rerank_top_k( sources: List[str], translations: List[str], @@ -178,10 +180,13 @@ def rerank_top_k( topk_indices = np.argsort(seg_scores, axis=1) topk_translations = [] for i in range(len(sources)): - topk_translations += [translations[i][idx] for idx in topk_indices[i][::-1][:topk]] + topk_translations += [ + translations[i][idx] for idx in topk_indices[i][::-1][:topk] + ] return topk_translations + def mbr_command() -> None: parser = ArgumentParser(description="Command for Minimum Bayes Risk Decoding.") parser.add_argument("-s", "--sources", type=Path_fr, required=True) @@ -238,10 +243,7 @@ def mbr_command() -> None: num_samples = cfg.num_samples # Running QE reranking before MBR! if cfg.rerank_top_k > 0: - if ( - cfg.qe_model.endswith(".ckpt") - and os.path.exists(cfg.qe_model) - ): + if cfg.qe_model.endswith(".ckpt") and os.path.exists(cfg.qe_model): qe_model_path = cfg.qe_model else: qe_model_path = download_model( @@ -256,7 +258,13 @@ def mbr_command() -> None: ), "--qe_model expects a Reference Free model!" translations = rerank_top_k( - sources, translations, model, cfg.batch_size, cfg.gpus, cfg.num_samples, cfg.rerank_top_k + sources, + translations, + model, + cfg.batch_size, + cfg.gpus, + cfg.num_samples, + cfg.rerank_top_k, ) num_samples = cfg.rerank_top_k @@ -264,19 +272,19 @@ def mbr_command() -> None: model_path = cfg.model else: model_path = download_model(cfg.model, saving_directory=cfg.model_storage_path) - + model = load_from_checkpoint(model_path) model.eval() model.cuda() model.half() - + if not isinstance(model, RegressionMetric): raise Exception( "Invalid model ({}). MBR command only works with Reference-based Regression models!".format( model.__class__.__name__ ) ) - + src_embeddings, mt_embeddings = build_embeddings( sources, translations, model, cfg.batch_size ) diff --git a/comet/cli/score.py b/comet/cli/score.py index 64323674..31ce9efa 100644 --- a/comet/cli/score.py +++ b/comet/cli/score.py @@ -39,12 +39,12 @@ Path to the directory where models will be stored. By default its saved in ~/.cache/torch/unbabel_comet/ (default: null) --num_workers NUM_WORKERS - Number of workers to use when loading data. (type: int, + Number of workers to use when loading data. (type: int, default: null) --disable_cache Disables sentence embeddings caching. This makes inference slower but saves memory. (default: False) --disable_length_batching - Disables length batching. This makes inference slower. + Disables length batching. This makes inference slower. (default: False) --print_cache_info Print information about COMET cache. (default: False) """ @@ -63,7 +63,8 @@ from comet import download_model, load_from_checkpoint from comet.models.utils import split_sequence_into_sublists -torch.set_float32_matmul_precision('high') +torch.set_float32_matmul_precision("high") + def score_command() -> None: parser = ArgumentParser(description="Command for scoring MT systems.") @@ -77,7 +78,9 @@ def score_command() -> None: "--quiet", action="store_true", help="Sets all loggers to ERROR level." ) parser.add_argument( - "--enable-context", action="store_true", help="Enables contextual extension of COMET on inputs preprocessed with context information." + "--enable-context", + action="store_true", + help="Enables contextual extension of COMET on inputs preprocessed with context information.", ) parser.add_argument( "--only_system", action="store_true", help="Prints only the final system score." @@ -163,7 +166,7 @@ def score_command() -> None: model = load_from_checkpoint(model_path) model.eval() model.half() - + if cfg.enable_context: model.enable_context() @@ -214,7 +217,7 @@ def score_command() -> None: errors = outputs.metadata.error_spans else: errors = [] - + if len(cfg.translations) > 1: seg_scores = np.array_split(seg_scores, len(cfg.translations)) sys_scores = [sum(split) / len(split) for split in seg_scores] @@ -227,7 +230,9 @@ def score_command() -> None: seg_scores = [ seg_scores, ] - errors = [errors, ] + errors = [ + errors, + ] data = [ np.array(data), ] @@ -262,7 +267,7 @@ def score_command() -> None: data[files[j]][i]["COMET"] = seg_scores[j][i] if errors and errors[j] and errors[j][i]: data[files[j]][i]["errors"] = errors[j][i] - + if not cfg.only_system: print( "{}\tSegment {}\tscore: {:.4f}".format( diff --git a/comet/cli/train.py b/comet/cli/train.py index 6ccc5193..893a4be4 100644 --- a/comet/cli/train.py +++ b/comet/cli/train.py @@ -42,7 +42,7 @@ from comet.models import (RankingMetric, ReferencelessRegression, RegressionMetric, UnifiedMetric) -torch.set_float32_matmul_precision('high') +torch.set_float32_matmul_precision("high") logger = logging.getLogger(__name__) diff --git a/comet/encoders/__init__.py b/comet/encoders/__init__.py index 5e41e879..6de0aecf 100644 --- a/comet/encoders/__init__.py +++ b/comet/encoders/__init__.py @@ -13,8 +13,8 @@ # limitations under the License. from .bert import BERTEncoder from .minilm import MiniLMEncoder -from .xlmr import XLMREncoder from .rembert import RemBERTEncoder +from .xlmr import XLMREncoder from .xlmr_xl import XLMRXLEncoder str2encoder = { diff --git a/comet/encoders/base.py b/comet/encoders/base.py index f4d31ab6..1de65536 100644 --- a/comet/encoders/base.py +++ b/comet/encoders/base.py @@ -277,7 +277,7 @@ def concat_sequences( concatenated into a single input. """ concat_input_ids = [] - + # Remove padding before concatenation for encoder_input in inputs: input_ids = encoder_input["input_ids"] @@ -348,4 +348,4 @@ def build_inputs_with_special_tokens( ) -> list[int]: if token_ids_1 is None: return token_ids_0 - return token_ids_0 + token_ids_1 \ No newline at end of file + return token_ids_0 + token_ids_1 diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index b4a20da5..c6e96796 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -182,9 +182,9 @@ def forward( "attention_mask": attention_mask, } - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] if token_ids_1 is None: return cls + token_ids_0 + sep - return cls + token_ids_0 + sep + token_ids_1 + sep \ No newline at end of file + return cls + token_ids_0 + sep + token_ids_1 + sep diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index 252fc0c2..13cc63d4 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -15,7 +15,7 @@ r""" MiniLM Encoder ============== - Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT + Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT architecture with an XLMR tokenizer. """ from transformers import BertConfig, BertModel, XLMRobertaTokenizerFast @@ -76,10 +76,10 @@ def from_pretrained( pretrained_model, load_pretrained_weights, local_files_only ) - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] if token_ids_1 is None: return cls + token_ids_0 + sep - return cls + token_ids_0 + sep + sep + token_ids_1 + sep \ No newline at end of file + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/comet/encoders/rembert.py b/comet/encoders/rembert.py index de76bcd6..448e685f 100644 --- a/comet/encoders/rembert.py +++ b/comet/encoders/rembert.py @@ -15,7 +15,7 @@ r""" RemBERT Encoder =============== - Pretrained RemBERT encoder from Google. This encoder is similar to BERT but uses + Pretrained RemBERT encoder from Google. This encoder is similar to BERT but uses sentencepiece like XLMR. """ from transformers import RemBertConfig, RemBertModel, RemBertTokenizerFast @@ -85,9 +85,9 @@ def from_pretrained( pretrained_model, load_pretrained_weights, local_files_only ) - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] if token_ids_1 is None: return cls + token_ids_0 + sep - return cls + token_ids_0 + sep + token_ids_1 + sep \ No newline at end of file + return cls + token_ids_0 + sep + token_ids_1 + sep diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index a4b7f3ea..136e1678 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -20,7 +20,8 @@ from typing import Dict import torch -from transformers import XLMRobertaConfig, XLMRobertaModel, XLMRobertaTokenizerFast +from transformers import (XLMRobertaConfig, XLMRobertaModel, + XLMRobertaTokenizerFast) from comet.encoders.base import Encoder from comet.encoders.bert import BERTEncoder @@ -106,10 +107,10 @@ def forward( "attention_mask": attention_mask, } - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] if token_ids_1 is None: return cls + token_ids_0 + sep - return cls + token_ids_0 + sep + sep + token_ids_1 + sep \ No newline at end of file + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/comet/encoders/xlmr_xl.py b/comet/encoders/xlmr_xl.py index f0f389b5..b3a1625f 100644 --- a/comet/encoders/xlmr_xl.py +++ b/comet/encoders/xlmr_xl.py @@ -17,7 +17,8 @@ ============== Pretrained XLM-RoBERTa-XL encoder from Hugging Face. """ -from transformers import XLMRobertaTokenizerFast, XLMRobertaXLConfig, XLMRobertaXLModel +from transformers import (XLMRobertaTokenizerFast, XLMRobertaXLConfig, + XLMRobertaXLModel) from comet.encoders.base import Encoder from comet.encoders.xlmr import XLMREncoder @@ -77,8 +78,8 @@ def from_pretrained( return XLMRXLEncoder( pretrained_model, load_pretrained_weights, local_files_only ) - - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1 = None): + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] diff --git a/comet/models/__init__.py b/comet/models/__init__.py index 8c1835b0..5211ff5f 100644 --- a/comet/models/__init__.py +++ b/comet/models/__init__.py @@ -22,13 +22,12 @@ from huggingface_hub import snapshot_download from .base import CometModel +from .download_utils import download_model_legacy from .multitask.unified_metric import UnifiedMetric from .multitask.xcomet_metric import XCOMETMetric from .ranking.ranking_metric import RankingMetric from .regression.referenceless import ReferencelessRegression from .regression.regression_metric import RegressionMetric -from .download_utils import download_model_legacy - str2model = { "referenceless_regression_metric": ReferencelessRegression, @@ -93,15 +92,20 @@ def load_from_checkpoint( hparams = yaml.load(yaml_file.read(), Loader=yaml.FullLoader) model_class = str2model[hparams["class_identifier"]] - + # Check comet version and hparams for layer_transformation # This is a workaround for the bug reported in version 2.2.4 # issue number #244 try: - import pkg_resources - comet_version = pkg_resources.get_distribution("unbabel-comet").version - use_softmax = (pkg_resources.parse_version(comet_version) >= pkg_resources.parse_version("2.2.4") and - hparams.get("layer_transformation") == "sparsemax_patch") + from importlib import metadata + + from packaging.version import Version as parse_version + + comet_version = metadata.distribution("unbabel-comet").version + use_softmax = ( + parse_version(comet_version) >= parse_version("2.2.4") + and hparams.get("layer_transformation") == "sparsemax_patch" + ) except: use_softmax = False @@ -116,7 +120,7 @@ def load_from_checkpoint( } if use_softmax: kwargs["layer_transformation"] = "softmax" - + model = model_class.load_from_checkpoint(**kwargs) return model else: diff --git a/comet/models/base.py b/comet/models/base.py index 9872d2b7..655fcb53 100644 --- a/comet/models/base.py +++ b/comet/models/base.py @@ -28,7 +28,8 @@ import numpy as np import pytorch_lightning as ptl import torch -from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Subset +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + Subset) from comet.encoders import str2encoder from comet.modules import LayerwiseAttention @@ -37,13 +38,8 @@ from .pooling_utils import average_pooling, max_pooling from .predict_pbar import PredictProgressBar from .predict_writer import CustomWriter -from .utils import ( - OrderedSampler, - Prediction, - Target, - flatten_metadata, - restore_list_order, -) +from .utils import (OrderedSampler, Prediction, Target, flatten_metadata, + restore_list_order) if "COMET_EMBEDDINGS_CACHE" in os.environ: CACHE_SIZE = int(os.environ["COMET_EMBEDDINGS_CACHE"]) @@ -162,7 +158,9 @@ def set_mc_dropout(self, value: int): def enable_context(self): """Function that extends COMET to use preceding context as described in https://statmt.org/wmt22/pdf/2022.wmt-1.6.pdf.""" - logger.warning("Context should only be enabled for RegressionMetric with Average Pooling.") + logger.warning( + "Context should only be enabled for RegressionMetric with Average Pooling." + ) @abc.abstractmethod def read_training_data(self) -> List[dict]: @@ -352,7 +350,7 @@ def compute_sentence_embedding( embeddings, attention_mask, self.encoder.tokenizer.pad_token_id, - self.encoder.tokenizer.sep_token_id, + self.encoder.tokenizer.sep_token_id, self.use_context, ) @@ -593,7 +591,7 @@ def predict( ) elif gpus > 0: devices = gpus - else: # gpu = 0 + else: # gpu = 0 devices = "auto" sampler = SequentialSampler(samples) @@ -622,7 +620,9 @@ def predict( sampler=sampler, collate_fn=self.prepare_for_inference, num_workers=num_workers, - multiprocessing_context="fork" if torch.backends.mps.is_available() else None, + multiprocessing_context=( + "fork" if torch.backends.mps.is_available() else None + ), ) if gpus > 1: pred_writer = CustomWriter() diff --git a/comet/models/download_utils.py b/comet/models/download_utils.py index afd5d84d..d0ebad23 100644 --- a/comet/models/download_utils.py +++ b/comet/models/download_utils.py @@ -33,7 +33,6 @@ "wmt20-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-da.tar.gz", "wmt20-comet-qe-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da.tar.gz", "wmt20-comet-qe-da-v2": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da-v2.tar.gz", - # WMT21 Models "wmt21-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-da.tar.gz", "wmt21-comet-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-mqm.tar.gz", @@ -41,7 +40,6 @@ "wmt21-cometinho-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-da.tar.gz", "wmt21-comet-qe-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-mqm.tar.gz", "wmt21-comet-qe-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-da.tar.gz", - # EAMT22 Models "eamt22-cometinho-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-cometinho-da.tar.gz", "eamt22-prune-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-prune-comet-da.tar.gz", diff --git a/comet/models/lru_cache.py b/comet/models/lru_cache.py index 5a200fe4..6b876c08 100644 --- a/comet/models/lru_cache.py +++ b/comet/models/lru_cache.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -LRU Cache +LRU Cache ========== LRU Cache function decorator modified to work with tensor arguments. @@ -53,7 +53,7 @@ def _make_key( if torch.is_tensor(x): if len(x.size()) == 0: raise Exception("Tensor needs to be at least 1-Dimensional.") - + if len(x.size()) == 1: new_args.append("\n".join([repr(x), repr(x.shape)])) else: diff --git a/comet/models/multitask/unified_metric.py b/comet/models/multitask/unified_metric.py index e0a76548..911e66eb 100644 --- a/comet/models/multitask/unified_metric.py +++ b/comet/models/multitask/unified_metric.py @@ -16,10 +16,10 @@ r""" Unified Metric ============== - Unified Metric is a multitask metric that performs word-level and segment-level - evaluation in a multitask manner. It can also be used with and without reference + Unified Metric is a multitask metric that performs word-level and segment-level + evaluation in a multitask manner. It can also be used with and without reference translations. - + Inspired on [UniTE](https://arxiv.org/pdf/2204.13346.pdf) """ from collections import OrderedDict @@ -475,8 +475,8 @@ def forward( raise Exception( "Invalid model sent layer {}.".format(self.hparams.word_layer) ) - sentemb = embeddings[:, 0, :] # We take the CLS token as sentence-embedding - + sentemb = embeddings[:, 0, :] # We take the CLS token as sentence-embedding + if self.word_level: sentence_output = self.estimator(sentemb) word_output = self.hidden2tag(wordemb) diff --git a/comet/models/pooling_utils.py b/comet/models/pooling_utils.py index 7c3d8edf..98fc5328 100644 --- a/comet/models/pooling_utils.py +++ b/comet/models/pooling_utils.py @@ -12,16 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import torch from typing import List, Union +import torch + + # From https://github.com/amazon-science/doc-mt-metrics/blob/5385cc28930aae9924edcb3201645dd3810b12c0/COMET/comet/models/pooling_utils.py#L18 def find_start_inds_and_mask_tokens( - mask: torch.Tensor, - tokens: torch.Tensor, - separator_index: int, + mask: torch.Tensor, + tokens: torch.Tensor, + separator_index: int, ) -> Union[List[int], torch.Tensor]: - """Finds the starting indices of each sentence for multi-sentence sequences and + """Finds the starting indices of each sentence for multi-sentence sequences and creates a new mask to omit all context sentences from the pooling function. Args: @@ -38,18 +40,19 @@ def find_start_inds_and_mask_tokens( # if there are more than one find where the last sentence starts ind = separators[-2].cpu().numpy().item() start_inds.append(ind) - ctx_mask[i, 1:ind+1] = 0 + ctx_mask[i, 1 : ind + 1] = 0 else: start_inds.append(0) return start_inds, ctx_mask + def average_pooling( tokens: torch.Tensor, embeddings: torch.Tensor, mask: torch.Tensor, padding_index: int, separator_index: int, - enable_context: bool = False + enable_context: bool = False, ) -> torch.Tensor: """Average pooling method. @@ -64,11 +67,13 @@ def average_pooling( torch.Tensor: Sentence embedding """ if enable_context: - start_inds, ctx_mask = find_start_inds_and_mask_tokens(mask, tokens, separator_index) + start_inds, ctx_mask = find_start_inds_and_mask_tokens( + mask, tokens, separator_index + ) wordemb = mask_fill_index(0.0, tokens, embeddings, start_inds, padding_index) sentemb = torch.sum(wordemb, 1) sum_mask = ctx_mask.unsqueeze(-1).expand(embeddings.size()).float().sum(1) - else: + else: wordemb = mask_fill(0.0, tokens, embeddings, padding_index) sentemb = torch.sum(wordemb, 1) sum_mask = mask.unsqueeze(-1).expand(embeddings.size()).float().sum(1) @@ -91,13 +96,14 @@ def max_pooling( """ return mask_fill(float("-inf"), tokens, embeddings, padding_index).max(dim=1)[0] + # From https://github.com/amazon-science/doc-mt-metrics/blob/5385cc28930aae9924edcb3201645dd3810b12c0/COMET/comet/models/pooling_utils.py#L18 def mask_fill_index( - fill_value: float, - tokens: torch.Tensor, - embeddings: torch.Tensor, - start_inds: list, - padding_index: int, + fill_value: float, + tokens: torch.Tensor, + embeddings: torch.Tensor, + start_inds: list, + padding_index: int, ) -> torch.Tensor: """ Masks embeddings representing padded elements and context sentences for multi-sentence sequences. @@ -108,17 +114,20 @@ def mask_fill_index( embeddings: word embeddings [bsz x seq_len x hiddens]. start_inds: Start of sentence indices. padding_index: Index of the padding token. - + Return: torch.Tensor: Sentence embedding """ padding_mask = tokens.eq(padding_index).unsqueeze(-1) - padding_maks2 = torch.zeros(tokens.shape, dtype=torch.bool, device=padding_mask.device) + padding_maks2 = torch.zeros( + tokens.shape, dtype=torch.bool, device=padding_mask.device + ) for i, start in enumerate(start_inds): - padding_maks2[i, 1: start+1] = True + padding_maks2[i, 1 : start + 1] = True padding_mask = torch.logical_or(padding_mask, padding_maks2.unsqueeze(-1)) return embeddings.float().masked_fill_(padding_mask, fill_value).type_as(embeddings) + def mask_fill( fill_value: float, tokens: torch.Tensor, diff --git a/comet/models/ranking/ranking_metric.py b/comet/models/ranking/ranking_metric.py index 111e85f1..f5d9825d 100644 --- a/comet/models/ranking/ranking_metric.py +++ b/comet/models/ranking/ranking_metric.py @@ -28,7 +28,8 @@ import torch import torch.nn.functional as F from torch import nn -from transformers.optimization import Adafactor, get_constant_schedule_with_warmup +from transformers.optimization import (Adafactor, + get_constant_schedule_with_warmup) from comet.models.base import CometModel from comet.models.metrics import WMTKendall diff --git a/comet/models/regression/referenceless.py b/comet/models/regression/referenceless.py index 8a95f6e5..b35ac95b 100644 --- a/comet/models/regression/referenceless.py +++ b/comet/models/regression/referenceless.py @@ -129,7 +129,7 @@ def __init__( def requires_references(self) -> bool: return False - + def enable_context(self): if self.pool == "avg": self.use_context = True @@ -160,7 +160,7 @@ def prepare_sample( if stage == "predict": return model_inputs - + scores = [float(s["score"]) for s in sample] targets = Target(score=torch.tensor(scores, dtype=torch.float)) diff --git a/comet/models/regression/regression_metric.py b/comet/models/regression/regression_metric.py index ba810e28..abaf0fab 100644 --- a/comet/models/regression/regression_metric.py +++ b/comet/models/regression/regression_metric.py @@ -24,7 +24,8 @@ import pandas as pd import torch from torch import nn -from transformers.optimization import Adafactor, get_constant_schedule_with_warmup +from transformers.optimization import (Adafactor, + get_constant_schedule_with_warmup) from comet.models.base import CometModel from comet.models.metrics import RegressionMetrics diff --git a/comet/models/utils.py b/comet/models/utils.py index a360f07a..8b98c9f4 100644 --- a/comet/models/utils.py +++ b/comet/models/utils.py @@ -13,26 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import itertools -from typing import List, Tuple, Any +from collections import OrderedDict +from typing import Any, List, Tuple import torch from torch.utils.data import Sampler -from collections import OrderedDict class ModelOutput(OrderedDict): - """ This was copied from previous versions of HuggingFace Transformers. Latest + """This was copied from previous versions of HuggingFace Transformers. Latest version made some breaking changes into ModelOutputs which impacted Prediction and Target classes defined bellow. Base class for all model outputs as dataclass. Has a `__getitem__` that allows - indexing by integer or slice (like a tuple) or strings (like a dictionary) + indexing by integer or slice (like a tuple) or strings (like a dictionary) that will ignore the `None` attributes. Otherwise behaves like a regular python dictionary. - You can't unpack a `ModelOutput` directly. Use the [`to_tuple`] method to + You can't unpack a `ModelOutput` directly. Use the [`to_tuple`] method to convert it to a tuple before. @@ -45,10 +45,14 @@ def __post_init__(self): if not len(class_fields): raise ValueError(f"{self.__class__.__name__} has no fields.") if not all(field.default is None for field in class_fields[1:]): - raise ValueError(f"{self.__class__.__name__} should not have more than one required field.") + raise ValueError( + f"{self.__class__.__name__} should not have more than one required field." + ) first_field = getattr(self, class_fields[0].name) - other_fields_are_none = all(getattr(self, field.name) is None for field in class_fields[1:]) + other_fields_are_none = all( + getattr(self, field.name) is None for field in class_fields[1:] + ) if other_fields_are_none and not is_tensor(first_field): if isinstance(first_field, dict): @@ -83,16 +87,24 @@ def __post_init__(self): self[field.name] = v def __delitem__(self, *args, **kwargs): - raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + raise Exception( + f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance." + ) def setdefault(self, *args, **kwargs): - raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + raise Exception( + f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance." + ) def pop(self, *args, **kwargs): - raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + raise Exception( + f"You cannot use ``pop`` on a {self.__class__.__name__} instance." + ) def update(self, *args, **kwargs): - raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + raise Exception( + f"You cannot use ``update`` on a {self.__class__.__name__} instance." + ) def __getitem__(self, k): if isinstance(k, str): diff --git a/comet/modules/feedforward.py b/comet/modules/feedforward.py index b7958bb6..a19a74b3 100644 --- a/comet/modules/feedforward.py +++ b/comet/modules/feedforward.py @@ -74,9 +74,9 @@ def forward(self, in_features: torch.Tensor) -> torch.Tensor: # Check the dtype of self.ff parameters ff_dtypes = {param.dtype for param in self.ff.parameters()} - + # If all parameters are float16 and in_features is not, convert it if ff_dtypes == {torch.float16} and in_features.dtype != torch.float16: in_features = in_features.to(torch.float16) - + return self.ff(in_features) diff --git a/comet/modules/layerwise_attention.py b/comet/modules/layerwise_attention.py index 859edb7f..b22ffba1 100644 --- a/comet/modules/layerwise_attention.py +++ b/comet/modules/layerwise_attention.py @@ -49,6 +49,7 @@ def __init__( self.transform_fn = torch.softmax if layer_transformation == "sparsemax": from entmax import sparsemax + self.transform_fn = sparsemax if layer_weights is None: diff --git a/tests/integration/models/test_ranking_metric.py b/tests/integration/models/test_ranking_metric.py index 903e61d9..a4e59f26 100644 --- a/tests/integration/models/test_ranking_metric.py +++ b/tests/integration/models/test_ranking_metric.py @@ -5,13 +5,14 @@ import warnings import torch -from comet.models import RankingMetric from pytorch_lightning import seed_everything from pytorch_lightning.trainer.trainer import Trainer from scipy.stats import pearsonr -from tests.data import DATA_PATH from torch.utils.data import DataLoader +from comet.models import RankingMetric +from tests.data import DATA_PATH + os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["OMP_NUM_THREADS"] = "1" diff --git a/tests/integration/models/test_referenceless_regression.py b/tests/integration/models/test_referenceless_regression.py index 44f2c8cf..06e88cc3 100644 --- a/tests/integration/models/test_referenceless_regression.py +++ b/tests/integration/models/test_referenceless_regression.py @@ -5,13 +5,14 @@ import warnings import torch -from comet.models import ReferencelessRegression from pytorch_lightning import seed_everything from pytorch_lightning.trainer.trainer import Trainer from scipy.stats import pearsonr -from tests.data import DATA_PATH from torch.utils.data import DataLoader +from comet.models import ReferencelessRegression +from tests.data import DATA_PATH + os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["OMP_NUM_THREADS"] = "1" diff --git a/tests/integration/models/test_regression_metric.py b/tests/integration/models/test_regression_metric.py index 7a7058ec..efadaa79 100644 --- a/tests/integration/models/test_regression_metric.py +++ b/tests/integration/models/test_regression_metric.py @@ -5,13 +5,14 @@ import warnings import torch -from comet.models import RegressionMetric from pytorch_lightning import seed_everything from pytorch_lightning.trainer.trainer import Trainer from scipy.stats import pearsonr -from tests.data import DATA_PATH from torch.utils.data import DataLoader +from comet.models import RegressionMetric +from tests.data import DATA_PATH + os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["OMP_NUM_THREADS"] = "1" diff --git a/tests/integration/models/test_unified_metric.py b/tests/integration/models/test_unified_metric.py index 13bd9c2a..1fc455e3 100644 --- a/tests/integration/models/test_unified_metric.py +++ b/tests/integration/models/test_unified_metric.py @@ -155,4 +155,3 @@ def test_regression_without_references(self): ) y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.9 - diff --git a/tests/integration/modules/test_feedforward.py b/tests/integration/modules/test_feedforward.py index 3b1aaf9f..3a28a905 100644 --- a/tests/integration/modules/test_feedforward.py +++ b/tests/integration/modules/test_feedforward.py @@ -2,12 +2,12 @@ import unittest import torch +from pytorch_lightning import seed_everything from sklearn.datasets import load_digits from sklearn.model_selection import train_test_split from torch import nn from comet.modules.feedforward import FeedForward -from pytorch_lightning import seed_everything class TestFeedForward(unittest.TestCase): diff --git a/tests/unit/test_download_load.py b/tests/unit/test_download_load.py index be334398..80ba4e61 100644 --- a/tests/unit/test_download_load.py +++ b/tests/unit/test_download_load.py @@ -1,10 +1,11 @@ # -*- coding: utf-8 -*- -import unittest import os import shutil -from tests.data import DATA_PATH +import unittest + from comet import download_model from comet.models import load_from_checkpoint +from tests.data import DATA_PATH class TestDownloadModel(unittest.TestCase): diff --git a/tests/unit/test_models_predict.py b/tests/unit/test_models_predict.py index a1d0d1b5..f80c27eb 100644 --- a/tests/unit/test_models_predict.py +++ b/tests/unit/test_models_predict.py @@ -11,32 +11,181 @@ from tests.data import DATA_PATH TEST_SAMPLES = [ - {"lp": "it-en", "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", "mt": "In 1884, Tesla accepted a job at the Pacific League of New York City and moved to the United States of America.", "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", "annotations": [{"start": 37, "end": 51, "text": "Pacific League", "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.5833333333333333}, - {"lp": "it-en", "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", "mt": "In 1884, Tesla accepted a job at the Edison Company of New York City and moved to the United States of America.", "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", "mt": "The American Civil War was also an inspiration for many other oppressed workers from various countries to start their own revolution.", "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", "annotations": [{"text": "American Civil War", "start": 4, "end": 22, "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.5454545454545454}, - {"lp": "it-en", "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", "mt": "The French Revolution was also an inspiration for many other oppressed workers from various countries to start their own revolution.", "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", "mt": "Lokomotiv Moscow activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", "annotations": [{"start": 0, "end": 16, "text": "Lokomotiv Moscow", "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.6}, - {"lp": "it-en", "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", "mt": "Kundalini Yoga activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", "mt": "The East African islands are located in the Longxi River, off the east coast of Africa.", "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", "annotations": [{"start": 44, "end": 56, "text": "Longxi River", "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.4444444444444444}, - {"lp": "it-en", "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of American Dollar (GBP).", "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", "annotations": [{"text": "of American Dollar", "start": 112, "end": 130, "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.6551724137931034}, - {"lp": "it-en", "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of the British Pound (GBP).", "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful Indonesian wildlife.", "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", "annotations": [{"text": "Indonesian", "start": 115, "end": 125, "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.6153846153846154}, - {"lp": "it-en", "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful African wildlife.", "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", "annotations": [], "score": 1.0} + { + "lp": "it-en", + "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", + "mt": "In 1884, Tesla accepted a job at the Pacific League of New York City and moved to the United States of America.", + "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", + "annotations": [ + { + "start": 37, + "end": 51, + "text": "Pacific League", + "category": "critical_id9_ne_replaced", + "severity": "major", + } + ], + "score": 0.5833333333333333, + }, + { + "lp": "it-en", + "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", + "mt": "In 1884, Tesla accepted a job at the Edison Company of New York City and moved to the United States of America.", + "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", + "annotations": [], + "score": 1.0, + }, + { + "lp": "it-en", + "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", + "mt": "The American Civil War was also an inspiration for many other oppressed workers from various countries to start their own revolution.", + "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", + "annotations": [ + { + "text": "American Civil War", + "start": 4, + "end": 22, + "category": "critical_id9_ne_replaced", + "severity": "major", + } + ], + "score": 0.5454545454545454, + }, + { + "lp": "it-en", + "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", + "mt": "The French Revolution was also an inspiration for many other oppressed workers from various countries to start their own revolution.", + "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", + "annotations": [], + "score": 1.0, + }, + { + "lp": "it-en", + "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", + "mt": "Lokomotiv Moscow activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", + "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", + "annotations": [ + { + "start": 0, + "end": 16, + "text": "Lokomotiv Moscow", + "category": "critical_id9_ne_replaced", + "severity": "major", + } + ], + "score": 0.6, + }, + { + "lp": "it-en", + "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", + "mt": "Kundalini Yoga activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", + "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", + "annotations": [], + "score": 1.0, + }, + { + "lp": "it-en", + "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + "mt": "The East African islands are located in the Longxi River, off the east coast of Africa.", + "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", + "annotations": [ + { + "start": 44, + "end": 56, + "text": "Longxi River", + "category": "critical_id9_ne_replaced", + "severity": "major", + } + ], + "score": 0.4444444444444444, + }, + { + "lp": "it-en", + "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", + "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", + "annotations": [], + "score": 1.0, + }, + { + "lp": "it-en", + "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", + "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of American Dollar (GBP).", + "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", + "annotations": [ + { + "text": "of American Dollar", + "start": 112, + "end": 130, + "category": "critical_id9_ne_replaced", + "severity": "major", + } + ], + "score": 0.6551724137931034, + }, + { + "lp": "it-en", + "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", + "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of the British Pound (GBP).", + "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", + "annotations": [], + "score": 1.0, + }, + { + "lp": "it-en", + "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", + "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful Indonesian wildlife.", + "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", + "annotations": [ + { + "text": "Indonesian", + "start": 115, + "end": 125, + "category": "critical_id9_ne_replaced", + "severity": "major", + } + ], + "score": 0.6153846153846154, + }, + { + "lp": "it-en", + "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", + "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful African wildlife.", + "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", + "annotations": [], + "score": 1.0, + }, ] CONTEXT_TEST_SAMPLES = [ - {"lp": "it-en", "context_src": None, "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", "context_mt": None, "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", "context_ref": None, "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", "annotations": [], "score": 1.0}, + { + "lp": "it-en", + "context_src": None, + "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + "context_mt": None, + "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", + "context_ref": None, + "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", + "annotations": [], + "score": 1.0, + }, ] + class TestUnifiedMetricPredict(unittest.TestCase): - - model = load_from_checkpoint(download_model("Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH)) + + model = load_from_checkpoint( + download_model( + "Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH + ) + ) gpus = 1 if torch.cuda.device_count() > 0 else 0 - + @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "models--Unbabel--test-model-whimsical-whisper")) + shutil.rmtree( + os.path.join(DATA_PATH, "models--Unbabel--test-model-whimsical-whisper") + ) def test_predict(self): model_output = self.model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) @@ -44,27 +193,43 @@ def test_predict(self): assert "src_scores" in model_output.metadata assert "ref_scores" in model_output.metadata assert "unified_scores" in model_output.metadata - + expected_scores = np.array( - [model_output.metadata.src_scores, model_output.metadata.ref_scores, model_output.metadata.unified_scores] + [ + model_output.metadata.src_scores, + model_output.metadata.ref_scores, + model_output.metadata.unified_scores, + ] ).mean(axis=0) - + # Assert for almost equal Arrays or Numbers - np.testing.assert_almost_equal(expected_scores, np.array(model_output.scores), decimal=5) - np.testing.assert_almost_equal(model_output.system_score, expected_scores.mean(), 5) + np.testing.assert_almost_equal( + expected_scores, np.array(model_output.scores), decimal=5 + ) + np.testing.assert_almost_equal( + model_output.system_score, expected_scores.mean(), 5 + ) def test_context_predict(self): self.model.enable_context() assert self.model.use_context == False - + def test_length_batching(self): - output_without_length_batching = self.model.predict(TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=False) - output_with_length_batching = self.model.predict(TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=True) - self.assertListEqual(output_without_length_batching.scores, output_with_length_batching.scores) - + output_without_length_batching = self.model.predict( + TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=False + ) + output_with_length_batching = self.model.predict( + TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=True + ) + self.assertListEqual( + output_without_length_batching.scores, output_with_length_batching.scores + ) + def test_xcomet_predict(self): model = XCOMETMetric.load_from_checkpoint( - checkpoint_path=download_model("Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH), + checkpoint_path=download_model( + "Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH + ), map_location=torch.device("cpu"), strict=False, **dict(self.model.hparams), @@ -72,15 +237,40 @@ def test_xcomet_predict(self): model.score_weights = [0.25, 0.25, 0.25, 0.25] model_output = model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) assert "mqm_scores" in model_output.metadata - + # on XCOMET we cap all scores at 1. and final score is a weighted average of 4 features. - expected_scores = np.array([ - np.array(list(map(lambda x: 1.0 if x > 1.0 else x, model_output.metadata.src_scores))), - np.array(list(map(lambda x: 1.0 if x > 1.0 else x, model_output.metadata.ref_scores))), - np.array(list(map(lambda x: 1.0 if x > 1.0 else x, model_output.metadata.unified_scores))), - model_output.metadata.mqm_scores - ]).mean(axis=0) - np.testing.assert_almost_equal(expected_scores, np.array(model_output.scores), decimal=5) + expected_scores = np.array( + [ + np.array( + list( + map( + lambda x: 1.0 if x > 1.0 else x, + model_output.metadata.src_scores, + ) + ) + ), + np.array( + list( + map( + lambda x: 1.0 if x > 1.0 else x, + model_output.metadata.ref_scores, + ) + ) + ), + np.array( + list( + map( + lambda x: 1.0 if x > 1.0 else x, + model_output.metadata.unified_scores, + ) + ) + ), + model_output.metadata.mqm_scores, + ] + ).mean(axis=0) + np.testing.assert_almost_equal( + expected_scores, np.array(model_output.scores), decimal=5 + ) # Put all the weight on MQM score. model.score_weights = [0, 0, 0, 1] @@ -89,14 +279,24 @@ def test_xcomet_predict(self): class TestRegressionMetricPredict(unittest.TestCase): - - model = load_from_checkpoint(download_model("Unbabel/eamt22-cometinho-da", saving_directory=DATA_PATH)) + + model = load_from_checkpoint( + download_model("Unbabel/eamt22-cometinho-da", saving_directory=DATA_PATH) + ) gpus = 1 if torch.cuda.device_count() > 0 else 0 - + def test_context_predict(self): # Enabling context should not change scores" - model_output_context_disabled = self.model.predict(CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus) + model_output_context_disabled = self.model.predict( + CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus + ) self.model.enable_context() assert self.model.use_context == True - model_output_context_enabled = self.model.predict(CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus) - np.testing.assert_almost_equal(np.array(model_output_context_disabled.scores), np.array(model_output_context_enabled.scores), decimal=5) + model_output_context_enabled = self.model.predict( + CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus + ) + np.testing.assert_almost_equal( + np.array(model_output_context_disabled.scores), + np.array(model_output_context_enabled.scores), + decimal=5, + ) From c176ae5344ec54d001a5a3ba791de340715acaae Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Thu, 14 May 2026 19:01:31 +0900 Subject: [PATCH 03/13] Update dependencies --- comet/encoders/bert.py | 23 +- comet/encoders/minilm.py | 13 +- comet/encoders/rembert.py | 12 +- comet/encoders/xlmr.py | 24 +- comet/encoders/xlmr_xl.py | 17 +- comet/models/__init__.py | 5 +- poetry.lock | 1855 +++++++++++++++++++++++++------------ pyproject.toml | 10 +- 8 files changed, 1355 insertions(+), 604 deletions(-) diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index c6e96796..e02e2c9f 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -20,7 +20,16 @@ from typing import Dict, Optional import torch -from transformers import BertConfig, BertModel, BertTokenizerFast +from transformers import BertConfig, BertModel + +import importlib_metadata +import packaging.version as packaging_version + +transformers_version = importlib_metadata.distribution("transformers").version +if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): + from transformers import BertTokenizer as BertTokenizer +else: + from transformers import BertTokenizerFast as BertTokenizer from comet.encoders.base import Encoder @@ -42,19 +51,19 @@ def __init__( local_files_only: bool = False, ) -> None: super().__init__() - self.tokenizer = BertTokenizerFast.from_pretrained( + self.tokenizer = BertTokenizer.from_pretrained( pretrained_model, use_fast=True, local_files_only=local_files_only ) if load_pretrained_weights: self.model = BertModel.from_pretrained( - pretrained_model, add_pooling_layer=False + pretrained_model, add_pooling_layer=True ) else: self.model = BertModel( BertConfig.from_pretrained( pretrained_model, local_files_only=local_files_only ), - add_pooling_layer=False, + add_pooling_layer=True, ) self.model.encoder.output_hidden_states = True @@ -168,13 +177,17 @@ def forward( Dict[str, torch.Tensor]: dictionary with 'sentemb', 'wordemb', 'all_layers' and 'attention_mask'. """ - last_hidden_states, pooler_output, all_layers = self.model( + output = self.model( input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask, output_hidden_states=True, return_dict=False, ) + if len(output) == 3: + last_hidden_states, pooler_output, all_layers = output + else: + last_hidden_states, all_layers = output return { "sentemb": pooler_output, "wordemb": last_hidden_states, diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index 13cc63d4..ce950199 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -18,7 +18,16 @@ Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT architecture with an XLMR tokenizer. """ -from transformers import BertConfig, BertModel, XLMRobertaTokenizerFast +import importlib_metadata +import packaging.version as packaging_version + +from transformers import BertConfig, BertModel + +transformers_version = importlib_metadata.distribution("transformers").version +if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): + from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer +else: + from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer from comet.encoders.xlmr import Encoder, XLMREncoder @@ -40,7 +49,7 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = XLMRobertaTokenizerFast.from_pretrained( + self.tokenizer = XLMRobertaTokenizer.from_pretrained( "xlm-roberta-base", use_fast=True, local_files_only=local_files_only ) if load_pretrained_weights: diff --git a/comet/encoders/rembert.py b/comet/encoders/rembert.py index 448e685f..da8257f1 100644 --- a/comet/encoders/rembert.py +++ b/comet/encoders/rembert.py @@ -18,7 +18,15 @@ Pretrained RemBERT encoder from Google. This encoder is similar to BERT but uses sentencepiece like XLMR. """ -from transformers import RemBertConfig, RemBertModel, RemBertTokenizerFast +from transformers import RemBertConfig, RemBertModel +import importlib_metadata +import packaging.version as packaging_version + +transformers_version = importlib_metadata.distribution("transformers").version +if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): + from transformers import RemBertTokenizer as RemBertTokenizer +else: + from transformers import RemBertTokenizerFast as RemBertTokenizer from comet.encoders.xlmr import Encoder, XLMREncoder @@ -40,7 +48,7 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = RemBertTokenizerFast.from_pretrained( + self.tokenizer = RemBertTokenizer.from_pretrained( pretrained_model, use_fast=True, local_files_only=local_files_only ) if load_pretrained_weights: diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index 136e1678..79d34da8 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -20,8 +20,19 @@ from typing import Dict import torch -from transformers import (XLMRobertaConfig, XLMRobertaModel, - XLMRobertaTokenizerFast) +import importlib_metadata +import packaging.version as packaging_version + +from transformers import ( + XLMRobertaConfig, + XLMRobertaModel +) + +transformers_version = importlib_metadata.distribution("transformers").version +if packaging_version.parse(transformers_version) >= packaging_version.parse("v5.0.0rc0"): + from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer +else: + from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer from comet.encoders.base import Encoder from comet.encoders.bert import BERTEncoder @@ -44,7 +55,7 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = XLMRobertaTokenizerFast.from_pretrained( + self.tokenizer = XLMRobertaTokenizer.from_pretrained( pretrained_model, local_files_only=local_files_only ) if load_pretrained_weights: @@ -93,13 +104,18 @@ def from_pretrained( def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs ) -> Dict[str, torch.Tensor]: - last_hidden_states, all_layers = self.model( + output = self.model( input_ids=input_ids, attention_mask=attention_mask, output_hidden_states=True, return_dict=False, ) + if len(output) == 2: + last_hidden_states, all_layers = output + else: + last_hidden_states, _, all_layers = output + return { "sentemb": last_hidden_states[:, 0, :], "wordemb": last_hidden_states, diff --git a/comet/encoders/xlmr_xl.py b/comet/encoders/xlmr_xl.py index b3a1625f..6334be5f 100644 --- a/comet/encoders/xlmr_xl.py +++ b/comet/encoders/xlmr_xl.py @@ -17,8 +17,19 @@ ============== Pretrained XLM-RoBERTa-XL encoder from Hugging Face. """ -from transformers import (XLMRobertaTokenizerFast, XLMRobertaXLConfig, - XLMRobertaXLModel) +import importlib_metadata +import packaging.version as packaging_version + +from transformers import ( + XLMRobertaXLConfig, + XLMRobertaXLModel +) + +transformers_version = importlib_metadata.distribution("transformers").version +if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): + from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer +else: + from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer from comet.encoders.base import Encoder from comet.encoders.xlmr import XLMREncoder @@ -41,7 +52,7 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = XLMRobertaTokenizerFast.from_pretrained( + self.tokenizer = XLMRobertaTokenizer.from_pretrained( pretrained_model, local_files_only=local_files_only ) if load_pretrained_weights: diff --git a/comet/models/__init__.py b/comet/models/__init__.py index 5211ff5f..29342665 100644 --- a/comet/models/__init__.py +++ b/comet/models/__init__.py @@ -98,12 +98,11 @@ def load_from_checkpoint( # issue number #244 try: from importlib import metadata - - from packaging.version import Version as parse_version + import packaging.version as parse_version comet_version = metadata.distribution("unbabel-comet").version use_softmax = ( - parse_version(comet_version) >= parse_version("2.2.4") + parse_version.parse(comet_version) >= parse_version.parse("2.2.4") and hparams.get("layer_transformation") == "sparsemax_patch" ) except: diff --git a/poetry.lock b/poetry.lock index aa407bb3..8dac3a6c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -6,6 +6,7 @@ version = "2.4.4" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"}, {file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"}, @@ -17,6 +18,7 @@ version = "3.10.11" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5077b1a5f40ffa3ba1f40d537d3bec4383988ee51fbba6b74aa8fb1bc466599e"}, {file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d6a14a4d93b5b3c2891fca94fa9d41b2322a68194422bef0dd5ec1e57d7d298"}, @@ -121,7 +123,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.12.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] +speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.2.0) ; sys_platform == \"linux\" or sys_platform == \"darwin\"", "brotlicffi ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" @@ -129,6 +131,7 @@ version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, @@ -137,12 +140,46 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "annotated-doc" +version = "0.0.4" +description = "Document parameters, class attributes, return types, and variables inline, with Annotated." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320"}, + {file = "annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4"}, +] + +[[package]] +name = "anyio" +version = "4.13.0" +description = "High-level concurrency and networking framework on top of asyncio or Trio" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708"}, + {file = "anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +trio = ["trio (>=0.32.0)"] + [[package]] name = "async-timeout" version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.11\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -154,18 +191,19 @@ version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""] [[package]] name = "certifi" @@ -173,131 +211,34 @@ version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] -name = "charset-normalizer" -version = "3.4.0" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, - {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, - {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +name = "click" +version = "8.3.3" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613"}, + {file = "click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2"}, ] +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -309,6 +250,7 @@ version = "5.5" description = "Code coverage measurement for Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +groups = ["dev"] files = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, {file = "coverage-5.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b"}, @@ -367,12 +309,112 @@ files = [ [package.extras] toml = ["toml"] +[[package]] +name = "cuda-bindings" +version = "13.2.0" +description = "Python bindings for CUDA" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, + {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, + {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, + {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, + {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, + {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, + {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, + {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, + {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, + {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, + {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, + {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, + {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, + {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, + {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, + {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, + {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, + {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, +] + +[package.dependencies] +cuda-pathfinder = ">=1.1,<2.0" + +[package.extras] +all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] + +[[package]] +name = "cuda-pathfinder" +version = "1.5.4" +description = "Pathfinder for CUDA components" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +description = "CUDA Toolkit meta-package" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb"}, +] + +[package.dependencies] +nvidia-cuda-cupti = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cupti\""} +nvidia-cuda-nvrtc = {version = "==13.0.88.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"nvrtc\""} +nvidia-cuda-runtime = {version = "==13.0.96.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cudart\""} +nvidia-cufft = {version = "==12.0.0.61.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cufft\""} +nvidia-cufile = {version = "==1.15.1.6.*", optional = true, markers = "sys_platform == \"linux\" and extra == \"cufile\""} +nvidia-curand = {version = "==10.4.0.35.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"curand\""} +nvidia-cusolver = {version = "==12.0.4.66.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cusolver\""} +nvidia-cusparse = {version = "==12.6.3.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cusparse\""} +nvidia-nvjitlink = {version = "==13.0.88.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"nvjitlink\""} +nvidia-nvtx = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"nvtx\""} + +[package.extras] +all = ["nvidia-cublas (==13.1.0.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\"", "nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\"", "nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvjitlink (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cccl = ["nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +crt = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cublas = ["nvidia-cublas (==13.1.0.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cudart = ["nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cufft = ["nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cufile = ["nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\""] +culibos = ["nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\""] +cupti = ["nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +curand = ["nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cusolver = ["nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cusparse = ["nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cuxxfilt = ["nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +npp = ["nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvcc = ["nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvfatbin = ["nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvjitlink = ["nvidia-nvjitlink (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvjpeg = ["nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvml = ["nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvptxcompiler = ["nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvrtc = ["nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvtx = ["nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvvm = ["nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +opencl = ["nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +profiler = ["nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +sanitizer = ["nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] + [[package]] name = "entmax" version = "1.3" description = "The entmax mapping and its loss, a family of sparse alternatives to softmax." optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "entmax-1.3-py3-none-any.whl", hash = "sha256:41bb1edbd497a1b53b1f0fc80befd543499dc704b4e5436ddf6c5728a2d00546"}, ] @@ -380,12 +422,32 @@ files = [ [package.dependencies] torch = ">=1.3" +[[package]] +name = "exceptiongroup" +version = "1.3.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, + {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "filelock" version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, @@ -394,7 +456,7 @@ files = [ [package.extras] docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] -typing = ["typing-extensions (>=4.12.2)"] +typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""] [[package]] name = "frozenlist" @@ -402,6 +464,7 @@ version = "1.5.0" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, @@ -503,6 +566,7 @@ version = "2024.10.0" description = "File-system specification" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, @@ -539,39 +603,139 @@ test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe, test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] tqdm = ["tqdm"] +[[package]] +name = "h11" +version = "0.16.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, +] + +[[package]] +name = "hf-xet" +version = "1.5.0" +description = "Fast transfer of large files with the Hugging Face Hub." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" +files = [ + {file = "hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c"}, + {file = "hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42"}, + {file = "hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a"}, + {file = "hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480"}, + {file = "hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216"}, + {file = "hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60"}, + {file = "hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d"}, + {file = "hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4"}, + {file = "hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c"}, + {file = "hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73"}, + {file = "hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682"}, + {file = "hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761"}, + {file = "hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded"}, + {file = "hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702"}, + {file = "hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e"}, + {file = "hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0"}, + {file = "hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56"}, + {file = "hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a"}, + {file = "hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949"}, + {file = "hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b"}, + {file = "hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18"}, + {file = "hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690"}, + {file = "hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4"}, + {file = "hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be"}, + {file = "hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948"}, +] + +[package.extras] +tests = ["pytest"] + +[[package]] +name = "httpcore" +version = "1.0.9" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.16" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "huggingface-hub" -version = "0.26.3" +version = "1.14.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false -python-versions = ">=3.8.0" +python-versions = ">=3.10.0" +groups = ["main"] files = [ - {file = "huggingface_hub-0.26.3-py3-none-any.whl", hash = "sha256:e66aa99e569c2d5419240a9e553ad07245a5b1300350bfbc5a4945cf7432991b"}, - {file = "huggingface_hub-0.26.3.tar.gz", hash = "sha256:90e1fe62ffc26757a073aaad618422b899ccf9447c2bba8c902a90bef5b42e1d"}, + {file = "huggingface_hub-1.14.0-py3-none-any.whl", hash = "sha256:efe075535c62e130b30e836b138e13785f6f043d1f0539e0a39aa411a99e90b8"}, + {file = "huggingface_hub-1.14.0.tar.gz", hash = "sha256:d6d2c9cd6be1d02ae9ec6672d5587d10a427f377db688e82528f426a041622c2"}, ] [package.dependencies] -filelock = "*" +filelock = ">=3.10.0" fsspec = ">=2023.5.0" +hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} +httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" -requests = "*" tqdm = ">=4.42.1" -typing-extensions = ">=3.7.4.3" +typer = ">=0.20.0" +typing-extensions = ">=4.1.0" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -hf-transfer = ["hf-transfer (>=0.1.4)"] -inference = ["aiohttp"] -quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.5.0)"] -tensorflow = ["graphviz", "pydot", "tensorflow"] -tensorflow-testing = ["keras (<3.0)", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +gradio = ["gradio (>=5.0.0)", "requests"] +hf-xet = ["hf-xet (>=1.4.3,<2.0.0)"] +mcp = ["mcp (>=1.8.0)"] +oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] +quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] +testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] +typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] [[package]] name = "idna" @@ -579,6 +743,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -587,35 +752,13 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] -[[package]] -name = "importlib-metadata" -version = "8.5.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, - {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, -] - -[package.dependencies] -zipp = ">=3.20" - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] -type = ["pytest-mypy"] - [[package]] name = "jinja2" version = "3.1.4" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, @@ -633,6 +776,7 @@ version = "1.4.2" description = "Lightweight pipelining with Python functions" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, @@ -644,6 +788,7 @@ version = "3.13.1" description = "Parsing of command line options, yaml/jsonnet config files and/or environment variables based on argparse." optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "jsonargparse-3.13.1-py3-none-any.whl", hash = "sha256:b58188b98f2ac2b1f5007ece7ea821628883ce3f3ee448eb17c72d9d3b8ec893"}, {file = "jsonargparse-3.13.1.tar.gz", hash = "sha256:705693e9911223bf928fe4a7ed9538f24d9229c55ef5c3d3e9a8bad0e7982cf2"}, @@ -653,15 +798,15 @@ files = [ PyYAML = ">=3.13" [package.extras] -all = ["argcomplete (>=1.12.1)", "docstring-parser (>=0.7.3)", "fsspec (>=0.8.4)", "jsonnet (>=0.13.0)", "jsonschema (>=3.2.0)", "reconplogger (>=4.4.0)", "requests (>=2.18.4)", "ruyaml (>=0.20.0)", "validators (>=0.14.2)"] +all = ["argcomplete (>=1.12.1)", "docstring-parser (>=0.7.3)", "fsspec (>=0.8.4) ; python_version >= \"3.6\"", "jsonnet (>=0.13.0)", "jsonschema (>=3.2.0)", "reconplogger (>=4.4.0)", "requests (>=2.18.4)", "ruyaml (>=0.20.0) ; python_version >= \"3.6\"", "validators (>=0.14.2)"] argcomplete = ["argcomplete (>=1.12.1)"] dev = ["bump2version (>=0.5.11)", "coverage (>=4.5.1)", "mypy (>=0.701)", "pycodestyle (>=2.5.0)", "pylint (>=1.8.3)", "responses (>=0.12.0)", "twine (>=3.1.1)"] doc = ["Sphinx (>=1.7.9)", "autodocsumm (>=0.1.10)", "sphinx-autodoc-typehints (>=1.11.1)", "sphinx-rtd-theme (>=0.4.3)"] -fsspec = ["fsspec (>=0.8.4)"] +fsspec = ["fsspec (>=0.8.4) ; python_version >= \"3.6\""] jsonnet = ["jsonnet (>=0.13.0)"] jsonschema = ["jsonschema (>=3.2.0)"] reconplogger = ["reconplogger (>=4.4.0)"] -ruyaml = ["ruyaml (>=0.20.0)"] +ruyaml = ["ruyaml (>=0.20.0) ; python_version >= \"3.6\""] signatures = ["docstring-parser (>=0.7.3)"] test = ["coverage (>=4.5.1)", "responses (>=0.12.0)"] test-no-urls = ["coverage (>=4.5.1)"] @@ -673,6 +818,7 @@ version = "0.11.9" description = "Lightning toolbox for across the our ecosystem." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "lightning_utilities-0.11.9-py3-none-any.whl", hash = "sha256:ac6d4e9e28faf3ff4be997876750fee10dc604753dbc429bf3848a95c5d7e0d2"}, {file = "lightning_utilities-0.11.9.tar.gz", hash = "sha256:f5052b81344cc2684aa9afd74b7ce8819a8f49a858184ec04548a5a109dfd053"}, @@ -694,6 +840,7 @@ version = "5.3.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, @@ -848,24 +995,47 @@ version = "3.7" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] -[package.dependencies] -importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} - [package.extras] docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] testing = ["coverage", "pyyaml"] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a"}, + {file = "markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "markdown-it-pyrs", "mistletoe (>=1.0,<2.0)", "mistune (>=3.0,<4.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins (>=0.5.0)"] +profiling = ["gprof2dot"] +rtd = ["ipykernel", "jupyter_sphinx", "mdit-py-plugins (>=0.5.0)", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme (>=1.0,<2.0)", "sphinx-copybutton", "sphinx-design"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions", "pytest-timeout", "requests"] + [[package]] name = "markupsafe" version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, @@ -929,12 +1099,25 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, @@ -943,7 +1126,7 @@ files = [ [package.extras] develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4)"] +gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] tests = ["pytest (>=4.6)"] [[package]] @@ -952,6 +1135,7 @@ version = "6.1.0" description = "multidict implementation" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, @@ -1056,6 +1240,7 @@ version = "3.1" description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, @@ -1074,6 +1259,8 @@ version = "1.24.4" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.8" +groups = ["main", "dev"] +markers = "python_version <= \"3.11\"" files = [ {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, @@ -1105,105 +1292,331 @@ files = [ {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] +[[package]] +name = "numpy" +version = "1.26.4" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +groups = ["main", "dev"] +markers = "python_version >= \"3.12\"" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "nvidia-cublas" +version = "13.1.1.3" +description = "CUBLAS native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5"}, + {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436"}, + {file = "nvidia_cublas-13.1.1.3-py3-none-win_amd64.whl", hash = "sha256:b6cdce694e47ff6aadf0a69df1cab6628d696f5ff56e8d16af50309d855fa20f"}, +] + +[package.dependencies] +nvidia-cuda-nvrtc = "*" + [[package]] name = "nvidia-cublas-cu12" -version = "12.4.5.8" +version = "12.6.4.1" description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0f8aa1706812e00b9f19dfe0cdb3999b092ccb8ca168c0db5b8ea712456fd9b3"}, - {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b"}, - {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-win_amd64.whl", hash = "sha256:5a796786da89203a0657eda402bcdcec6180254a8ac22d72213abc42069522dc"}, + {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb"}, + {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:235f728d6e2a409eddf1df58d5b0921cf80cfa9e72b9f2775ccb7b4a87984668"}, + {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-win_amd64.whl", hash = "sha256:9e4fa264f4d8a4eb0cdbd34beadc029f453b3bafae02401e999cf3d5a5af75f8"}, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +description = "CUDA profiling tools runtime libs." +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151"}, + {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8"}, + {file = "nvidia_cuda_cupti-13.0.85-py3-none-win_amd64.whl", hash = "sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00"}, ] [[package]] name = "nvidia-cuda-cupti-cu12" -version = "12.4.127" +version = "12.6.80" description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:79279b35cf6f91da114182a5ce1864997fd52294a87a16179ce275773799458a"}, - {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb"}, - {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:5688d203301ab051449a2b1cb6690fbe90d2b372f411521c86018b950f3d7922"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:166ee35a3ff1587f2490364f90eeeb8da06cd867bd5b701bf7f9a02b78bc63fc"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.whl", hash = "sha256:358b4a1d35370353d52e12f0a7d1769fc01ff74a191689d3870b2123156184c4"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-win_amd64.whl", hash = "sha256:bbe6ae76e83ce5251b56e8c8e61a964f757175682bbad058b170b136266ab00a"}, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +description = "NVRTC native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575"}, + {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b"}, + {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872"}, ] [[package]] name = "nvidia-cuda-nvrtc-cu12" -version = "12.4.127" +version = "12.6.77" description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5847f1d6e5b757f1d2b3991a01082a44aad6f10ab3c5c0213fa3e25bddc25a13"}, + {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53"}, + {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:f7007dbd914c56bd80ea31bc43e8e149da38f68158f423ba845fc3292684e45a"}, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +description = "CUDA Runtime native Libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" files = [ - {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0eedf14185e04b76aa05b1fea04133e59f465b6f960c0cbf4e37c3cb6b0ea198"}, - {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338"}, - {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:a961b2f1d5f17b14867c619ceb99ef6fcec12e46612711bcec78eb05068a60ec"}, + {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55"}, + {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548"}, + {file = "nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492"}, ] [[package]] name = "nvidia-cuda-runtime-cu12" -version = "12.4.127" +version = "12.6.77" description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:961fe0e2e716a2a1d967aab7caee97512f71767f852f67432d572e36cb3a11f3"}, - {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5"}, - {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:09c2e35f48359752dfa822c09918211844a3d93c100a715d79b59591130c5e1e"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6116fad3e049e04791c0256a9778c16237837c08b27ed8c8401e2e45de8d60cd"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d461264ecb429c84c8879a7153499ddc7b19b5f8d84c204307491989a365588e"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba3b56a4f896141e25e19ab287cd71e52a6a0f4b29d0d31609f60e3b4d5219b7"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a84d15d5e1da416dd4774cb42edf5e954a3e60cc945698dc1d5be02321c44dc8"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:86c58044c824bf3c173c49a2dbc7a6c8b53cb4e4dca50068be0bf64e9dab3f7f"}, ] [[package]] name = "nvidia-cudnn-cu12" -version = "9.1.0.70" +version = "9.5.1.17" description = "cuDNN runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f"}, - {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-win_amd64.whl", hash = "sha256:6278562929433d68365a07a4a1546c237ba2849852c0d4b2262a486e805b977a"}, + {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9fd4584468533c61873e5fda8ca41bac3a38bcb2d12350830c69b0a96a7e4def"}, + {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2"}, + {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-win_amd64.whl", hash = "sha256:d7af0f8a4f3b4b9dbb3122f2ef553b45694ed9c384d5a75bab197b8eefb79ab8"}, ] [package.dependencies] nvidia-cublas-cu12 = "*" +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.20.0.48" +description = "cuDNN runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1"}, + {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304"}, + {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-win_amd64.whl", hash = "sha256:af8139732b99c0118be65ea5aac97f0d46018f8c552889e49d2fb0c6261a4a24"}, +] + +[package.dependencies] +nvidia-cublas = "*" + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +description = "CUFFT native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5"}, + {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3"}, + {file = "nvidia_cufft-12.0.0.61-py3-none-win_amd64.whl", hash = "sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb"}, +] + +[package.dependencies] +nvidia-nvjitlink = "*" + [[package]] name = "nvidia-cufft-cu12" -version = "11.2.1.3" +version = "11.3.0.4" description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399"}, - {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9"}, - {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-win_amd64.whl", hash = "sha256:d802f4954291101186078ccbe22fc285a902136f974d369540fd4a5333d1440b"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d16079550df460376455cba121db6564089176d9bac9e4f360493ca4741b22a6"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8510990de9f96c803a051822618d42bf6cb8f069ff3f48d93a8486efdacb48fb"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.whl", hash = "sha256:768160ac89f6f7b459bee747e8d175dbf53619cfe74b2a5636264163138013ca"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-win_amd64.whl", hash = "sha256:6048ebddfb90d09d2707efb1fd78d4e3a77cb3ae4dc60e19aab6be0ece2ae464"}, ] [package.dependencies] nvidia-nvjitlink-cu12 = "*" +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +description = "cuFile GPUDirect libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "sys_platform == \"linux\" and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44"}, + {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1"}, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.11.1.6" +description = "cuFile GPUDirect libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159"}, + {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8f57a0051dcf2543f6dc2b98a98cb2719c37d3cee1baba8965d57f3bbc90d4db"}, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +description = "CURAND native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a"}, + {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc"}, + {file = "nvidia_curand-10.4.0.35-py3-none-win_amd64.whl", hash = "sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f"}, +] + [[package]] name = "nvidia-curand-cu12" -version = "10.3.5.147" +version = "10.3.7.77" description = "CURAND native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1f173f09e3e3c76ab084aba0de819c49e56614feae5c12f69883f4ae9bb5fad9"}, - {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b"}, - {file = "nvidia_curand_cu12-10.3.5.147-py3-none-win_amd64.whl", hash = "sha256:f307cc191f96efe9e8f05a87096abc20d08845a841889ef78cb06924437f6771"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6e82df077060ea28e37f48a3ec442a8f47690c7499bff392a5938614b56c98d8"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:99f1a32f1ac2bd134897fc7a203f779303261268a65762a623bf30cc9fe79117"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:7b2ed8e95595c3591d984ea3603dd66fe6ce6812b886d59049988a712ed06b6e"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-win_amd64.whl", hash = "sha256:6d6d935ffba0f3d439b7cd968192ff068fafd9018dbf1b85b37261b13cfc9905"}, ] +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +description = "CUDA solver native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2"}, + {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112"}, + {file = "nvidia_cusolver-12.0.4.66-py3-none-win_amd64.whl", hash = "sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65"}, +] + +[package.dependencies] +nvidia-cublas = "*" +nvidia-cusparse = "*" +nvidia-nvjitlink = "*" + [[package]] name = "nvidia-cusolver-cu12" -version = "11.6.1.9" +version = "11.7.1.2" description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e"}, - {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260"}, - {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-win_amd64.whl", hash = "sha256:e77314c9d7b694fcebc84f58989f3aa4fb4cb442f12ca1a9bde50f5e8f6d1b9c"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0ce237ef60acde1efc457335a2ddadfd7610b892d94efee7b776c64bb1cac9e0"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6cf28f17f64107a0c4d7802be5ff5537b2130bfc112f25d5a30df227058ca0e6"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dbbe4fc38ec1289c7e5230e16248365e375c3673c9c8bac5796e2e20db07f56e"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-win_amd64.whl", hash = "sha256:6813f9d8073f555444a8705f3ab0296d3e1cb37a16d694c5fc8b862a0d8706d7"}, ] [package.dependencies] @@ -1211,53 +1624,165 @@ nvidia-cublas-cu12 = "*" nvidia-cusparse-cu12 = "*" nvidia-nvjitlink-cu12 = "*" +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +description = "CUSPARSE native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c"}, + {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b"}, + {file = "nvidia_cusparse-12.6.3.3-py3-none-win_amd64.whl", hash = "sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79"}, +] + +[package.dependencies] +nvidia-nvjitlink = "*" + [[package]] name = "nvidia-cusparse-cu12" -version = "12.3.1.170" +version = "12.5.4.2" description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3"}, - {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1"}, - {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-win_amd64.whl", hash = "sha256:9bc90fb087bc7b4c15641521f31c0371e9a612fc2ba12c338d3ae032e6b6797f"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d25b62fb18751758fe3c93a4a08eff08effedfe4edf1c6bb5afd0890fe88f887"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7aa32fa5470cf754f72d1116c7cbc300b4e638d3ae5304cfa4a638a5b87161b1"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-win_amd64.whl", hash = "sha256:4acb8c08855a26d737398cba8fb6f8f5045d93f82612b4cfd84645a2332ccf20"}, ] [package.dependencies] nvidia-nvjitlink-cu12 = "*" +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.3" +description = "NVIDIA cuSPARSELt" +optional = false +python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1"}, + {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46"}, + {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-win_amd64.whl", hash = "sha256:3b325bcbd9b754ba43df5a311488fca11a6b5dc3d11df4d190c000cf1a0765c7"}, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.1" +description = "NVIDIA cuSPARSELt" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f"}, + {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0"}, + {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-win_amd64.whl", hash = "sha256:dccbd362f91a7b9024d1f55ee9f548ac065027ff15d8c8b0db889ab3a8f31215"}, +] + [[package]] name = "nvidia-nccl-cu12" -version = "2.21.5" +version = "2.26.2" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522"}, + {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6"}, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.29.7" description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" files = [ - {file = "nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0"}, + {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5"}, + {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d"}, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.0.88" +description = "Nvidia JIT LTO Library" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b"}, + {file = "nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c"}, + {file = "nvidia_nvjitlink-13.0.88-py3-none-win_amd64.whl", hash = "sha256:634e96e3da9ef845ae744097a1f289238ecf946ce0b82e93cdce14b9782e682f"}, ] [[package]] name = "nvidia-nvjitlink-cu12" -version = "12.4.127" +version = "12.6.85" description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83"}, - {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57"}, - {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1"}, + {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a"}, + {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf4eaa7d4b6b543ffd69d6abfb11efdeb2db48270d94dfd3a452c24150829e41"}, + {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-win_amd64.whl", hash = "sha256:e61120e52ed675747825cdd16febc6a0730537451d867ee58bee3853b1b13d1c"}, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +description = "NVSHMEM creates a global address space that provides efficient and scalable communication for NVIDIA GPU clusters." +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9"}, + {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80"}, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +description = "NVIDIA Tools Extension" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4"}, + {file = "nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6"}, + {file = "nvidia_nvtx-13.0.85-py3-none-win_amd64.whl", hash = "sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519"}, ] [[package]] name = "nvidia-nvtx-cu12" -version = "12.4.127" +version = "12.6.77" description = "NVIDIA Tools Extension" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3"}, - {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a"}, - {file = "nvidia_nvtx_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:641dccaaa1139f3ffb0d3164b4b84f9d253397e38246a4f2f36728b48566d485"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f44f8d86bb7d5629988d61c8d3ae61dddb2015dee142740536bc7481b022fe4b"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:adcaabb9d436c9761fca2b13959a2d237c5f9fd406c8e4b723c695409ff88059"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b90bed3df379fa79afbd21be8e04a0314336b8ae16768b58f2d34cb1d04cd7d2"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6574241a3ec5fdc9334353ab8c479fe75841dbe8f4532a8fc97ce63503330ba1"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:2fb11a4af04a5e6c84073e6404d26588a34afd35379f0855a99797897efa75c0"}, ] [[package]] @@ -1266,6 +1791,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -1273,70 +1799,194 @@ files = [ [[package]] name = "pandas" -version = "2.0.3" +version = "2.3.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false -python-versions = ">=3.8" -files = [ - {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, - {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, - {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, - {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, - {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, - {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, - {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, - {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, - {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, - {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, - {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version <= \"3.11\" or python_version >= \"3.14\"" +files = [ + {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, + {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, + {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, + {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, + {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, + {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, + {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, + {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, + {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, ] [package.dependencies] numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" -tzdata = ">=2022.1" +tzdata = ">=2022.7" [package.extras] -all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] -aws = ["s3fs (>=2021.08.0)"] -clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] -compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] -computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2021.07.0)"] -gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] -hdf5 = ["tables (>=3.6.1)"] -html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] -mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] -spss = ["pyreadstat (>=1.1.2)"] -sql-other = ["SQLAlchemy (>=1.4.16)"] -test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.6.3)"] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pandas" +version = "3.0.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.11" +groups = ["main"] +markers = "python_version < \"3.14\" and python_version >= \"3.12\"" +files = [ + {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, + {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, + {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, + {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, + {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, + {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, + {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, + {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, + {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, + {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, + {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, + {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, + {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, + {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, + {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, + {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, + {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, + {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, + {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, + {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, + {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, + {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, + {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, + {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, + {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, + {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, + {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, + {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, + {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, + {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, + {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, + {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, + {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, + {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, + {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, + {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, + {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, + {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, + {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, + {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, + {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, + {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, + {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, + {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, + {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, + {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, + {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, + {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} +python-dateutil = ">=2.8.2" +tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] +aws = ["s3fs (>=2024.10.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] +compression = ["zstandard (>=0.23.0)"] +computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] +feather = ["pyarrow (>=13.0.0)"] +fss = ["fsspec (>=2024.10.0)"] +gcp = ["gcsfs (>=2024.10.0)"] +hdf5 = ["tables (>=3.10.1)"] +html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] +iceberg = ["pyiceberg (>=0.8.1)"] +mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] +output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=13.0.0)"] +performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] +plot = ["matplotlib (>=3.9.3)"] +postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] +pyarrow = ["pyarrow (>=13.0.0)"] +spss = ["pyreadstat (>=1.2.8)"] +sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +timezone = ["pytz (>=2020.1)"] +xml = ["lxml (>=5.3.0)"] [[package]] name = "portalocker" @@ -1344,6 +1994,7 @@ version = "3.0.0" description = "Wraps the portalocker recipe for easy usage" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "portalocker-3.0.0-py3-none-any.whl", hash = "sha256:211916b539a0dc3c128a3d9e86893ecfefec5379c4ff684e798f0a00f99db406"}, {file = "portalocker-3.0.0.tar.gz", hash = "sha256:21f535de2e7a82c94c130c054adb5c7421d480d5619d61073996e2f89bcb879b"}, @@ -1363,6 +2014,7 @@ version = "0.2.0" description = "Accelerated property cache" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, @@ -1470,6 +2122,7 @@ version = "4.25.5" description = "" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"}, {file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"}, @@ -1484,12 +2137,28 @@ files = [ {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"}, ] +[[package]] +name = "pygments" +version = "2.20.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, + {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -1504,6 +2173,7 @@ version = "2.4.0" description = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pytorch-lightning-2.4.0.tar.gz", hash = "sha256:6aa897fd9d6dfa7b7b49f37c2f04e13592861831d08deae584dfda423fdb71c8"}, {file = "pytorch_lightning-2.4.0-py3-none-any.whl", hash = "sha256:9ac7935229ac022ef06994c928217ed37f525ac6700f7d4fc57009624570e655"}, @@ -1520,12 +2190,12 @@ tqdm = ">=4.57.0" typing-extensions = ">=4.4.0" [package.extras] -all = ["bitsandbytes (>=0.42.0)", "deepspeed (>=0.8.2,<=0.9.3)", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "requests (<2.32.0)", "rich (>=12.3.0)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)"] -deepspeed = ["deepspeed (>=0.8.2,<=0.9.3)"] -dev = ["bitsandbytes (>=0.42.0)", "cloudpickle (>=1.3)", "coverage (==7.3.1)", "deepspeed (>=0.8.2,<=0.9.3)", "fastapi", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "numpy (>=1.17.2)", "omegaconf (>=2.2.3)", "onnx (>=1.12.0)", "onnxruntime (>=1.12.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "requests (<2.32.0)", "rich (>=12.3.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)", "uvicorn"] +all = ["bitsandbytes (>=0.42.0)", "deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\"", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "requests (<2.32.0)", "rich (>=12.3.0)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)"] +deepspeed = ["deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\""] +dev = ["bitsandbytes (>=0.42.0)", "cloudpickle (>=1.3)", "coverage (==7.3.1)", "deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\"", "fastapi", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "numpy (>=1.17.2)", "omegaconf (>=2.2.3)", "onnx (>=1.12.0)", "onnxruntime (>=1.12.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "requests (<2.32.0)", "rich (>=12.3.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)", "uvicorn"] examples = ["ipython[all] (<8.15.0)", "lightning-utilities (>=0.8.0)", "requests (<2.32.0)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)"] extra = ["bitsandbytes (>=0.42.0)", "hydra-core (>=1.2.0)", "jsonargparse[signatures] (>=4.27.7)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "rich (>=12.3.0)", "tensorboardX (>=2.2)"] -strategies = ["deepspeed (>=0.8.2,<=0.9.3)"] +strategies = ["deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\""] test = ["cloudpickle (>=1.3)", "coverage (==7.3.1)", "fastapi", "numpy (>=1.17.2)", "onnx (>=1.12.0)", "onnxruntime (>=1.12.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "uvicorn"] [[package]] @@ -1534,6 +2204,8 @@ version = "2024.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["main"] +markers = "python_version <= \"3.11\" or python_version >= \"3.14\"" files = [ {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, @@ -1545,6 +2217,8 @@ version = "308" description = "Python for Window Extensions" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Windows\"" files = [ {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, @@ -1572,6 +2246,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1634,6 +2309,7 @@ version = "2024.11.6" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"}, {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"}, @@ -1732,25 +2408,23 @@ files = [ ] [[package]] -name = "requests" -version = "2.32.3" -description = "Python HTTP for Humans." +name = "rich" +version = "15.0.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9.0" +groups = ["main"] files = [ - {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, - {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, + {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, + {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, ] [package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "sacrebleu" @@ -1758,6 +2432,7 @@ version = "2.4.3" description = "Hassle-free computation of shareable, comparable, and reproducible BLEU, chrF, and TER scores" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "sacrebleu-2.4.3-py3-none-any.whl", hash = "sha256:a976fd6998d8ced267a722120ec7fc47083c8e9745d8808ccee6424464a0aa31"}, {file = "sacrebleu-2.4.3.tar.gz", hash = "sha256:e734b1e0baeaea6ade0fefc9d23bac3df50bf15775d8b78edc108db63654192a"}, @@ -1782,6 +2457,7 @@ version = "0.4.5" description = "" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "safetensors-0.4.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a63eaccd22243c67e4f2b1c3e258b257effc4acd78f3b9d397edc8cf8f1298a7"}, {file = "safetensors-0.4.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:23fc9b4ec7b602915cbb4ec1a7c1ad96d2743c322f20ab709e2c35d1b66dad27"}, @@ -1914,6 +2590,7 @@ version = "1.3.2" description = "A set of python modules for machine learning and data mining" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "scikit-learn-1.3.2.tar.gz", hash = "sha256:a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05"}, {file = "scikit_learn-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e326c0eb5cf4d6ba40f93776a20e9a7a69524c4db0757e7ce24ba222471ee8a1"}, @@ -1957,41 +2634,67 @@ tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc ( [[package]] name = "scipy" -version = "1.9.3" +version = "1.15.3" description = "Fundamental algorithms for scientific computing in Python" optional = false -python-versions = ">=3.8" -files = [ - {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"}, - {file = "scipy-1.9.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd"}, - {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b"}, - {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9"}, - {file = "scipy-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523"}, - {file = "scipy-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096"}, - {file = "scipy-1.9.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c"}, - {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab"}, - {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb"}, - {file = "scipy-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31"}, - {file = "scipy-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840"}, - {file = "scipy-1.9.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5"}, - {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108"}, - {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc"}, - {file = "scipy-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e"}, - {file = "scipy-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c"}, - {file = "scipy-1.9.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95"}, - {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e"}, - {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0"}, - {file = "scipy-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58"}, - {file = "scipy-1.9.3.tar.gz", hash = "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"}, +python-versions = ">=3.10" +groups = ["main", "dev"] +files = [ + {file = "scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92"}, + {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82"}, + {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40"}, + {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e"}, + {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c"}, + {file = "scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1"}, + {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889"}, + {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982"}, + {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9"}, + {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594"}, + {file = "scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c"}, + {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45"}, + {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49"}, + {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e"}, + {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539"}, + {file = "scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730"}, + {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825"}, + {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7"}, + {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11"}, + {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126"}, + {file = "scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb"}, + {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723"}, + {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb"}, + {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4"}, + {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5"}, + {file = "scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca"}, + {file = "scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf"}, ] [package.dependencies] -numpy = ">=1.18.5,<1.26.0" +numpy = ">=1.23.5,<2.5" [package.extras] -dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] -doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] -test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] +doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "sentencepiece" @@ -1999,6 +2702,7 @@ version = "0.2.0" description = "SentencePiece python wrapper" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:188779e1298a1c8b8253c7d3ad729cb0a9891e5cef5e5d07ce4592c54869e227"}, {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bed9cf85b296fa2b76fc2547b9cbb691a523864cebaee86304c43a7b4cb1b452"}, @@ -2061,19 +2765,32 @@ version = "75.6.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "setuptools-75.6.0-py3-none-any.whl", hash = "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d"}, {file = "setuptools-75.6.0.tar.gz", hash = "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.7.0)"] -core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.7.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (>=1.12,<1.14)", "pytest-mypy"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] [[package]] name = "six" @@ -2081,6 +2798,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -2092,6 +2810,7 @@ version = "0.0.15" description = "A Sphinx extension for rendering tables written in markdown" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "sphinx-markdown-tables-0.0.15.tar.gz", hash = "sha256:722b70a3a14156b6777cf04920d015215eb65b9dfde5cc72f430763ff5e84173"}, {file = "sphinx_markdown_tables-0.0.15-py3-none-any.whl", hash = "sha256:24a37662d86ee8bceb7d4f7003df0b25bc52401369d1ddc40d13ae7b58697031"}, @@ -2102,27 +2821,14 @@ markdown = ">=3.0.1" [[package]] name = "sympy" -version = "1.12.1" +version = "1.14.0" description = "Computer algebra system (CAS) in Python" optional = false -python-versions = ">=3.8" -files = [ - {file = "sympy-1.12.1-py3-none-any.whl", hash = "sha256:9b2cbc7f1a640289430e13d2a56f02f867a1da0190f2f99d8968c2f74da0e515"}, - {file = "sympy-1.12.1.tar.gz", hash = "sha256:2877b03f998cd8c08f07cd0de5b767119cd3ef40d09f41c30d722f6686b0fb88"}, -] - -[package.dependencies] -mpmath = ">=1.1.0,<1.4.0" - -[[package]] -name = "sympy" -version = "1.13.1" -description = "Computer algebra system (CAS) in Python" -optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8"}, - {file = "sympy-1.13.1.tar.gz", hash = "sha256:9cebf7e04ff162015ce31c9c6c9144daa34a93bd082f54fd8f12deca4f47515f"}, + {file = "sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5"}, + {file = "sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517"}, ] [package.dependencies] @@ -2137,6 +2843,7 @@ version = "0.9.0" description = "Pretty-print tabular data" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, @@ -2151,6 +2858,7 @@ version = "3.5.0" description = "threadpoolctl" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, @@ -2158,157 +2866,79 @@ files = [ [[package]] name = "tokenizers" -version = "0.20.3" +version = "0.22.2" description = "" optional = false -python-versions = ">=3.7" -files = [ - {file = "tokenizers-0.20.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:31ccab28dbb1a9fe539787210b0026e22debeab1662970f61c2d921f7557f7e4"}, - {file = "tokenizers-0.20.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6361191f762bda98c773da418cf511cbaa0cb8d0a1196f16f8c0119bde68ff8"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f128d5da1202b78fa0a10d8d938610472487da01b57098d48f7e944384362514"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79c4121a2e9433ad7ef0769b9ca1f7dd7fa4c0cd501763d0a030afcbc6384481"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7850fde24197fe5cd6556e2fdba53a6d3bae67c531ea33a3d7c420b90904141"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b357970c095dc134978a68c67d845a1e3803ab7c4fbb39195bde914e7e13cf8b"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a333d878c4970b72d6c07848b90c05f6b045cf9273fc2bc04a27211721ad6118"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd9fee817f655a8f50049f685e224828abfadd436b8ff67979fc1d054b435f1"}, - {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9e7816808b402129393a435ea2a509679b41246175d6e5e9f25b8692bfaa272b"}, - {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba96367db9d8a730d3a1d5996b4b7babb846c3994b8ef14008cd8660f55db59d"}, - {file = "tokenizers-0.20.3-cp310-none-win32.whl", hash = "sha256:ee31ba9d7df6a98619426283e80c6359f167e2e9882d9ce1b0254937dbd32f3f"}, - {file = "tokenizers-0.20.3-cp310-none-win_amd64.whl", hash = "sha256:a845c08fdad554fe0871d1255df85772f91236e5fd6b9287ef8b64f5807dbd0c"}, - {file = "tokenizers-0.20.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:585b51e06ca1f4839ce7759941e66766d7b060dccfdc57c4ca1e5b9a33013a90"}, - {file = "tokenizers-0.20.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61cbf11954f3b481d08723ebd048ba4b11e582986f9be74d2c3bdd9293a4538d"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef820880d5e4e8484e2fa54ff8d297bb32519eaa7815694dc835ace9130a3eea"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:67ef4dcb8841a4988cd00dd288fb95dfc8e22ed021f01f37348fd51c2b055ba9"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff1ef8bd47a02b0dc191688ccb4da53600df5d4c9a05a4b68e1e3de4823e78eb"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:444d188186eab3148baf0615b522461b41b1f0cd58cd57b862ec94b6ac9780f1"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37c04c032c1442740b2c2d925f1857885c07619224a533123ac7ea71ca5713da"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453c7769d22231960ee0e883d1005c93c68015025a5e4ae56275406d94a3c907"}, - {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4bb31f7b2847e439766aaa9cc7bccf7ac7088052deccdb2275c952d96f691c6a"}, - {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:843729bf0f991b29655a069a2ff58a4c24375a553c70955e15e37a90dd4e045c"}, - {file = "tokenizers-0.20.3-cp311-none-win32.whl", hash = "sha256:efcce3a927b1e20ca694ba13f7a68c59b0bd859ef71e441db68ee42cf20c2442"}, - {file = "tokenizers-0.20.3-cp311-none-win_amd64.whl", hash = "sha256:88301aa0801f225725b6df5dea3d77c80365ff2362ca7e252583f2b4809c4cc0"}, - {file = "tokenizers-0.20.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:49d12a32e190fad0e79e5bdb788d05da2f20d8e006b13a70859ac47fecf6ab2f"}, - {file = "tokenizers-0.20.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:282848cacfb9c06d5e51489f38ec5aa0b3cd1e247a023061945f71f41d949d73"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abe4e08c7d0cd6154c795deb5bf81d2122f36daf075e0c12a8b050d824ef0a64"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca94fc1b73b3883c98f0c88c77700b13d55b49f1071dfd57df2b06f3ff7afd64"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef279c7e239f95c8bdd6ff319d9870f30f0d24915b04895f55b1adcf96d6c60d"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16384073973f6ccbde9852157a4fdfe632bb65208139c9d0c0bd0176a71fd67f"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:312d522caeb8a1a42ebdec87118d99b22667782b67898a76c963c058a7e41d4f"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b7cb962564785a83dafbba0144ecb7f579f1d57d8c406cdaa7f32fe32f18ad"}, - {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:124c5882ebb88dadae1fc788a582299fcd3a8bd84fc3e260b9918cf28b8751f5"}, - {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2b6e54e71f84c4202111a489879005cb14b92616a87417f6c102c833af961ea2"}, - {file = "tokenizers-0.20.3-cp312-none-win32.whl", hash = "sha256:83d9bfbe9af86f2d9df4833c22e94d94750f1d0cd9bfb22a7bb90a86f61cdb1c"}, - {file = "tokenizers-0.20.3-cp312-none-win_amd64.whl", hash = "sha256:44def74cee574d609a36e17c8914311d1b5dbcfe37c55fd29369d42591b91cf2"}, - {file = "tokenizers-0.20.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e0b630e0b536ef0e3c8b42c685c1bc93bd19e98c0f1543db52911f8ede42cf84"}, - {file = "tokenizers-0.20.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a02d160d2b19bcbfdf28bd9a4bf11be4cb97d0499c000d95d4c4b1a4312740b6"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e3d80d89b068bc30034034b5319218c7c0a91b00af19679833f55f3becb6945"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:174a54910bed1b089226512b4458ea60d6d6fd93060254734d3bc3540953c51c"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:098b8a632b8656aa5802c46689462c5c48f02510f24029d71c208ec2c822e771"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78c8c143e3ae41e718588281eb3e212c2b31623c9d6d40410ec464d7d6221fb5"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b26b0aadb18cd8701077362ba359a06683662d5cafe3e8e8aba10eb05c037f1"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07d7851a72717321022f3774e84aa9d595a041d643fafa2e87fbc9b18711dac0"}, - {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:bd44e48a430ada902c6266a8245f5036c4fe744fcb51f699999fbe82aa438797"}, - {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a4c186bb006ccbe1f5cc4e0380d1ce7806f5955c244074fd96abc55e27b77f01"}, - {file = "tokenizers-0.20.3-cp313-none-win32.whl", hash = "sha256:6e19e0f1d854d6ab7ea0c743d06e764d1d9a546932be0a67f33087645f00fe13"}, - {file = "tokenizers-0.20.3-cp313-none-win_amd64.whl", hash = "sha256:d50ede425c7e60966a9680d41b58b3a0950afa1bb570488e2972fa61662c4273"}, - {file = "tokenizers-0.20.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:9adda1ff5fb9dcdf899ceca672a4e2ce9e797adb512a6467305ca3d8bfcfbdd0"}, - {file = "tokenizers-0.20.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:6dde2cae6004ba7a3badff4a11911cae03ebf23e97eebfc0e71fef2530e5074f"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4a7fd678b35614fca708579eb95b7587a5e8a6d328171bd2488fd9f27d82be4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b80e3c7283a01a356bd2210f53d1a4a5d32b269c2024389ed0173137708d50e"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8cc0e8176b762973758a77f0d9c4467d310e33165fb74173418ca3734944da4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5634b2e2f5f3d2b4439d2d74066e22eb4b1f04f3fea05cb2a3c12d89b5a3bcd"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4ba635165bc1ea46f2da8e5d80b5f70f6ec42161e38d96dbef33bb39df73964"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e4c7c64172e7789bd8b07aa3087ea87c4c4de7e90937a2aa036b5d92332536"}, - {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1f74909ef7675c26d4095a817ec3393d67f3158ca4836c233212e5613ef640c4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0e9b81321a1e05b16487d312b4264984513f8b4a7556229cafac6e88c2036b09"}, - {file = "tokenizers-0.20.3-cp37-none-win32.whl", hash = "sha256:ab48184cd58b4a03022a2ec75b54c9f600ffea9a733612c02325ed636f353729"}, - {file = "tokenizers-0.20.3-cp37-none-win_amd64.whl", hash = "sha256:60ac483cebee1c12c71878523e768df02fa17e4c54412966cb3ac862c91b36c1"}, - {file = "tokenizers-0.20.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3229ef103c89583d10b9378afa5d601b91e6337530a0988e17ca8d635329a996"}, - {file = "tokenizers-0.20.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6ac52cc24bad3de865c7e65b1c4e7b70d00938a8ae09a92a453b8f676e714ad5"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04627b7b502fa6a2a005e1bd446fa4247d89abcb1afaa1b81eb90e21aba9a60f"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c27ceb887f0e81a3c377eb4605dca7a95a81262761c0fba308d627b2abb98f2b"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65ab780194da4e1fcf5670523a2f377c4838ebf5249efe41fa1eddd2a84fb49d"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98d343134f47159e81f7f242264b0eb222e6b802f37173c8d7d7b64d5c9d1388"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2475bb004ab2009d29aff13b5047bfdb3d4b474f0aa9d4faa13a7f34dbbbb43"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b6583a65c01db1197c1eb36857ceba8ec329d53afadd268b42a6b04f4965724"}, - {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:62d00ba208358c037eeab7bfc00a905adc67b2d31b68ab40ed09d75881e114ea"}, - {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0fc7a39e5bedc817bda395a798dfe2d9c5f7c71153c90d381b5135a0328d9520"}, - {file = "tokenizers-0.20.3-cp38-none-win32.whl", hash = "sha256:84d40ee0f8550d64d3ea92dd7d24a8557a9172165bdb986c9fb2503b4fe4e3b6"}, - {file = "tokenizers-0.20.3-cp38-none-win_amd64.whl", hash = "sha256:205a45246ed7f1718cf3785cff88450ba603352412aaf220ace026384aa3f1c0"}, - {file = "tokenizers-0.20.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:93e37f0269a11dc3b1a953f1fca9707f0929ebf8b4063c591c71a0664219988e"}, - {file = "tokenizers-0.20.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f4cb0c614b0135e781de96c2af87e73da0389ac1458e2a97562ed26e29490d8d"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7eb2fb1c432f5746b22f8a7f09fc18c4156cb0031c77f53cb19379d82d43297a"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfa8d029bb156181b006643309d6b673615a24e4ed24cf03aa191d599b996f51"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f90549622de3bf476ad9f1dd6f3f952ec3ed6ab8615ae88ef060d0c5bfad55d"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1d469c74eebf5c43fd61cd9b030e271d17198edd7bd45392e03a3c091d7d6d4"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bee8f53b2594749f4460d53253bae55d718f04e9b633efa0f5df8938bd98e4f0"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:938441babf3e5720e4459e306ef2809fb267680df9d1ff2873458b22aef60248"}, - {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7310ab23d7b0caebecc0e8be11a1146f320f5f07284000f6ea54793e83de1b75"}, - {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:16121eb030a2b13094cfec936b0c12e8b4063c5f839591ea7d0212336d8f9921"}, - {file = "tokenizers-0.20.3-cp39-none-win32.whl", hash = "sha256:401cc21ef642ee235985d747f65e18f639464d377c70836c9003df208d582064"}, - {file = "tokenizers-0.20.3-cp39-none-win_amd64.whl", hash = "sha256:7498f3ea7746133335a6adb67a77cf77227a8b82c8483f644a2e5f86fea42b8d"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e919f2e3e68bb51dc31de4fcbbeff3bdf9c1cad489044c75e2b982a91059bd3c"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b8e9608f2773996cc272156e305bd79066163a66b0390fe21750aff62df1ac07"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39270a7050deaf50f7caff4c532c01b3c48f6608d42b3eacdebdc6795478c8df"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e005466632b1c5d2d2120f6de8aa768cc9d36cd1ab7d51d0c27a114c91a1e6ee"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a07962340b36189b6c8feda552ea1bfeee6cf067ff922a1d7760662c2ee229e5"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:55046ad3dd5f2b3c67501fcc8c9cbe3e901d8355f08a3b745e9b57894855f85b"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:efcf0eb939988b627558aaf2b9dc3e56d759cad2e0cfa04fcab378e4b48fc4fd"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f3558a7ae6a6d38a77dfce12172a1e2e1bf3e8871e744a1861cd7591ea9ebe24"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d53029fe44bc70c3ff14ef512460a0cf583495a0f8e2f4b70e26eb9438e38a9"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a2a56397b2bec5a629b516b23f0f8a3e4f978c7488d4a299980f8375954b85"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e5bfaae740ef9ece000f8a07e78ac0e2b085c5ce9648f8593ddf0243c9f76d"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fbaf3ea28fedfb2283da60e710aff25492e795a7397cad8a50f1e079b65a5a70"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c47c037116310dc976eb96b008e41b9cfaba002ed8005848d4d632ee0b7ba9ae"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c31751f0721f58f5e19bb27c1acc259aeff860d8629c4e1a900b26a1979ada8e"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:c697cbd3be7a79ea250ea5f380d6f12e534c543cfb137d5c734966b3ee4f34cc"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48971b88ef9130bf35b41b35fd857c3c4dae4a9cd7990ebc7fc03e59cc92438"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e615de179bbe060ab33773f0d98a8a8572b5883dd7dac66c1de8c056c7e748c"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da1ec842035ed9999c62e45fbe0ff14b7e8a7e02bb97688cc6313cf65e5cd755"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6ee4954c1dd23aadc27958dad759006e71659d497dcb0ef0c7c87ea992c16ebd"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3eda46ca402751ec82553a321bf35a617b76bbed7586e768c02ccacbdda94d6d"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:de082392a85eb0055cc055c535bff2f0cc15d7a000bdc36fbf601a0f3cf8507a"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c3db46cc0647bfd88263afdb739b92017a02a87ee30945cb3e86c7e25c7c9917"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a292392f24ab9abac5cfa8197e5a6208f2e43723420217e1ceba0b4ec77816ac"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dcd91f4e60f62b20d83a87a84fe062035a1e3ff49a8c2bbdeb2d441c8e311f4"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900991a2b8ee35961b1095db7e265342e0e42a84c1a594823d5ee9f8fb791958"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5a8d8261ca2133d4f98aa9627c748189502b3787537ba3d7e2beb4f7cfc5d627"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c4fd4d71e6deb6ddf99d8d0eab87d1d16f635898906e631914a9bae8ae9f2cfb"}, - {file = "tokenizers-0.20.3.tar.gz", hash = "sha256:2278b34c5d0dd78e087e1ca7f9b1dcbf129d80211afa645f214bd6e051037539"}, +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c"}, + {file = "tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5"}, + {file = "tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92"}, + {file = "tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48"}, + {file = "tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113"}, + {file = "tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917"}, ] [package.dependencies] -huggingface-hub = ">=0.16.4,<1.0" +huggingface-hub = ">=0.16.4,<2.0" [package.extras] dev = ["tokenizers[testing]"] docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] +testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] [[package]] name = "torch" -version = "2.5.1" +version = "2.7.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false -python-versions = ">=3.8.0" -files = [ - {file = "torch-2.5.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:71328e1bbe39d213b8721678f9dcac30dfc452a46d586f1d514a6aa0a99d4744"}, - {file = "torch-2.5.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:34bfa1a852e5714cbfa17f27c49d8ce35e1b7af5608c4bc6e81392c352dbc601"}, - {file = "torch-2.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:32a037bd98a241df6c93e4c789b683335da76a2ac142c0973675b715102dc5fa"}, - {file = "torch-2.5.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:23d062bf70776a3d04dbe74db950db2a5245e1ba4f27208a87f0d743b0d06e86"}, - {file = "torch-2.5.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:de5b7d6740c4b636ef4db92be922f0edc425b65ed78c5076c43c42d362a45457"}, - {file = "torch-2.5.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:340ce0432cad0d37f5a31be666896e16788f1adf8ad7be481196b503dad675b9"}, - {file = "torch-2.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:603c52d2fe06433c18b747d25f5c333f9c1d58615620578c326d66f258686f9a"}, - {file = "torch-2.5.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:31f8c39660962f9ae4eeec995e3049b5492eb7360dd4f07377658ef4d728fa4c"}, - {file = "torch-2.5.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:ed231a4b3a5952177fafb661213d690a72caaad97d5824dd4fc17ab9e15cec03"}, - {file = "torch-2.5.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:3f4b7f10a247e0dcd7ea97dc2d3bfbfc90302ed36d7f3952b0008d0df264e697"}, - {file = "torch-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:73e58e78f7d220917c5dbfad1a40e09df9929d3b95d25e57d9f8558f84c9a11c"}, - {file = "torch-2.5.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:8c712df61101964eb11910a846514011f0b6f5920c55dbf567bff8a34163d5b1"}, - {file = "torch-2.5.1-cp313-cp313-manylinux1_x86_64.whl", hash = "sha256:9b61edf3b4f6e3b0e0adda8b3960266b9009d02b37555971f4d1c8f7a05afed7"}, - {file = "torch-2.5.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1f3b7fb3cf7ab97fae52161423f81be8c6b8afac8d9760823fd623994581e1a3"}, - {file = "torch-2.5.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:7974e3dce28b5a21fb554b73e1bc9072c25dde873fa00d54280861e7a009d7dc"}, - {file = "torch-2.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:46c817d3ea33696ad3b9df5e774dba2257e9a4cd3c4a3afbf92f6bb13ac5ce2d"}, - {file = "torch-2.5.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:8046768b7f6d35b85d101b4b38cba8aa2f3cd51952bc4c06a49580f2ce682291"}, +python-versions = ">=3.9.0" +groups = ["main"] +markers = "python_version >= \"3.14\"" +files = [ + {file = "torch-2.7.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a103b5d782af5bd119b81dbcc7ffc6fa09904c423ff8db397a1e6ea8fd71508f"}, + {file = "torch-2.7.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:fe955951bdf32d182ee8ead6c3186ad54781492bf03d547d31771a01b3d6fb7d"}, + {file = "torch-2.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:885453d6fba67d9991132143bf7fa06b79b24352f4506fd4d10b309f53454162"}, + {file = "torch-2.7.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:d72acfdb86cee2a32c0ce0101606f3758f0d8bb5f8f31e7920dc2809e963aa7c"}, + {file = "torch-2.7.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:236f501f2e383f1cb861337bdf057712182f910f10aeaf509065d54d339e49b2"}, + {file = "torch-2.7.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:06eea61f859436622e78dd0cdd51dbc8f8c6d76917a9cf0555a333f9eac31ec1"}, + {file = "torch-2.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:8273145a2e0a3c6f9fd2ac36762d6ee89c26d430e612b95a99885df083b04e52"}, + {file = "torch-2.7.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:aea4fc1bf433d12843eb2c6b2204861f43d8364597697074c8d38ae2507f8730"}, + {file = "torch-2.7.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:27ea1e518df4c9de73af7e8a720770f3628e7f667280bce2be7a16292697e3fa"}, + {file = "torch-2.7.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c33360cfc2edd976c2633b3b66c769bdcbbf0e0b6550606d188431c81e7dd1fc"}, + {file = "torch-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:d8bf6e1856ddd1807e79dc57e54d3335f2b62e6f316ed13ed3ecfe1fc1df3d8b"}, + {file = "torch-2.7.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:787687087412c4bd68d315e39bc1223f08aae1d16a9e9771d95eabbb04ae98fb"}, + {file = "torch-2.7.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:03563603d931e70722dce0e11999d53aa80a375a3d78e6b39b9f6805ea0a8d28"}, + {file = "torch-2.7.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d632f5417b6980f61404a125b999ca6ebd0b8b4bbdbb5fbbba44374ab619a412"}, + {file = "torch-2.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:23660443e13995ee93e3d844786701ea4ca69f337027b05182f5ba053ce43b38"}, + {file = "torch-2.7.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:0da4f4dba9f65d0d203794e619fe7ca3247a55ffdcbd17ae8fb83c8b2dc9b585"}, + {file = "torch-2.7.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e08d7e6f21a617fe38eeb46dd2213ded43f27c072e9165dc27300c9ef9570934"}, + {file = "torch-2.7.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:30207f672328a42df4f2174b8f426f354b2baa0b7cca3a0adb3d6ab5daf00dc8"}, + {file = "torch-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:79042feca1c634aaf6603fe6feea8c6b30dfa140a6bbc0b973e2260c7e79a22e"}, + {file = "torch-2.7.1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:988b0cbc4333618a1056d2ebad9eb10089637b659eb645434d0809d8d937b946"}, + {file = "torch-2.7.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:e0d81e9a12764b6f3879a866607c8ae93113cbcad57ce01ebde63eb48a576369"}, + {file = "torch-2.7.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:8394833c44484547ed4a47162318337b88c97acdb3273d85ea06e03ffff44998"}, + {file = "torch-2.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:df41989d9300e6e3c19ec9f56f856187a6ef060c3662fe54f4b6baf1fc90bd19"}, + {file = "torch-2.7.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:a737b5edd1c44a5c1ece2e9f3d00df9d1b3fb9541138bee56d83d38293fb6c9d"}, ] [package.dependencies] @@ -2316,29 +2946,89 @@ filelock = "*" fsspec = "*" jinja2 = "*" networkx = "*" -nvidia-cublas-cu12 = {version = "12.4.5.8", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-cupti-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-nvrtc-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-runtime-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cudnn-cu12 = {version = "9.1.0.70", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cufft-cu12 = {version = "11.2.1.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-curand-cu12 = {version = "10.3.5.147", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusolver-cu12 = {version = "11.6.1.9", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusparse-cu12 = {version = "12.3.1.170", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nccl-cu12 = {version = "2.21.5", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nvjitlink-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nvtx-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cublas-cu12 = {version = "12.6.4.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu12 = {version = "12.6.80", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu12 = {version = "12.6.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu12 = {version = "12.6.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu12 = {version = "9.5.1.17", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu12 = {version = "11.3.0.4", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufile-cu12 = {version = "1.11.1.6", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu12 = {version = "10.3.7.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu12 = {version = "11.7.1.2", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu12 = {version = "12.5.4.2", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparselt-cu12 = {version = "0.6.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu12 = {version = "2.26.2", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvjitlink-cu12 = {version = "12.6.85", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu12 = {version = "12.6.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} setuptools = {version = "*", markers = "python_version >= \"3.12\""} -sympy = [ - {version = "1.12.1", markers = "python_version == \"3.8\""}, - {version = "1.13.1", markers = "python_version >= \"3.9\""}, +sympy = ">=1.13.3" +triton = {version = "3.3.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +typing-extensions = ">=4.10.0" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] +optree = ["optree (>=0.13.0)"] + +[[package]] +name = "torch" +version = "2.12.0" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version < \"3.14\"" +files = [ + {file = "torch-2.12.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:1834bd984f8a2f4f16bdfbeecca9146184b220aa46276bf5756735b5dae12812"}, + {file = "torch-2.12.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d4d029801cb7b6df858804a2a21b00cc2aa0bf0ee5d2ab18d343c9e9e5681f35"}, + {file = "torch-2.12.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:d47e7dee68ac4cd7a068b26bcd6b989935427709fae1c8f7bd0019978f829e15"}, + {file = "torch-2.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:cf9839790285dd472e7a16aafcb4a4e6bf58ec1b494045044b0eefb0eb4bd1f2"}, + {file = "torch-2.12.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:10802fd383bbfed646212e765a72c37d2185205d4f26eb197a254e8ac7ddcb25"}, + {file = "torch-2.12.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c12592630aef72feaf18bd3f197ef587bbfa21131b31c38b23ab2e55fce92e36"}, + {file = "torch-2.12.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:415c1b8d0412f67551c8e89a2daca0fb3e56694af0281ba155eaa9da481f58b4"}, + {file = "torch-2.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:dd37188ea325042cb1f6cafa56822b11ada2520c04791a52629b0af25bdfbfd9"}, + {file = "torch-2.12.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b41339df93d491435e790ff8bcbae1c0ce777175889bfd1281d119862793e6a2"}, + {file = "torch-2.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8fbef9f108a863e7722a73740998967e3b074742a834fc5be3a535a2befa7057"}, + {file = "torch-2.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4b4f64c2c2b11f7510d93dd6412b87025ff6eddd6bb61c3b5a3d892ea20c4756"}, + {file = "torch-2.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:8b958caff4a14d3a3b0b2dfc6a378f64dda9728a9dad28c08a0db9ce4dafb549"}, + {file = "torch-2.12.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:90dd587a5f61bfe1307148b581e2084fc5bc4a06e2b90a20e9a36b81087ff16b"}, + {file = "torch-2.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:864392c73b7654f4d2b3ae712f607937d0dbb1101c4555fbb41848106b297f39"}, + {file = "torch-2.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5d6b560dfa7d56291c07d615c3bb73e8d9943d9b6d87f76cd0d9d570c4797fa6"}, + {file = "torch-2.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:3fee918902090ade827643e758e98363278815de583c75d111fdd665ebffde9f"}, + {file = "torch-2.12.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:10ee1448a9f304d3b987eb4656f664ba6e4d7b410ca7a5a7c642199777a2cf88"}, + {file = "torch-2.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af68dbf403439cae9ceaeaaf92f8352b460787dcd27b92aa05c40dd4a19c0f1e"}, + {file = "torch-2.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a6a2eebb237d3b1d9ad3b378e86d9b9e0782afdea8b1e0eba6a13646b9b49c07"}, + {file = "torch-2.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2140e373e9a51a3e22ef62e8d14366d0b470d18f0adf19fdc757368077133a34"}, + {file = "torch-2.12.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f7dfae4a519197dfa050e98d8e36378a0fb5899625a875c2b54445005a2e404e"}, + {file = "torch-2.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:891c769072637c74e9a5a77a3bc782894696d8ffec83b938df8536dee7f0ba78"}, + {file = "torch-2.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e2ad3eb85d39c3cab62dfa93ed5a73516e6a53c6713cb97d004004fe089f0f1f"}, + {file = "torch-2.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:c66696857e987efb8bc1777a37357ec4f60ab5e8af6250b83d6034437fa2d8f3"}, + {file = "torch-2.12.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:b4556715c8572758625d62b6e0ae3b1f76c440221913a6fb5e100f321fb4fb02"}, + {file = "torch-2.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a43ac605a5e13116c72b64c359644cce0229f213dde48d2ae0ae5eb5becf7feb"}, + {file = "torch-2.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a7512adfdd7f6732e40de1c620831e3c75b39b98cef60b11d0c5f0a76473ec5"}, + {file = "torch-2.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f96b63f8287f66a005dd1b5a6abba2920f11156c5e5c4d815f3e2050fd1aa16"}, ] -triton = {version = "3.1.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version < \"3.13\""} -typing-extensions = ">=4.8.0" + +[package.dependencies] +cuda-bindings = {version = ">=13.0.3,<14", markers = "platform_system == \"Linux\""} +cuda-toolkit = {version = "13.0.2", extras = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], markers = "platform_system == \"Linux\""} +filelock = "*" +fsspec = ">=0.8.5" +jinja2 = "*" +networkx = ">=2.5.1" +nvidia-cublas = {version = ">=13.1.0.3,<=13.1.1.3", markers = "platform_system == \"Linux\""} +nvidia-cudnn-cu13 = {version = "9.20.0.48", markers = "platform_system == \"Linux\""} +nvidia-cusparselt-cu13 = {version = "0.8.1", markers = "platform_system == \"Linux\""} +nvidia-nccl-cu13 = {version = "2.29.7", markers = "platform_system == \"Linux\""} +nvidia-nvshmem-cu13 = {version = "3.4.5", markers = "platform_system == \"Linux\""} +setuptools = "<82" +sympy = ">=1.13.3" +triton = {version = "3.7.0", markers = "platform_system == \"Linux\""} +typing-extensions = ">=4.10.0" [package.extras] opt-einsum = ["opt-einsum (>=3.3)"] -optree = ["optree (>=0.12.0)"] +optree = ["optree (>=0.13.0)"] +pyyaml = ["pyyaml"] [[package]] name = "torchmetrics" @@ -2346,6 +3036,7 @@ version = "0.10.3" description = "PyTorch native Metrics" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "torchmetrics-0.10.3-py3-none-any.whl", hash = "sha256:b12cf92897545e24a825b0d168888c0f3052700c2901e2d4f7d90b252bc4a343"}, {file = "torchmetrics-0.10.3.tar.gz", hash = "sha256:9e6ab66175f2dc13e246c37485b2c27c77931dfe47fc2b81c76217b8efdc1e57"}, @@ -2355,7 +3046,6 @@ files = [ numpy = ">=1.17.2" packaging = "*" torch = ">=1.3.1" -typing-extensions = {version = "*", markers = "python_version < \"3.9\""} [package.extras] all = ["lpips", "nltk (>=3.6)", "pycocotools", "pystoi", "pytorch-lightning (>=1.5)", "regex (>=2021.9.24)", "scipy", "torch-fidelity", "torchvision", "torchvision (>=0.8)", "tqdm (>=4.41.0)"] @@ -2373,6 +3063,7 @@ version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, @@ -2390,101 +3081,138 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.46.3" -description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +version = "5.3.0" +description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false -python-versions = ">=3.8.0" +python-versions = ">=3.10.0" +groups = ["main"] files = [ - {file = "transformers-4.46.3-py3-none-any.whl", hash = "sha256:a12ef6f52841fd190a3e5602145b542d03507222f2c64ebb7ee92e8788093aef"}, - {file = "transformers-4.46.3.tar.gz", hash = "sha256:8ee4b3ae943fe33e82afff8e837f4b052058b07ca9be3cb5b729ed31295f72cc"}, + {file = "transformers-5.3.0-py3-none-any.whl", hash = "sha256:50ac8c89c3c7033444fb3f9f53138096b997ebb70d4b5e50a2e810bf12d3d29a"}, + {file = "transformers-5.3.0.tar.gz", hash = "sha256:009555b364029da9e2946d41f1c5de9f15e6b1df46b189b7293f33a161b9c557"}, ] [package.dependencies] -filelock = "*" -huggingface-hub = ">=0.23.2,<1.0" +huggingface-hub = ">=1.3.0,<2.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" -requests = "*" -safetensors = ">=0.4.1" -tokenizers = ">=0.20,<0.21" +safetensors = ">=0.4.3" +tokenizers = ">=0.22.0,<=0.23.0" tqdm = ">=4.27" +typer = "*" [package.extras] -accelerate = ["accelerate (>=0.26.0)"] -agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision"] -audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +accelerate = ["accelerate (>=1.1.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.10.2,<0.11)", "librosa", "mistral-common[image] (>=1.8.8)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] -codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.26.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.26.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk (<=3.8.1)", "optuna", "parameterized", "protobuf", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "librosa", "nltk (<=3.8.1)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "librosa", "nltk (<=3.8.1)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.20,<0.21)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "libcst", "librosa", "nltk (<=3.8.1)", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"] -flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -ftfy = ["ftfy"] -integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.6,<0.15.0)"] -onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] -onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +chat-template = ["jinja2 (>=3.1.0)", "jmespath (>=1.0.1)"] +codecarbon = ["codecarbon (>=2.8.1)"] +deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "libcst", "mistral-common[image] (>=1.8.8)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "torch (>=2.4)", "ty (==0.0.12)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.10.2,<0.11)", "libcst", "librosa", "mistral-common[image] (>=1.8.8)", "mistral-common[image] (>=1.8.8)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "ty (==0.0.12)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.10.2,<0.11)", "optuna", "ray[tune] (>=2.7.0)"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] +kernels = ["kernels (>=0.10.2,<0.11)"] +mistral-common = ["mistral-common[image] (>=1.8.8)"] +num2words = ["num2words"] +open-telemetry = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "isort (>=5.5.4)", "libcst", "rich", "ruff (==0.5.1)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "ty (==0.0.12)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] -retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] -ruff = ["ruff (==0.5.1)"] +retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic", "starlette", "uvicorn"] -sigopt = ["sigopt"] +serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk (<=3.8.1)", "parameterized", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-cpu = ["keras (>2.9,<2.16)", "keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow-cpu (>2.9,<2.16)", "tensorflow-probability (<0.24)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "libcst", "mistral-common[image] (>=1.8.8)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "torch (>=2.4)", "ty (==0.0.12)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] -timm = ["timm (<=0.9.16)"] -tokenizers = ["tokenizers (>=0.20,<0.21)"] -torch = ["accelerate (>=0.26.0)", "torch"] -torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.23.2,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.20,<0.21)", "torch", "tqdm (>=4.27)"] -video = ["av (==9.2.0)"] -vision = ["Pillow (>=10.0.1,<=15.0)"] +timm = ["timm (>=1.0.23)"] +torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] +video = ["av"] +vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] [[package]] name = "triton" -version = "3.1.0" +version = "3.3.1" description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0dd10a925263abbe9fa37dcde67a5e9b2383fc269fdf59f5657cac38c5d1d8"}, - {file = "triton-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f34f6e7885d1bf0eaaf7ba875a5f0ce6f3c13ba98f9503651c1e6dc6757ed5c"}, - {file = "triton-3.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8182f42fd8080a7d39d666814fa36c5e30cc00ea7eeeb1a2983dbb4c99a0fdc"}, - {file = "triton-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dadaca7fc24de34e180271b5cf864c16755702e9f63a16f62df714a8099126a"}, - {file = "triton-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aafa9a20cd0d9fee523cd4504aa7131807a864cd77dcf6efe7e981f18b8c6c11"}, + {file = "triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e"}, + {file = "triton-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b"}, + {file = "triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43"}, + {file = "triton-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b89d846b5a4198317fec27a5d3a609ea96b6d557ff44b56c23176546023c4240"}, + {file = "triton-3.3.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3198adb9d78b77818a5388bff89fa72ff36f9da0bc689db2f0a651a67ce6a42"}, + {file = "triton-3.3.1-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6139aeb04a146b0b8e0fbbd89ad1e65861c57cfed881f21d62d3cb94a36bab7"}, ] [package.dependencies] -filelock = "*" +setuptools = ">=40.8.0" [package.extras] build = ["cmake (>=3.20)", "lit"] -tests = ["autopep8", "flake8", "isort", "llnl-hatchet", "numpy", "pytest", "scipy (>=1.7.1)"] +tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] tutorials = ["matplotlib", "pandas", "tabulate"] +[[package]] +name = "triton" +version = "3.7.0" +description = "A language and compiler for custom Deep Learning operations" +optional = false +python-versions = "<3.15,>=3.10" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "triton-3.7.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223ac302091491436c248a34ee1e6c47a1026486579103c906ffd805be50cb89"}, + {file = "triton-3.7.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c631b65668d4951213b948a413c0564184305b77bb45cc9d686d3e1ecc4701a3"}, + {file = "triton-3.7.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9e71fc392675fac364e0ecf4ef3f76f85b7f5433a16f4c3c5fe5f05a52c85fe"}, + {file = "triton-3.7.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22bacffce443f54593dd20f05294d5a40622e0ea9ab632816f87154504356221"}, + {file = "triton-3.7.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4bf49b00a7a377a68a6da603a876e797614e6455a80e9021669c476a953ad9a"}, + {file = "triton-3.7.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f111161d49bf903c0eaedde3962353a3d841c08a836839b7cc1025b8426efcf"}, + {file = "triton-3.7.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abdf6beaa89b1bcfb9a43cd990536ce66091a997841a4814b260b7bee4c88c3c"}, + {file = "triton-3.7.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a35d7afe3f3f058e7ec49fcce09794049e0ffc5c59019ac25ec3413741b8c4e7"}, + {file = "triton-3.7.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc1d61c172d257db80ddf42595131fb196ad2e9bdd751e90fe2ef13531734e8b"}, + {file = "triton-3.7.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70fb9bbdc9f400afc54bbf6eb2670af28829a6ae3996863317964783141daf56"}, + {file = "triton-3.7.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a44a8476d0d3571eac4e4d1048e1ff75aad81a09ff4602ccfc56c6dea1672e"}, + {file = "triton-3.7.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b85e72968a9d8bba5ddb24e9b64aaabaf48affb042f2755cb7cfa92b7531ce"}, + {file = "triton-3.7.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18a160de426fd99f92b0baf509045360afbd3bfaa0b4a5171dde800ec9f09684"}, + {file = "triton-3.7.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce061073102714b725f3660ec6939d94a1da7984b3aa99c921417cae273672f5"}, +] + +[package.extras] +build = ["cmake (>=3.20,<4.0)", "lit"] +tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] +tutorials = ["matplotlib", "pandas", "tabulate"] + +[[package]] +name = "typer" +version = "0.25.1" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89"}, + {file = "typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc"}, +] + +[package.dependencies] +annotated-doc = ">=0.0.2" +click = ">=8.2.1" +rich = ">=13.8.0" +shellingham = ">=1.3.0" + [[package]] name = "typing-extensions" version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -2496,34 +3224,20 @@ version = "2024.2" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version <= \"3.11\" or python_version >= \"3.14\"" files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] -[[package]] -name = "urllib3" -version = "2.2.3" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.8" -files = [ - {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, - {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - [[package]] name = "yarl" version = "1.15.2" description = "Yet another URL library" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "yarl-1.15.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e4ee8b8639070ff246ad3649294336b06db37a94bdea0d09ea491603e0be73b8"}, {file = "yarl-1.15.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a7cf963a357c5f00cb55b1955df8bbe68d2f2f65de065160a1c26b85a1e44172"}, @@ -2630,26 +3344,7 @@ idna = ">=2.0" multidict = ">=4.0" propcache = ">=0.2.0" -[[package]] -name = "zipp" -version = "3.20.2" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, - {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] -type = ["pytest-mypy"] - [metadata] -lock-version = "2.0" -python-versions = "^3.8.0" -content-hash = "2aff3e907a2e72ce04fdcd5c61192068ee9131d861ca0079cacf2bf7e02b4921" +lock-version = "2.1" +python-versions = "^3.10.0" +content-hash = "8dc08aef9797ad49400e4254019ddf1403f1a8a245b2e2caa1931c321c1df30b" diff --git a/pyproject.toml b/pyproject.toml index 479ff363..bdd5f32e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,17 +36,17 @@ comet-compare = 'comet.cli.compare:compare_command' comet-mbr = 'comet.cli.mbr:mbr_command' [tool.poetry.dependencies] -python = "^3.8.0" +python = "^3.10.0" sentencepiece = ">=0.2.0" -pandas = ">=1.4.1" +pandas = ">=2.3.3" transformers = ">=4.51.1" pytorch-lightning = ">=2.0.0" -jsonargparse = "3.13.1" -torch = ">=1.6.0" +jsonargparse = ">=3.13.1" +torch = ">=2.6.0" numpy = ">=1.20.0" torchmetrics = ">=0.10.2" sacrebleu = ">=2.0.0" -scipy = ">=1.5.4" +scipy = ">=1.10.0" entmax = ">=1.1" huggingface-hub = ">=0.30.0" protobuf = ">=4.24.4" From d18f2c6893bd137e96d7348ec3081054c508b0ca Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Mon, 18 May 2026 10:41:10 +0900 Subject: [PATCH 04/13] Fixed pooling error --- README.md | 7 +------ comet/encoders/bert.py | 7 +++---- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 81b98876..33909bcf 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,6 @@ Please check all available models [here](https://github.com/Unbabel/COMET/blob/m # Quick Installation -> For this particular repo: -```bash - pip install https://github.com/daniazie/COMET.git -``` - COMET requires python 3.8 or above. Simple installation from PyPI ```bash @@ -321,4 +316,4 @@ If you use COMET please cite our work **and don't forget to say which model you - [Unbabel's Participation in the WMT20 Metrics Shared Task](https://aclanthology.org/2020.wmt-1.101/) -- [COMET: A Neural Framework for MT Evaluation](https://www.aclweb.org/anthology/2020.emnlp-main.213) +- [COMET: A Neural Framework for MT Evaluation](https://www.aclweb.org/anthology/2020.emnlp-main.213) \ No newline at end of file diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index e02e2c9f..a84bf9ae 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -184,10 +184,9 @@ def forward( output_hidden_states=True, return_dict=False, ) - if len(output) == 3: - last_hidden_states, pooler_output, all_layers = output - else: - last_hidden_states, all_layers = output + + last_hidden_states, pooler_output, all_layers = output + return { "sentemb": pooler_output, "wordemb": last_hidden_states, From cbfc48fb8225a032716e309292ed0b4f51f874ba Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 12:52:13 +0900 Subject: [PATCH 05/13] commit --- poetry.lock | 95 ++++++++++++++++++++++++++------------------------ pyproject.toml | 1 + 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8dac3a6c..ff91f77b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -752,6 +752,30 @@ files = [ [package.extras] all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] +[[package]] +name = "importlib-metadata" +version = "9.0.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7"}, + {file = "importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc"}, +] + +[package.dependencies] +zipp = ">=3.20" + +[package.extras] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +perf = ["ipython"] +test = ["packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] + [[package]] name = "jinja2" version = "3.1.4" @@ -1253,45 +1277,6 @@ doc = ["nb2plots (>=0.6)", "numpydoc (>=1.5)", "pillow (>=9.4)", "pydata-sphinx- extra = ["lxml (>=4.6)", "pydot (>=1.4.2)", "pygraphviz (>=1.10)", "sympy (>=1.10)"] test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] -[[package]] -name = "numpy" -version = "1.24.4" -description = "Fundamental package for array computing in Python" -optional = false -python-versions = ">=3.8" -groups = ["main", "dev"] -markers = "python_version <= \"3.11\"" -files = [ - {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, - {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, - {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, - {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, - {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, - {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, - {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, - {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, - {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, - {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, - {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, - {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, - {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, - {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, - {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, - {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, - {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, - {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, - {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, - {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, - {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, - {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, - {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, - {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, - {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, -] - [[package]] name = "numpy" version = "1.26.4" @@ -1299,7 +1284,6 @@ description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" groups = ["main", "dev"] -markers = "python_version >= \"3.12\"" files = [ {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, @@ -1804,7 +1788,7 @@ description = "Powerful data structures for data analysis, time series, and stat optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version <= \"3.11\" or python_version >= \"3.14\"" +markers = "python_version < \"3.11\" or python_version >= \"3.14\"" files = [ {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, @@ -1866,7 +1850,6 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" @@ -1905,7 +1888,7 @@ description = "Powerful data structures for data analysis, time series, and stat optional = false python-versions = ">=3.11" groups = ["main"] -markers = "python_version < \"3.14\" and python_version >= \"3.12\"" +markers = "python_version < \"3.14\" and python_version >= \"3.11\"" files = [ {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, @@ -2205,7 +2188,7 @@ description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] -markers = "python_version <= \"3.11\" or python_version >= \"3.14\"" +markers = "python_version < \"3.11\" or python_version >= \"3.14\"" files = [ {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, @@ -3225,7 +3208,7 @@ description = "Provider of IANA time zone data" optional = false python-versions = ">=2" groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version <= \"3.11\" or python_version >= \"3.14\"" +markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version < \"3.11\" or python_version >= \"3.14\"" files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, @@ -3344,7 +3327,27 @@ idna = ">=2.0" multidict = ">=4.0" propcache = ">=0.2.0" +[[package]] +name = "zipp" +version = "4.1.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f"}, + {file = "zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] + [metadata] lock-version = "2.1" python-versions = "^3.10.0" -content-hash = "8dc08aef9797ad49400e4254019ddf1403f1a8a245b2e2caa1931c321c1df30b" +content-hash = "7a53c261eb178883327062bc9b093a6acf1c40329f6290498f33ce9414ba36b2" diff --git a/pyproject.toml b/pyproject.toml index bdd5f32e..ee32177e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ scipy = ">=1.10.0" entmax = ">=1.1" huggingface-hub = ">=0.30.0" protobuf = ">=4.24.4" +importlib-metadata = "^9.0.0" [tool.poetry.dev-dependencies] sphinx-markdown-tables = ">=0.0.15" From e65ed5cdef991775c262a138d8c68cdf31fe6cf2 Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 12:55:29 +0900 Subject: [PATCH 06/13] commit --- comet/encoders/bert.py | 5 ++--- comet/encoders/minilm.py | 1 - comet/encoders/rembert.py | 2 +- comet/encoders/xlmr.py | 8 ++------ comet/encoders/xlmr_xl.py | 6 +----- comet/models/__init__.py | 1 + 6 files changed, 7 insertions(+), 16 deletions(-) diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index a84bf9ae..44d74715 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -19,11 +19,10 @@ """ from typing import Dict, Optional -import torch -from transformers import BertConfig, BertModel - import importlib_metadata import packaging.version as packaging_version +import torch +from transformers import BertConfig, BertModel transformers_version = importlib_metadata.distribution("transformers").version if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index ce950199..c0f02b8c 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -20,7 +20,6 @@ """ import importlib_metadata import packaging.version as packaging_version - from transformers import BertConfig, BertModel transformers_version = importlib_metadata.distribution("transformers").version diff --git a/comet/encoders/rembert.py b/comet/encoders/rembert.py index da8257f1..a66f0fe7 100644 --- a/comet/encoders/rembert.py +++ b/comet/encoders/rembert.py @@ -18,9 +18,9 @@ Pretrained RemBERT encoder from Google. This encoder is similar to BERT but uses sentencepiece like XLMR. """ -from transformers import RemBertConfig, RemBertModel import importlib_metadata import packaging.version as packaging_version +from transformers import RemBertConfig, RemBertModel transformers_version = importlib_metadata.distribution("transformers").version if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index 79d34da8..6a5590e5 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -19,14 +19,10 @@ """ from typing import Dict -import torch import importlib_metadata import packaging.version as packaging_version - -from transformers import ( - XLMRobertaConfig, - XLMRobertaModel -) +import torch +from transformers import XLMRobertaConfig, XLMRobertaModel transformers_version = importlib_metadata.distribution("transformers").version if packaging_version.parse(transformers_version) >= packaging_version.parse("v5.0.0rc0"): diff --git a/comet/encoders/xlmr_xl.py b/comet/encoders/xlmr_xl.py index 6334be5f..eeffe0c3 100644 --- a/comet/encoders/xlmr_xl.py +++ b/comet/encoders/xlmr_xl.py @@ -19,11 +19,7 @@ """ import importlib_metadata import packaging.version as packaging_version - -from transformers import ( - XLMRobertaXLConfig, - XLMRobertaXLModel -) +from transformers import XLMRobertaXLConfig, XLMRobertaXLModel transformers_version = importlib_metadata.distribution("transformers").version if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): diff --git a/comet/models/__init__.py b/comet/models/__init__.py index 29342665..d139daa1 100644 --- a/comet/models/__init__.py +++ b/comet/models/__init__.py @@ -98,6 +98,7 @@ def load_from_checkpoint( # issue number #244 try: from importlib import metadata + import packaging.version as parse_version comet_version = metadata.distribution("unbabel-comet").version From de48c8a56c671b5cef8214f1eca9690f7fb631f4 Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 13:08:49 +0900 Subject: [PATCH 07/13] commit --- comet/__init__.py | 6 +- comet/cli/compare.py | 243 ++++++++-------- comet/cli/mbr.py | 101 ++++--- comet/cli/score.py | 137 +++++---- comet/cli/train.py | 100 ++++--- comet/encoders/__init__.py | 10 +- comet/encoders/base.py | 49 ++-- comet/encoders/bert.py | 37 +-- comet/encoders/minilm.py | 11 +- comet/encoders/rembert.py | 7 +- comet/encoders/xlmr.py | 19 +- comet/encoders/xlmr_xl.py | 7 +- comet/models/__init__.py | 47 ++-- comet/models/base.py | 177 +++++++----- comet/models/download_utils.py | 102 +++---- comet/models/lru_cache.py | 17 +- comet/models/metrics.py | 51 ++-- comet/models/multitask/unified_metric.py | 243 +++++++++------- comet/models/multitask/xcomet_metric.py | 46 +-- comet/models/pooling_utils.py | 24 +- comet/models/predict_pbar.py | 2 +- comet/models/predict_writer.py | 42 ++- comet/models/ranking/ranking_metric.py | 123 ++++---- comet/models/regression/referenceless.py | 86 +++--- comet/models/regression/regression_metric.py | 125 ++++---- comet/models/utils.py | 26 +- comet/modules/feedforward.py | 5 +- comet/modules/layerwise_attention.py | 46 +-- docs/source/conf.py | 50 ++-- .../integration/models/test_ranking_metric.py | 50 ++-- .../models/test_referenceless_regression.py | 46 +-- .../models/test_regression_metric.py | 46 +-- .../integration/models/test_unified_metric.py | 88 +++--- tests/integration/modules/test_feedforward.py | 11 +- tests/unit/encoders/test_bert.py | 35 ++- tests/unit/encoders/test_minilm.py | 52 +++- tests/unit/encoders/test_rembert.py | 52 ++-- tests/unit/encoders/test_xlmr.py | 50 +++- tests/unit/test_cache.py | 3 +- tests/unit/test_download_load.py | 16 +- tests/unit/test_models_predict.py | 266 +++++++++--------- 41 files changed, 1521 insertions(+), 1133 deletions(-) diff --git a/comet/__init__.py b/comet/__init__.py index e5dfe971..dc511745 100644 --- a/comet/__init__.py +++ b/comet/__init__.py @@ -18,9 +18,9 @@ from .models import download_model, load_from_checkpoint -logging.basicConfig(level=logging.INFO, format="%(message)s") +logging.basicConfig(level=logging.INFO, format='%(message)s') logger = logging.getLogger(__name__) -__version__ = "2.2.7" -__copyright__ = "2020 Unbabel. All rights reserved." +__version__ = '2.2.7' +__copyright__ = '2020 Unbabel. All rights reserved.' diff --git a/comet/cli/compare.py b/comet/cli/compare.py index fd59dcd0..1d753509 100644 --- a/comet/cli/compare.py +++ b/comet/cli/compare.py @@ -57,6 +57,7 @@ (default: False) --print_cache_info Print information about COMET cache. (default: False) """ + import json import os from itertools import combinations @@ -73,7 +74,7 @@ from comet import download_model, load_from_checkpoint -torch.set_float32_matmul_precision("high") +torch.set_float32_matmul_precision('high') Statistical_test_info = Dict[str, Union[Path_fr, Dict[str, float]]] @@ -88,36 +89,38 @@ def display_statistical_results(data: Statistical_test_info) -> None: Args: data (Statistical_test_info): Stats to be printed out. """ - print("==========================") - print("x_name:", data["x_name"].rel_path) - print("y_name:", data["y_name"].rel_path) + print('==========================') + print('x_name:', data['x_name'].rel_path) + print('y_name:', data['y_name'].rel_path) - print("\nBootstrap Resampling Results:") - for k, v in data["bootstrap_resampling"].items(): - print("{}:\t{:.4f}".format(k, v)) + print('\nBootstrap Resampling Results:') + for k, v in data['bootstrap_resampling'].items(): + print('{}:\t{:.4f}'.format(k, v)) - print("\nPaired T-Test Results:") - for k, v in data["paired_t-test"].items(): - print("{}:\t{:.4f}".format(k, v)) + print('\nPaired T-Test Results:') + for k, v in data['paired_t-test'].items(): + print('{}:\t{:.4f}'.format(k, v)) - x_seg_scores = data["bootstrap_resampling"]["x-mean"] - y_seg_scores = data["bootstrap_resampling"]["y-mean"] + x_seg_scores = data['bootstrap_resampling']['x-mean'] + y_seg_scores = data['bootstrap_resampling']['y-mean'] best_system = ( - data["x_name"].rel_path + data['x_name'].rel_path if x_seg_scores > y_seg_scores - else data["y_name"].rel_path + else data['y_name'].rel_path ) worse_system = ( - data["x_name"].rel_path + data['x_name'].rel_path if x_seg_scores < y_seg_scores - else data["y_name"].rel_path + else data['y_name'].rel_path ) - if data["paired_t-test"]["p_value"] <= 0.05: - print("Null hypothesis rejected according to t-test.") - print("Scores differ significantly across samples.") - print(f"{best_system} outperforms {worse_system}.") + if data['paired_t-test']['p_value'] <= 0.05: + print('Null hypothesis rejected according to t-test.') + print('Scores differ significantly across samples.') + print(f'{best_system} outperforms {worse_system}.') else: - print("Null hypothesis can't be rejected.\nBoth systems have equal averages.") + print( + "Null hypothesis can't be rejected.\nBoth systems have equal averages." + ) def t_tests_summary( @@ -136,14 +139,14 @@ def t_tests_summary( name2id = {name: i for i, name in enumerate(translations)} grid = [[None] * n for name in translations] for t_test in t_test_results: - p_value = t_test["paired_t-test"]["p_value"] - x_id = name2id[t_test["x_name"]] - y_id = name2id[t_test["y_name"]] + p_value = t_test['paired_t-test']['p_value'] + x_id = name2id[t_test['x_name']] + y_id = name2id[t_test['y_name']] grid[x_id][y_id] = False grid[y_id][x_id] = False if p_value < threshold_p_value: - x_seg_scores = t_test["bootstrap_resampling"]["x-mean"] - y_seg_scores = t_test["bootstrap_resampling"]["y-mean"] + x_seg_scores = t_test['bootstrap_resampling']['x-mean'] + y_seg_scores = t_test['bootstrap_resampling']['y-mean'] if x_seg_scores > y_seg_scores: grid[x_id][y_id] = True else: @@ -152,17 +155,20 @@ def t_tests_summary( # Add the row's name aka the system's name. grid = [(name,) + tuple(row) for name, row in zip(translations, grid)] - print("Summary") - print("If system_x is better than system_y then:") + print('Summary') + print('If system_x is better than system_y then:') print( - f"Null hypothesis rejected according to t-test with p_value={threshold_p_value}." + f'Null hypothesis rejected according to t-test with p_value={threshold_p_value}.' ) - print("Scores differ significantly across samples.") - print(tabulate(grid, headers=("system_x \ system_y",) + translations)) + print('Scores differ significantly across samples.') + print(tabulate(grid, headers=('system_x \ system_y',) + translations)) def calculate_bootstrap( - x_sys_scores: np.ndarray, y_sys_scores: np.ndarray, x_name: Path_fr, y_name: Path_fr + x_sys_scores: np.ndarray, + y_sys_scores: np.ndarray, + x_name: Path_fr, + y_name: Path_fr, ) -> Statistical_test_info: """Calculate bootstrap score, wins and ties for a system pair. @@ -182,14 +188,14 @@ def calculate_bootstrap( x_wins = float(len(delta[delta >= EPS])) y_wins = float(len(delta[delta <= -EPS])) return { - "x_name": x_name, - "y_name": y_name, - "bootstrap_resampling": { - "x-mean": float(np.mean(x_sys_scores)), - "y-mean": float(np.mean(y_sys_scores)), - "ties (%)": ties / num_splits, - "x_wins (%)": x_wins / num_splits, - "y_wins (%)": y_wins / num_splits, + 'x_name': x_name, + 'y_name': y_name, + 'bootstrap_resampling': { + 'x-mean': float(np.mean(x_sys_scores)), + 'y-mean': float(np.mean(y_sys_scores)), + 'ties (%)': ties / num_splits, + 'x_wins (%)': x_wins / num_splits, + 'y_wins (%)': y_wins / num_splits, }, } @@ -206,7 +212,9 @@ def pairwise_bootstrap( Return: Generator(Statistical_test_info): bootstrap resampling stats between systems. """ - assert sys_scores.shape[0] == len(systems), "Each system should have its sys_score." + assert sys_scores.shape[0] == len( + systems + ), 'Each system should have its sys_score.' pairs = combinations(zip(systems, sys_scores), 2) for (x_name, x_sys_scores), (y_name, y_sys_scores) in pairs: @@ -271,7 +279,7 @@ def score(cfg: Namespace, systems: List[Dict[str, List[str]]]) -> np.ndarray: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="auto", + accelerator='auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) @@ -286,16 +294,18 @@ def score(cfg: Namespace, systems: List[Dict[str, List[str]]]) -> np.ndarray: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="cpu" if cfg.gpus == 0 else "auto", + accelerator='cpu' if cfg.gpus == 0 else 'auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) seg_scores += outputs.scores - n = len(systems[0]["src"]) + n = len(systems[0]['src']) # [grouper](https://docs.python.org/3/library/itertools.html#itertools-recipes) seg_scores = list(zip(*[iter(seg_scores)] * n)) - seg_scores = np.array(seg_scores, dtype="float32") # num_systems x num_translations + seg_scores = np.array( + seg_scores, dtype='float32' + ) # num_systems x num_translations return seg_scores @@ -308,111 +318,115 @@ def get_cfg() -> Namespace: parser = ArgumentParser( description="Command for comparing multiple MT systems' translations." ) - parser.add_argument("-s", "--sources", type=Path_fr) - parser.add_argument("-r", "--references", type=Path_fr) - parser.add_argument("-t", "--translations", nargs="*", type=Path_fr) - parser.add_argument("-d", "--sacrebleu_dataset", type=str) - parser.add_argument("--batch_size", type=int, default=8) - parser.add_argument("--gpus", type=int, default=1) + parser.add_argument('-s', '--sources', type=Path_fr) + parser.add_argument('-r', '--references', type=Path_fr) + parser.add_argument('-t', '--translations', nargs='*', type=Path_fr) + parser.add_argument('-d', '--sacrebleu_dataset', type=str) + parser.add_argument('--batch_size', type=int, default=8) + parser.add_argument('--gpus', type=int, default=1) parser.add_argument( - "--quiet", action="store_true", help="Sets all loggers to ERROR level." + '--quiet', action='store_true', help='Sets all loggers to ERROR level.' ) parser.add_argument( - "--only_system", action="store_true", help="Prints only the final system score." + '--only_system', + action='store_true', + help='Prints only the final system score.', ) parser.add_argument( - "--num_splits", + '--num_splits', type=int, default=300, - help="Number of random partitions used in Bootstrap resampling.", + help='Number of random partitions used in Bootstrap resampling.', ) parser.add_argument( - "--sample_ratio", + '--sample_ratio', type=float, default=0.4, - help="Percentage of the testset to use in each split.", + help='Percentage of the testset to use in each split.', ) parser.add_argument( - "--t_test_alternative", + '--t_test_alternative', type=str, - default="two-sided", + default='two-sided', help=( - "Alternative hypothesis from scipy.stats.ttest_rel. The following options" + 'Alternative hypothesis from scipy.stats.ttest_rel. The following options' + " are available: 'two-sided', 'less', 'greater'. Defaults to 'two-sided'" ), ) parser.add_argument( - "--to_json", + '--to_json', type=str, - default="", - help="Exports results to a json file.", + default='', + help='Exports results to a json file.', ) parser.add_argument( - "--model", + '--model', type=str, required=False, - default="Unbabel/wmt22-comet-da", - help="COMET model to be used.", + default='Unbabel/wmt22-comet-da', + help='COMET model to be used.', ) parser.add_argument( - "--model_storage_path", + '--model_storage_path', help=( - "Path to the directory where models will be stored. " - + "By default its saved in ~/.cache/torch/unbabel_comet/" + 'Path to the directory where models will be stored. ' + + 'By default its saved in ~/.cache/torch/unbabel_comet/' ), default=None, ) parser.add_argument( - "--num_workers", - help="Number of workers to use when loading data.", + '--num_workers', + help='Number of workers to use when loading data.', type=int, default=None, ) parser.add_argument( - "--disable_cache", - action="store_true", + '--disable_cache', + action='store_true', help=( - "Disables sentence embeddings caching." - + " This makes inference slower but saves memory." + 'Disables sentence embeddings caching.' + + ' This makes inference slower but saves memory.' ), ) parser.add_argument( - "--disable_length_batching", - action="store_true", - help="Disables length batching. This makes inference slower.", + '--disable_length_batching', + action='store_true', + help='Disables length batching. This makes inference slower.', ) parser.add_argument( - "--print_cache_info", - action="store_true", - help="Print information about COMET cache.", + '--print_cache_info', + action='store_true', + help='Print information about COMET cache.', ) cfg = parser.parse_args() if cfg.sources is None and cfg.sacrebleu_dataset is None: - parser.error(f"You must specify a source (-s) or a sacrebleu dataset (-d)") + parser.error( + f'You must specify a source (-s) or a sacrebleu dataset (-d)' + ) if cfg.sacrebleu_dataset is not None: if cfg.references is not None or cfg.sources is not None: parser.error( - f"Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)" + f'Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)' ) try: - testset, langpair = cfg.sacrebleu_dataset.rsplit(":", maxsplit=1) + testset, langpair = cfg.sacrebleu_dataset.rsplit(':', maxsplit=1) cfg.sources = Path_fr(get_source_file(testset, langpair)) cfg.references = Path_fr(get_reference_files(testset, langpair)[0]) except ValueError: parser.error( - "SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en" + 'SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en' ) except Exception as e: import sys - print("SacreBLEU error:", e, file=sys.stderr) + print('SacreBLEU error:', e, file=sys.stderr) sys.exit(1) - if cfg.model.endswith(".ckpt") and os.path.exists(cfg.model): + if cfg.model.endswith('.ckpt') and os.path.exists(cfg.model): cfg.model_path = cfg.model else: @@ -434,7 +448,9 @@ def compare_command() -> None: if model.requires_references() and (cfg.references is None): parser.error( - "{} requires -r/--references or -d/--sacrebleu_dataset.".format(cfg.model) + '{} requires -r/--references or -d/--sacrebleu_dataset.'.format( + cfg.model + ) ) references = cfg.references if cfg.references is not None else None if not cfg.disable_cache: @@ -443,24 +459,27 @@ def compare_command() -> None: if cfg.print_cache_info: print(model.retrieve_sentence_embedding.cache_info()) - assert len(cfg.translations) > 1, "You must provide at least 2 translation files" + assert ( + len(cfg.translations) > 1 + ), 'You must provide at least 2 translation files' - with open(cfg.sources(), encoding="utf-8") as fp: + with open(cfg.sources(), encoding='utf-8') as fp: sources = [line.strip() for line in fp.readlines()] translations = [] for system in cfg.translations: - with open(system, mode="r", encoding="utf-8") as fp: + with open(system, mode='r', encoding='utf-8') as fp: translations.append([line.strip() for line in fp.readlines()]) if cfg.references is not None: - with open(cfg.references(), encoding="utf-8") as fp: + with open(cfg.references(), encoding='utf-8') as fp: references = [line.strip() for line in fp.readlines()] systems = [ - {"src": sources, "mt": system, "ref": references} for system in translations + {'src': sources, 'mt': system, 'ref': references} + for system in translations ] else: - systems = [{"src": sources, "mt": system} for system in translations] + systems = [{'src': sources, 'mt': system} for system in translations] seg_scores = score(cfg, systems) population_size = seg_scores.shape[1] @@ -478,28 +497,30 @@ def compare_command() -> None: x_seg_scores, y_seg_scores, alternative=cfg.t_test_alternative ) for res in results: - if res["x_name"] == x_name and res["y_name"] == y_name: - res["paired_t-test"] = { - "statistic": ttest_result.statistic, - "p_value": ttest_result.pvalue, + if res['x_name'] == x_name and res['y_name'] == y_name: + res['paired_t-test'] = { + 'statistic': ttest_result.statistic, + 'p_value': ttest_result.pvalue, } info = { - "model": cfg.model, - "statistical_results": results, - "source": sources, - "translations": [ + 'model': cfg.model, + 'statistical_results': results, + 'source': sources, + 'translations': [ { - "name": name, - "mt": trans, - "scores": scores.tolist(), + 'name': name, + 'mt': trans, + 'scores': scores.tolist(), } - for name, trans, scores in zip(cfg.translations, translations, seg_scores) + for name, trans, scores in zip( + cfg.translations, translations, seg_scores + ) ], } if references is not None: - info["reference"] = references + info['reference'] = references for data in results: display_statistical_results(data) @@ -507,11 +528,11 @@ def compare_command() -> None: print() t_tests_summary(results, tuple(cfg.translations)) - if cfg.to_json != "": - with open(cfg.to_json, "w", encoding="utf-8") as outfile: + if cfg.to_json != '': + with open(cfg.to_json, 'w', encoding='utf-8') as outfile: json.dump(info, outfile, ensure_ascii=False, indent=4) - print("Predictions saved in: {}.".format(cfg.to_json)) + print('Predictions saved in: {}.'.format(cfg.to_json)) -if __name__ == "__main__": +if __name__ == '__main__': compare_command() diff --git a/comet/cli/mbr.py b/comet/cli/mbr.py index 6d018527..7fa53fd5 100644 --- a/comet/cli/mbr.py +++ b/comet/cli/mbr.py @@ -44,6 +44,7 @@ -o OUTPUT, --output OUTPUT Best candidates after running MBR decoding. (required, type: str) """ + import os from typing import List, Tuple @@ -55,7 +56,7 @@ from comet.models import RegressionMetric, download_model, load_from_checkpoint -torch.set_float32_matmul_precision("high") +torch.set_float32_matmul_precision('high') def build_embeddings( @@ -88,8 +89,8 @@ def build_embeddings( src_embeddings = [] with torch.no_grad(): for batch in src_inputs: - input_ids = batch["input_ids"].to(model.device) - attention_mask = batch["attention_mask"].to(model.device) + input_ids = batch['input_ids'].to(model.device) + attention_mask = batch['attention_mask'].to(model.device) src_embeddings.append( model.get_sentence_embedding(input_ids, attention_mask) ) @@ -97,9 +98,11 @@ def build_embeddings( mt_embeddings = [] with torch.no_grad(): - for batch in tqdm(mt_inputs, desc="Encoding sentences...", dynamic_ncols=True): - input_ids = batch["input_ids"].to(model.device) - attention_mask = batch["attention_mask"].to(model.device) + for batch in tqdm( + mt_inputs, desc='Encoding sentences...', dynamic_ncols=True + ): + input_ids = batch['input_ids'].to(model.device) + attention_mask = batch['attention_mask'].to(model.device) mt_embeddings.append( model.get_sentence_embedding(input_ids, attention_mask) ) @@ -109,7 +112,9 @@ def build_embeddings( def mbr_decoding( - src_embeddings: torch.Tensor, mt_embeddings: torch.Tensor, model: RegressionMetric + src_embeddings: torch.Tensor, + mt_embeddings: torch.Tensor, + model: RegressionMetric, ) -> torch.Tensor: """Performs MBR Decoding for each translation for a given source. @@ -128,7 +133,9 @@ def mbr_decoding( with torch.no_grad(): # Loop over all source sentences for i in tqdm( - range(mbr_matrix.shape[0]), desc="MBR Scores...", dynamic_ncols=True + range(mbr_matrix.shape[0]), + desc='MBR Scores...', + dynamic_ncols=True, ): source = src_embeddings[i, :].repeat(num_samples, 1) # Loop over all hypothesis @@ -136,7 +143,9 @@ def mbr_decoding( translation = mt_embeddings[i, j, :].repeat(num_samples, 1) # Score current hypothesis against all others pseudo_refs = mt_embeddings[i, :] - scores = model.estimate(source, translation, pseudo_refs)["score"] + scores = model.estimate(source, translation, pseudo_refs)[ + 'score' + ] scores = torch.cat([scores[0:j], scores[j + 1 :]]) mbr_matrix[i, j] = scores.mean() @@ -173,10 +182,12 @@ def rerank_top_k( data = [] for i in range(len(sources)): for j in range(num_samples): - data.append({"src": sources[i], "mt": translations[i][j]}) + data.append({'src': sources[i], 'mt': translations[i][j]}) model_output = qe_model.predict(data, batch_size=batch_size, gpus=gpus) - seg_scores = np.array(model_output.scores).reshape(len(sources), num_samples) + seg_scores = np.array(model_output.scores).reshape( + len(sources), num_samples + ) topk_indices = np.argsort(seg_scores, axis=1) topk_translations = [] for i in range(len(sources)): @@ -188,62 +199,64 @@ def rerank_top_k( def mbr_command() -> None: - parser = ArgumentParser(description="Command for Minimum Bayes Risk Decoding.") - parser.add_argument("-s", "--sources", type=Path_fr, required=True) - parser.add_argument("-t", "--translations", type=Path_fr, required=True) - parser.add_argument("--num_samples", type=int, required=True) - parser.add_argument("--batch_size", type=int, default=32) - parser.add_argument("--gpus", type=int, default=1) + parser = ArgumentParser( + description='Command for Minimum Bayes Risk Decoding.' + ) + parser.add_argument('-s', '--sources', type=Path_fr, required=True) + parser.add_argument('-t', '--translations', type=Path_fr, required=True) + parser.add_argument('--num_samples', type=int, required=True) + parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--gpus', type=int, default=1) parser.add_argument( - "--rerank_top_k", + '--rerank_top_k', type=int, default=0, help=( - "Chooses the topK candidates according to --qe_model before applying MBR." - + " Disabled by default." + 'Chooses the topK candidates according to --qe_model before applying MBR.' + + ' Disabled by default.' ), ) parser.add_argument( - "--qe_model", + '--qe_model', type=str, required=False, - default="Unbabel/wmt22-cometkiwi-da", - help="Reference Free model used for reranking before MBR.", + default='Unbabel/wmt22-cometkiwi-da', + help='Reference Free model used for reranking before MBR.', ) parser.add_argument( - "--model", + '--model', type=str, required=False, - default="Unbabel/wmt22-comet-da", - help="COMET model to be used.", + default='Unbabel/wmt22-comet-da', + help='COMET model to be used.', ) parser.add_argument( - "--model_storage_path", + '--model_storage_path', help=( - "Path to the directory where models will be stored. " - + "By default its saved in ~/.cache/torch/unbabel_comet/" + 'Path to the directory where models will be stored. ' + + 'By default its saved in ~/.cache/torch/unbabel_comet/' ), default=None, ) parser.add_argument( - "-o", - "--output", + '-o', + '--output', type=str, required=True, - help="Best candidates after running MBR decoding.", + help='Best candidates after running MBR decoding.', ) cfg = parser.parse_args() - with open(cfg.sources(), encoding="utf-8") as fp: + with open(cfg.sources(), encoding='utf-8') as fp: sources = [line.strip() for line in fp.readlines()] - with open(cfg.translations(), encoding="utf-8") as fp: + with open(cfg.translations(), encoding='utf-8') as fp: translations = [line.strip() for line in fp.readlines()] num_samples = cfg.num_samples # Running QE reranking before MBR! if cfg.rerank_top_k > 0: - if cfg.qe_model.endswith(".ckpt") and os.path.exists(cfg.qe_model): + if cfg.qe_model.endswith('.ckpt') and os.path.exists(cfg.qe_model): qe_model_path = cfg.qe_model else: qe_model_path = download_model( @@ -251,11 +264,11 @@ def mbr_command() -> None: ) assert ( cfg.rerank_top_k < cfg.num_samples - ), "--rerank_top_k needs to be smaller than number of candidates provided!" + ), '--rerank_top_k needs to be smaller than number of candidates provided!' model = load_from_checkpoint(qe_model_path) assert ( not model.requires_references() - ), "--qe_model expects a Reference Free model!" + ), '--qe_model expects a Reference Free model!' translations = rerank_top_k( sources, @@ -268,10 +281,12 @@ def mbr_command() -> None: ) num_samples = cfg.rerank_top_k - if cfg.model.endswith(".ckpt") and os.path.exists(cfg.model): + if cfg.model.endswith('.ckpt') and os.path.exists(cfg.model): model_path = cfg.model else: - model_path = download_model(cfg.model, saving_directory=cfg.model_storage_path) + model_path = download_model( + cfg.model, saving_directory=cfg.model_storage_path + ) model = load_from_checkpoint(model_path) model.eval() @@ -280,7 +295,7 @@ def mbr_command() -> None: if not isinstance(model, RegressionMetric): raise Exception( - "Invalid model ({}). MBR command only works with Reference-based Regression models!".format( + 'Invalid model ({}). MBR command only works with Reference-based Regression models!'.format( model.__class__.__name__ ) ) @@ -301,10 +316,10 @@ def mbr_command() -> None: best_cand_idx = torch.argmax(mbr_matrix[i, :]) best_candidates.append(samples[best_cand_idx]) - with open(cfg.output, "w", encoding="utf-8") as fp: + with open(cfg.output, 'w', encoding='utf-8') as fp: for sample in best_candidates: - fp.write(sample + "\n") + fp.write(sample + '\n') -if __name__ == "__main__": +if __name__ == '__main__': mbr_command() diff --git a/comet/cli/score.py b/comet/cli/score.py index 31ce9efa..6b0e1e51 100644 --- a/comet/cli/score.py +++ b/comet/cli/score.py @@ -48,6 +48,7 @@ (default: False) --print_cache_info Print information about COMET cache. (default: False) """ + import itertools import json import logging @@ -63,105 +64,113 @@ from comet import download_model, load_from_checkpoint from comet.models.utils import split_sequence_into_sublists -torch.set_float32_matmul_precision("high") +torch.set_float32_matmul_precision('high') def score_command() -> None: - parser = ArgumentParser(description="Command for scoring MT systems.") - parser.add_argument("-s", "--sources", type=Path_fr) - parser.add_argument("-t", "--translations", type=Path_fr, nargs="+") - parser.add_argument("-r", "--references", type=Path_fr) - parser.add_argument("-d", "--sacrebleu_dataset", type=str) - parser.add_argument("--batch_size", type=int, default=16) - parser.add_argument("--gpus", type=int, default=1) + parser = ArgumentParser(description='Command for scoring MT systems.') + parser.add_argument('-s', '--sources', type=Path_fr) + parser.add_argument('-t', '--translations', type=Path_fr, nargs='+') + parser.add_argument('-r', '--references', type=Path_fr) + parser.add_argument('-d', '--sacrebleu_dataset', type=str) + parser.add_argument('--batch_size', type=int, default=16) + parser.add_argument('--gpus', type=int, default=1) parser.add_argument( - "--quiet", action="store_true", help="Sets all loggers to ERROR level." + '--quiet', action='store_true', help='Sets all loggers to ERROR level.' ) parser.add_argument( - "--enable-context", - action="store_true", - help="Enables contextual extension of COMET on inputs preprocessed with context information.", + '--enable-context', + action='store_true', + help='Enables contextual extension of COMET on inputs preprocessed with context information.', ) parser.add_argument( - "--only_system", action="store_true", help="Prints only the final system score." + '--only_system', + action='store_true', + help='Prints only the final system score.', ) parser.add_argument( - "--to_json", + '--to_json', type=str, - default="", - help="Exports results to a json file.", + default='', + help='Exports results to a json file.', ) parser.add_argument( - "--model", + '--model', type=str, required=False, - default="Unbabel/wmt22-comet-da", - help="COMET model to be used.", + default='Unbabel/wmt22-comet-da', + help='COMET model to be used.', ) parser.add_argument( - "--model_storage_path", + '--model_storage_path', help=( - "Path to the directory where models will be stored. " - + "By default its saved in ~/.cache/torch/unbabel_comet/" + 'Path to the directory where models will be stored. ' + + 'By default its saved in ~/.cache/torch/unbabel_comet/' ), default=None, ) parser.add_argument( - "--num_workers", - help="Number of workers to use when loading data.", + '--num_workers', + help='Number of workers to use when loading data.', type=int, default=None, ) parser.add_argument( - "--disable_cache", - action="store_true", - help="Disables sentence embeddings caching. This makes inference slower but saves memory.", + '--disable_cache', + action='store_true', + help='Disables sentence embeddings caching. This makes inference slower but saves memory.', ) parser.add_argument( - "--disable_length_batching", - action="store_true", - help="Disables length batching. This makes inference slower.", + '--disable_length_batching', + action='store_true', + help='Disables length batching. This makes inference slower.', ) parser.add_argument( - "--print_cache_info", - action="store_true", - help="Print information about COMET cache.", + '--print_cache_info', + action='store_true', + help='Print information about COMET cache.', ) cfg = parser.parse_args() if cfg.quiet: - loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] + loggers = [ + logging.getLogger(name) for name in logging.root.manager.loggerDict + ] for logger in loggers: logger.setLevel(logging.ERROR) seed_everything(1) if cfg.sources is None and cfg.sacrebleu_dataset is None: - parser.error(f"You must specify a source (-s) or a sacrebleu dataset (-d)") + parser.error( + f'You must specify a source (-s) or a sacrebleu dataset (-d)' + ) if cfg.sacrebleu_dataset is not None: if cfg.references is not None or cfg.sources is not None: parser.error( - f"Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)" + f'Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)' ) try: - testset, langpair = cfg.sacrebleu_dataset.rsplit(":", maxsplit=1) + testset, langpair = cfg.sacrebleu_dataset.rsplit(':', maxsplit=1) cfg.sources = Path_fr(get_source_file(testset, langpair)) cfg.references = Path_fr(get_reference_files(testset, langpair)[0]) except ValueError: parser.error( - "SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en" + 'SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en' ) except Exception as e: import sys - print("SacreBLEU error:", e, file=sys.stderr) + print('SacreBLEU error:', e, file=sys.stderr) sys.exit(1) - if cfg.model.endswith(".ckpt") and os.path.exists(cfg.model): + if cfg.model.endswith('.ckpt') and os.path.exists(cfg.model): model_path = cfg.model else: - model_path = download_model(cfg.model, saving_directory=cfg.model_storage_path) + model_path = download_model( + cfg.model, saving_directory=cfg.model_storage_path + ) model = load_from_checkpoint(model_path) model.eval() @@ -172,30 +181,32 @@ def score_command() -> None: if model.requires_references() and (cfg.references is None): parser.error( - "{} requires -r/--references or -d/--sacrebleu_dataset.".format(cfg.model) + '{} requires -r/--references or -d/--sacrebleu_dataset.'.format( + cfg.model + ) ) if not cfg.disable_cache: model.set_embedding_cache() - with open(cfg.sources(), encoding="utf-8") as fp: + with open(cfg.sources(), encoding='utf-8') as fp: sources = [line.strip() for line in fp.readlines()] translations = [] for path_fr in cfg.translations: - with open(path_fr(), encoding="utf-8") as fp: + with open(path_fr(), encoding='utf-8') as fp: translations.append([line.strip() for line in fp.readlines()]) if cfg.references is not None: - with open(cfg.references(), encoding="utf-8") as fp: + with open(cfg.references(), encoding='utf-8') as fp: references = [line.strip() for line in fp.readlines()] data = { - "src": [sources for _ in translations], - "mt": translations, - "ref": [references for _ in translations], + 'src': [sources for _ in translations], + 'mt': translations, + 'ref': [references for _ in translations], } else: - data = {"src": [sources for _ in translations], "mt": translations} + data = {'src': [sources for _ in translations], 'mt': translations} if cfg.gpus > 1: # Flatten all data to score across multiple GPUs @@ -208,12 +219,12 @@ def score_command() -> None: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="auto", + accelerator='auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) seg_scores = outputs.scores - if "metadata" in outputs and "error_spans" in outputs.metadata: + if 'metadata' in outputs and 'error_spans' in outputs.metadata: errors = outputs.metadata.error_spans else: errors = [] @@ -250,42 +261,44 @@ def score_command() -> None: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="cpu" if cfg.gpus == 0 else "auto", + accelerator='cpu' if cfg.gpus == 0 else 'auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) seg_scores.append(outputs.scores) sys_scores.append(outputs.system_score) - if "metadata" in outputs and "error_spans" in outputs.metadata: + if 'metadata' in outputs and 'error_spans' in outputs.metadata: errors.append(outputs.metadata.error_spans) data = new_data files = [path_fr.rel_path for path_fr in cfg.translations] - data = {file: system_data.tolist() for file, system_data in zip(files, data)} + data = { + file: system_data.tolist() for file, system_data in zip(files, data) + } for i in range(len(data[files[0]])): # loop over (src, ref) for j in range(len(files)): # loop of system - data[files[j]][i]["COMET"] = seg_scores[j][i] + data[files[j]][i]['COMET'] = seg_scores[j][i] if errors and errors[j] and errors[j][i]: - data[files[j]][i]["errors"] = errors[j][i] + data[files[j]][i]['errors'] = errors[j][i] if not cfg.only_system: print( - "{}\tSegment {}\tscore: {:.4f}".format( + '{}\tSegment {}\tscore: {:.4f}'.format( files[j], i, seg_scores[j][i] ) ) for j in range(len(files)): - print("{}\tscore: {:.4f}".format(files[j], sys_scores[j])) + print('{}\tscore: {:.4f}'.format(files[j], sys_scores[j])) - if cfg.to_json != "": - with open(cfg.to_json, "w", encoding="utf-8") as outfile: + if cfg.to_json != '': + with open(cfg.to_json, 'w', encoding='utf-8') as outfile: json.dump(data, outfile, ensure_ascii=False, indent=4) - print("Predictions saved in: {}.".format(cfg.to_json)) + print('Predictions saved in: {}.'.format(cfg.to_json)) if cfg.print_cache_info: print(model.retrieve_sentence_embedding.cache_info()) -if __name__ == "__main__": +if __name__ == '__main__': score_command() diff --git a/comet/cli/train.py b/comet/cli/train.py index 893a4be4..854a9705 100644 --- a/comet/cli/train.py +++ b/comet/cli/train.py @@ -28,6 +28,7 @@ comet-train --help ``` """ + import json import logging import warnings @@ -35,44 +36,51 @@ import torch from jsonargparse import ActionConfigFile, ArgumentParser, namespace_to_dict from pytorch_lightning import seed_everything -from pytorch_lightning.callbacks import (EarlyStopping, LearningRateMonitor, - ModelCheckpoint) +from pytorch_lightning.callbacks import ( + EarlyStopping, + LearningRateMonitor, + ModelCheckpoint, +) from pytorch_lightning.trainer.trainer import Trainer -from comet.models import (RankingMetric, ReferencelessRegression, - RegressionMetric, UnifiedMetric) +from comet.models import ( + RankingMetric, + ReferencelessRegression, + RegressionMetric, + UnifiedMetric, +) -torch.set_float32_matmul_precision("high") +torch.set_float32_matmul_precision('high') logger = logging.getLogger(__name__) def read_arguments() -> ArgumentParser: - parser = ArgumentParser(description="Command for training COMET models.") + parser = ArgumentParser(description='Command for training COMET models.') parser.add_argument( - "--seed_everything", + '--seed_everything', type=int, default=12, - help="Training Seed.", + help='Training Seed.', ) - parser.add_argument("--cfg", action=ActionConfigFile) - parser.add_subclass_arguments(RegressionMetric, "regression_metric") + parser.add_argument('--cfg', action=ActionConfigFile) + parser.add_subclass_arguments(RegressionMetric, 'regression_metric') parser.add_subclass_arguments( - ReferencelessRegression, "referenceless_regression_metric" + ReferencelessRegression, 'referenceless_regression_metric' ) - parser.add_subclass_arguments(RankingMetric, "ranking_metric") - parser.add_subclass_arguments(UnifiedMetric, "unified_metric") - parser.add_subclass_arguments(EarlyStopping, "early_stopping") - parser.add_subclass_arguments(ModelCheckpoint, "model_checkpoint") - parser.add_subclass_arguments(Trainer, "trainer") + parser.add_subclass_arguments(RankingMetric, 'ranking_metric') + parser.add_subclass_arguments(UnifiedMetric, 'unified_metric') + parser.add_subclass_arguments(EarlyStopping, 'early_stopping') + parser.add_subclass_arguments(ModelCheckpoint, 'model_checkpoint') + parser.add_subclass_arguments(Trainer, 'trainer') parser.add_argument( - "--load_from_checkpoint", - help="Loads a model checkpoint for fine-tuning", + '--load_from_checkpoint', + help='Loads a model checkpoint for fine-tuning', default=None, ) parser.add_argument( - "--strict_load", - action="store_true", + '--strict_load', + action='store_true', help="Strictly enforce that the keys in checkpoint_path match the keys returned by this module's state dict.", ) return parser @@ -86,16 +94,20 @@ def initialize_trainer(configs) -> Trainer: **namespace_to_dict(configs.early_stopping.init_args) ) trainer_args = namespace_to_dict(configs.trainer.init_args) - lr_monitor = LearningRateMonitor(logging_interval="step") - trainer_args["callbacks"] = [early_stop_callback, checkpoint_callback, lr_monitor] - print("TRAINER ARGUMENTS: ") + lr_monitor = LearningRateMonitor(logging_interval='step') + trainer_args['callbacks'] = [ + early_stop_callback, + checkpoint_callback, + lr_monitor, + ] + print('TRAINER ARGUMENTS: ') print(json.dumps(trainer_args, indent=4, default=lambda x: x.__dict__)) trainer = Trainer(**trainer_args) return trainer def initialize_model(configs): - print("MODEL ARGUMENTS: ") + print('MODEL ARGUMENTS: ') if configs.regression_metric is not None: print( json.dumps( @@ -105,7 +117,7 @@ def initialize_model(configs): ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = RegressionMetric.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, @@ -124,48 +136,60 @@ def initialize_model(configs): ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = ReferencelessRegression.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, - **namespace_to_dict(configs.referenceless_regression_metric.init_args), + **namespace_to_dict( + configs.referenceless_regression_metric.init_args + ), ) else: model = ReferencelessRegression( - **namespace_to_dict(configs.referenceless_regression_metric.init_args) + **namespace_to_dict( + configs.referenceless_regression_metric.init_args + ) ) elif configs.ranking_metric is not None: print( json.dumps( - configs.ranking_metric.init_args, indent=4, default=lambda x: x.__dict__ + configs.ranking_metric.init_args, + indent=4, + default=lambda x: x.__dict__, ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = RankingMetric.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, **namespace_to_dict(configs.ranking_metric.init_args), ) else: - model = RankingMetric(**namespace_to_dict(configs.ranking_metric.init_args)) + model = RankingMetric( + **namespace_to_dict(configs.ranking_metric.init_args) + ) elif configs.unified_metric is not None: print( json.dumps( - configs.unified_metric.init_args, indent=4, default=lambda x: x.__dict__ + configs.unified_metric.init_args, + indent=4, + default=lambda x: x.__dict__, ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = UnifiedMetric.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, **namespace_to_dict(configs.unified_metric.init_args), ) else: - model = UnifiedMetric(**namespace_to_dict(configs.unified_metric.init_args)) + model = UnifiedMetric( + **namespace_to_dict(configs.unified_metric.init_args) + ) else: - raise Exception("Model configurations missing!") + raise Exception('Model configurations missing!') return model @@ -181,12 +205,12 @@ def train_command() -> None: # 2 workers per gpu is enough! If set to the number of cpus on this machine # it throws another exception saying its too many workers. warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) -if __name__ == "__main__": +if __name__ == '__main__': train_command() diff --git a/comet/encoders/__init__.py b/comet/encoders/__init__.py index 6de0aecf..27b04b8f 100644 --- a/comet/encoders/__init__.py +++ b/comet/encoders/__init__.py @@ -18,9 +18,9 @@ from .xlmr_xl import XLMRXLEncoder str2encoder = { - "BERT": BERTEncoder, - "XLM-RoBERTa": XLMREncoder, - "MiniLM": MiniLMEncoder, - "XLM-RoBERTa-XL": XLMRXLEncoder, - "RemBERT": RemBERTEncoder, + 'BERT': BERTEncoder, + 'XLM-RoBERTa': XLMREncoder, + 'MiniLM': MiniLMEncoder, + 'XLM-RoBERTa-XL': XLMRXLEncoder, + 'RemBERT': RemBERTEncoder, } diff --git a/comet/encoders/base.py b/comet/encoders/base.py index 1de65536..ebd8c674 100644 --- a/comet/encoders/base.py +++ b/comet/encoders/base.py @@ -16,6 +16,7 @@ ==================== Module defining the common interface between all pretrained encoder models. """ + import abc from typing import Dict, List, Optional, Tuple @@ -129,16 +130,16 @@ def align_tokens_and_annotations( ) -> Tuple[List[int], List[int]]: """Inspired by: https://github.com/LightTag/sequence-labeling-with-transformers/""" tokens = tokenized.tokens - aligned_labels = ["O"] * len(tokens) + aligned_labels = ['O'] * len(tokens) for anno in annotations: # A set that stores the token indices of the annotation annotation_token_ix_set = set() - for char_ix in range(anno["start"], anno["end"]): + for char_ix in range(anno['start'], anno['end']): token_ix = tokenized.char_to_token(char_ix) if token_ix is not None: annotation_token_ix_set.add(token_ix) for _, token_ix in enumerate(sorted(annotation_token_ix_set)): - prefix = "I" + prefix = 'I' aligned_labels[token_ix] = f"{prefix}-{anno['severity']}" def get_label_id(item): @@ -148,7 +149,7 @@ def get_label_id(item): label = self.labelset.labels_to_id.get(item) if label is None: raise Exception( - f"{label} does not exist in self.labelset: {self.labelset.labels_to_id.keys()}" + f'{label} does not exist in self.labelset: {self.labelset.labels_to_id.keys()}' ) return label @@ -183,14 +184,16 @@ def subword_tokenize( attention_mask = [[1 for _ in seq] for seq in input_ids] max_length = max([len(l) for l in input_ids]) - input_ids = self.pad_list(input_ids, max_length, self.tokenizer.pad_token_id) + input_ids = self.pad_list( + input_ids, max_length, self.tokenizer.pad_token_id + ) label_ids = self.pad_list(label_ids, max_length, -1) attention_mask = self.pad_list(attention_mask, max_length, 0) return { - "input_ids": torch.tensor(input_ids), - "label_ids": torch.tensor(label_ids), - "attention_mask": torch.tensor(attention_mask), - "offsets": offsets, # Used during inference + 'input_ids': torch.tensor(input_ids), + 'label_ids': torch.tensor(label_ids), + 'attention_mask': torch.tensor(attention_mask), + 'offsets': offsets, # Used during inference } def prepare_sample( @@ -217,7 +220,7 @@ def prepare_sample( else: tokenizer_output = self.tokenizer( sample, - return_tensors="pt", + return_tensors='pt', padding=True, truncation=True, max_length=self.max_positions - 2, @@ -280,7 +283,7 @@ def concat_sequences( # Remove padding before concatenation for encoder_input in inputs: - input_ids = encoder_input["input_ids"] + input_ids = encoder_input['input_ids'] input_ids = [ x.masked_select(x.ne(self.tokenizer.pad_token_id)).tolist() for x in input_ids.unbind(dim=0) @@ -316,30 +319,36 @@ def concat_sequences( lengths = [t.shape[0] for t in batch] max_len = max(lengths) padded = [ - self.pad_tensor(t, max_len, self.tokenizer.pad_token_id) for t in batch + self.pad_tensor(t, max_len, self.tokenizer.pad_token_id) + for t in batch ] lengths = torch.tensor(lengths, dtype=torch.long) padded = torch.stack(padded, dim=0).contiguous() attention_mask = torch.arange(max_len)[None, :] < lengths[:, None] if return_label_ids: label_ids = [ - self.pad_tensor(t, max_len, -1) for t in inputs[0]["label_ids"] + self.pad_tensor(t, max_len, -1) for t in inputs[0]['label_ids'] ] label_ids = torch.stack(label_ids, dim=0).contiguous() encoder_input = { - "input_ids": padded, - "attention_mask": attention_mask, - "label_ids": label_ids, - "mt_offsets": inputs[0]["offsets"], # Used during inference + 'input_ids': padded, + 'attention_mask': attention_mask, + 'label_ids': label_ids, + 'mt_offsets': inputs[0]['offsets'], # Used during inference } else: - encoder_input = {"input_ids": padded, "attention_mask": attention_mask} + encoder_input = { + 'input_ids': padded, + 'attention_mask': attention_mask, + } if self.uses_token_type_ids: - token_type_ids = [self.pad_tensor(t, max_len, 1) for t in token_type_ids] + token_type_ids = [ + self.pad_tensor(t, max_len, 1) for t in token_type_ids + ] token_type_ids = torch.stack(token_type_ids, dim=0).contiguous() - encoder_input["token_type_ids"] = token_type_ids + encoder_input['token_type_ids'] = token_type_ids return encoder_input, lengths, max_len diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index 44d74715..53925573 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -17,6 +17,7 @@ ============== Pretrained BERT encoder from Hugging Face. """ + from typing import Dict, Optional import importlib_metadata @@ -24,8 +25,10 @@ import torch from transformers import BertConfig, BertModel -transformers_version = importlib_metadata.distribution("transformers").version -if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): from transformers import BertTokenizer as BertTokenizer else: from transformers import BertTokenizerFast as BertTokenizer @@ -116,7 +119,9 @@ def from_pretrained( Returns: Encoder: XLMREncoder object. """ - return BERTEncoder(pretrained_model, load_pretrained_weights, local_files_only) + return BERTEncoder( + pretrained_model, load_pretrained_weights, local_files_only + ) def freeze_embeddings(self) -> None: """Frezees the embedding layer.""" @@ -136,23 +141,23 @@ def layerwise_lr(self, lr: float, decay: float): # Last layer keeps LR opt_parameters = [ { - "params": self.model.encoder.layer[-1].parameters(), - "lr": lr, + 'params': self.model.encoder.layer[-1].parameters(), + 'lr': lr, } ] # Decay at each layer. for i in range(2, self.num_layers): opt_parameters.append( { - "params": self.model.encoder.layer[-i].parameters(), - "lr": lr * decay ** (i - 1), + 'params': self.model.encoder.layer[-i].parameters(), + 'lr': lr * decay ** (i - 1), } ) # Embedding Layer opt_parameters.append( { - "params": self.model.embeddings.parameters(), - "lr": lr * decay ** (self.num_layers), + 'params': self.model.embeddings.parameters(), + 'lr': lr * decay ** (self.num_layers), } ) return opt_parameters @@ -162,7 +167,7 @@ def forward( input_ids: torch.Tensor, attention_mask: torch.Tensor, token_type_ids: Optional[torch.tensor] = None, - **kwargs + **kwargs, ) -> Dict[str, torch.Tensor]: """BERT model forward @@ -183,14 +188,14 @@ def forward( output_hidden_states=True, return_dict=False, ) - + last_hidden_states, pooler_output, all_layers = output - + return { - "sentemb": pooler_output, - "wordemb": last_hidden_states, - "all_layers": all_layers, - "attention_mask": attention_mask, + 'sentemb': pooler_output, + 'wordemb': last_hidden_states, + 'all_layers': all_layers, + 'attention_mask': attention_mask, } def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index c0f02b8c..baf37f6a 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -18,12 +18,15 @@ Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT architecture with an XLMR tokenizer. """ + import importlib_metadata import packaging.version as packaging_version from transformers import BertConfig, BertModel -transformers_version = importlib_metadata.distribution("transformers").version -if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer else: from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer @@ -49,7 +52,9 @@ def __init__( ) -> None: super(Encoder, self).__init__() self.tokenizer = XLMRobertaTokenizer.from_pretrained( - "xlm-roberta-base", use_fast=True, local_files_only=local_files_only + 'xlm-roberta-base', + use_fast=True, + local_files_only=local_files_only, ) if load_pretrained_weights: self.model = BertModel.from_pretrained(pretrained_model) diff --git a/comet/encoders/rembert.py b/comet/encoders/rembert.py index a66f0fe7..ae0e0109 100644 --- a/comet/encoders/rembert.py +++ b/comet/encoders/rembert.py @@ -18,12 +18,15 @@ Pretrained RemBERT encoder from Google. This encoder is similar to BERT but uses sentencepiece like XLMR. """ + import importlib_metadata import packaging.version as packaging_version from transformers import RemBertConfig, RemBertModel -transformers_version = importlib_metadata.distribution("transformers").version -if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): from transformers import RemBertTokenizer as RemBertTokenizer else: from transformers import RemBertTokenizerFast as RemBertTokenizer diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index 6a5590e5..dca310c4 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -17,6 +17,7 @@ ============== Pretrained XLM-RoBERTa encoder from Hugging Face. """ + from typing import Dict import importlib_metadata @@ -24,8 +25,10 @@ import torch from transformers import XLMRobertaConfig, XLMRobertaModel -transformers_version = importlib_metadata.distribution("transformers").version -if packaging_version.parse(transformers_version) >= packaging_version.parse("v5.0.0rc0"): +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.parse(transformers_version) >= packaging_version.parse( + 'v5.0.0rc0' +): from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer else: from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer @@ -95,7 +98,9 @@ def from_pretrained( Returns: Encoder: XLMREncoder object. """ - return XLMREncoder(pretrained_model, load_pretrained_weights, local_files_only) + return XLMREncoder( + pretrained_model, load_pretrained_weights, local_files_only + ) def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs @@ -113,10 +118,10 @@ def forward( last_hidden_states, _, all_layers = output return { - "sentemb": last_hidden_states[:, 0, :], - "wordemb": last_hidden_states, - "all_layers": all_layers, - "attention_mask": attention_mask, + 'sentemb': last_hidden_states[:, 0, :], + 'wordemb': last_hidden_states, + 'all_layers': all_layers, + 'attention_mask': attention_mask, } def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): diff --git a/comet/encoders/xlmr_xl.py b/comet/encoders/xlmr_xl.py index eeffe0c3..57db132d 100644 --- a/comet/encoders/xlmr_xl.py +++ b/comet/encoders/xlmr_xl.py @@ -17,12 +17,15 @@ ============== Pretrained XLM-RoBERTa-XL encoder from Hugging Face. """ + import importlib_metadata import packaging.version as packaging_version from transformers import XLMRobertaXLConfig, XLMRobertaXLModel -transformers_version = importlib_metadata.distribution("transformers").version -if packaging_version.Version(transformers_version) >= packaging_version.Version("v5.0.0rc0"): +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer else: from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer diff --git a/comet/models/__init__.py b/comet/models/__init__.py index d139daa1..193617db 100644 --- a/comet/models/__init__.py +++ b/comet/models/__init__.py @@ -30,11 +30,11 @@ from .regression.regression_metric import RegressionMetric str2model = { - "referenceless_regression_metric": ReferencelessRegression, - "regression_metric": RegressionMetric, - "ranking_metric": RankingMetric, - "unified_metric": UnifiedMetric, - "xcomet_metric": XCOMETMetric, + 'referenceless_regression_metric': ReferencelessRegression, + 'regression_metric': RegressionMetric, + 'ranking_metric': RankingMetric, + 'unified_metric': UnifiedMetric, + 'xcomet_metric': XCOMETMetric, } @@ -45,7 +45,9 @@ def download_model( ) -> str: try: model_path = snapshot_download( - repo_id=model, cache_dir=saving_directory, local_files_only=local_files_only + repo_id=model, + cache_dir=saving_directory, + local_files_only=local_files_only, ) except Exception: try: @@ -53,7 +55,9 @@ def download_model( except Exception: raise KeyError(f"Model '{model}' not supported by COMET.") else: - checkpoint_path = os.path.join(*[model_path, "checkpoints", "model.ckpt"]) + checkpoint_path = os.path.join( + *[model_path, 'checkpoints', 'model.ckpt'] + ) return checkpoint_path @@ -82,16 +86,16 @@ def load_from_checkpoint( checkpoint_path = Path(checkpoint_path) if not checkpoint_path.is_file(): - raise Exception(f"Invalid checkpoint path: {checkpoint_path}") + raise Exception(f'Invalid checkpoint path: {checkpoint_path}') parent_folder = checkpoint_path.parents[1] # .parent.parent - hparams_file = parent_folder / "hparams.yaml" + hparams_file = parent_folder / 'hparams.yaml' if hparams_file.is_file(): with open(hparams_file) as yaml_file: hparams = yaml.load(yaml_file.read(), Loader=yaml.FullLoader) - model_class = str2model[hparams["class_identifier"]] + model_class = str2model[hparams['class_identifier']] # Check comet version and hparams for layer_transformation # This is a workaround for the bug reported in version 2.2.4 @@ -101,27 +105,28 @@ def load_from_checkpoint( import packaging.version as parse_version - comet_version = metadata.distribution("unbabel-comet").version + comet_version = metadata.distribution('unbabel-comet').version use_softmax = ( - parse_version.parse(comet_version) >= parse_version.parse("2.2.4") - and hparams.get("layer_transformation") == "sparsemax_patch" + parse_version.parse(comet_version) + >= parse_version.parse('2.2.4') + and hparams.get('layer_transformation') == 'sparsemax_patch' ) except: use_softmax = False # Add the override parameter only if needed kwargs = { - "checkpoint_path": checkpoint_path, - "load_pretrained_weights": False, - "hparams_file": hparams_file if reload_hparams else None, - "map_location": torch.device("cpu"), - "strict": strict, - "local_files_only": local_files_only, + 'checkpoint_path': checkpoint_path, + 'load_pretrained_weights': False, + 'hparams_file': hparams_file if reload_hparams else None, + 'map_location': torch.device('cpu'), + 'strict': strict, + 'local_files_only': local_files_only, } if use_softmax: - kwargs["layer_transformation"] = "softmax" + kwargs['layer_transformation'] = 'softmax' model = model_class.load_from_checkpoint(**kwargs) return model else: - raise Exception(f"hparams.yaml file is missing from {parent_folder}!") + raise Exception(f'hparams.yaml file is missing from {parent_folder}!') diff --git a/comet/models/base.py b/comet/models/base.py index 655fcb53..6bc14b37 100644 --- a/comet/models/base.py +++ b/comet/models/base.py @@ -18,6 +18,7 @@ Abstract Model class that implements some of the Pytorch Lightning logic. Extend this class to create new model and metrics within COMET. """ + import abc import logging import os @@ -28,8 +29,12 @@ import numpy as np import pytorch_lightning as ptl import torch -from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, - Subset) +from torch.utils.data import ( + DataLoader, + RandomSampler, + SequentialSampler, + Subset, +) from comet.encoders import str2encoder from comet.modules import LayerwiseAttention @@ -38,11 +43,16 @@ from .pooling_utils import average_pooling, max_pooling from .predict_pbar import PredictProgressBar from .predict_writer import CustomWriter -from .utils import (OrderedSampler, Prediction, Target, flatten_metadata, - restore_list_order) - -if "COMET_EMBEDDINGS_CACHE" in os.environ: - CACHE_SIZE = int(os.environ["COMET_EMBEDDINGS_CACHE"]) +from .utils import ( + OrderedSampler, + Prediction, + Target, + flatten_metadata, + restore_list_order, +) + +if 'COMET_EMBEDDINGS_CACHE' in os.environ: + CACHE_SIZE = int(os.environ['COMET_EMBEDDINGS_CACHE']) else: CACHE_SIZE = 1024 @@ -92,18 +102,18 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1.0e-06, learning_rate: float = 1.5e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-large", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-large', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], @@ -115,11 +125,13 @@ def __init__( super().__init__() self.save_hyperparameters() self.encoder = str2encoder[self.hparams.encoder_model].from_pretrained( - self.hparams.pretrained_model, load_pretrained_weights, local_files_only + self.hparams.pretrained_model, + load_pretrained_weights, + local_files_only, ) self.epoch_nr = 0 - if self.hparams.layer == "mix": + if self.hparams.layer == 'mix': self.layerwise_attention = LayerwiseAttention( layer_transformation=layer_transformation, num_layers=self.encoder.num_layers, @@ -159,7 +171,7 @@ def enable_context(self): """Function that extends COMET to use preceding context as described in https://statmt.org/wmt22/pdf/2022.wmt-1.6.pdf.""" logger.warning( - "Context should only be enabled for RegressionMetric with Average Pooling." + 'Context should only be enabled for RegressionMetric with Average Pooling.' ) @abc.abstractmethod @@ -186,7 +198,7 @@ def read_validation_data(self): def prepare_sample( self, sample: List[dict], - stage: str = "fit", + stage: str = 'fit', *args, **kwargs, ): @@ -225,7 +237,7 @@ def requires_references(self) -> bool: def freeze_encoder(self) -> None: """Deactivates training for encoder model parameters (keeping them frozen)""" - logger.info("Encoder model frozen.") + logger.info('Encoder model frozen.') self.encoder.freeze() @property @@ -233,7 +245,9 @@ def loss(self): """Loss function""" return torch.nn.MSELoss() - def compute_loss(self, prediction: Prediction, target: Target) -> torch.Tensor: + def compute_loss( + self, prediction: Prediction, target: Target + ) -> torch.Tensor: """Computes Loss value between a batch Prediction and respective Target.""" return self.loss(prediction.score, target.score) @@ -241,7 +255,7 @@ def unfreeze_encoder(self) -> None: """Activates fine-tuning of encoder parameters.""" if self._frozen: if self.trainer.is_global_zero: - logger.info("Encoder model fine-tuning") + logger.info('Encoder model fine-tuning') self.encoder.unfreeze() self._frozen = False @@ -327,24 +341,29 @@ def compute_sentence_embedding( ) if self.layerwise_attention: embeddings = self.layerwise_attention( - encoder_out["all_layers"], attention_mask + encoder_out['all_layers'], attention_mask ) - elif self.hparams.layer >= 0 and self.hparams.layer < self.encoder.num_layers: - embeddings = encoder_out["all_layers"][self.hparams.layer] + elif ( + self.hparams.layer >= 0 + and self.hparams.layer < self.encoder.num_layers + ): + embeddings = encoder_out['all_layers'][self.hparams.layer] else: - raise Exception("Invalid model layer {}.".format(self.hparams.layer)) + raise Exception( + 'Invalid model layer {}.'.format(self.hparams.layer) + ) - if self.hparams.pool == "default": - sentemb = encoder_out["sentemb"] + if self.hparams.pool == 'default': + sentemb = encoder_out['sentemb'] - elif self.hparams.pool == "max": + elif self.hparams.pool == 'max': sentemb = max_pooling( input_ids, embeddings, self.encoder.tokenizer.pad_token_id ) - elif self.hparams.pool == "avg": + elif self.hparams.pool == 'avg': sentemb = average_pooling( input_ids, embeddings, @@ -354,11 +373,11 @@ def compute_sentence_embedding( self.use_context, ) - elif self.hparams.pool == "cls": + elif self.hparams.pool == 'cls': sentemb = embeddings[:, 0, :] else: - raise Exception("Invalid pooling technique.") + raise Exception('Invalid pooling technique.') return sentemb @@ -389,7 +408,7 @@ def training_step( self._frozen = False self.log( - "train_loss", + 'train_loss', loss_value, on_step=True, on_epoch=True, @@ -412,13 +431,15 @@ def validation_step( batch_input, batch_target = batch batch_prediction = self.forward(**batch_input) if dataloader_idx == 0: - self.train_metrics.update(batch_prediction.score, batch_target["score"]) + self.train_metrics.update( + batch_prediction.score, batch_target['score'] + ) elif dataloader_idx > 0: self.val_metrics[dataloader_idx - 1].update( batch_prediction.score, - batch_target["score"], - batch_target["system"] if "system" in batch_target else None, + batch_target['score'], + batch_target['system'] if 'system' in batch_target else None, ) def on_predict_start(self) -> None: @@ -450,7 +471,7 @@ def predict_step( mcd_outputs = torch.stack( [self(**batch).score for _ in range(self.mc_dropout)] ) - model_outputs["metadata"] = Prediction( + model_outputs['metadata'] = Prediction( mcd_scores=mcd_outputs.mean(dim=0), mcd_std=mcd_outputs.std(dim=0), ) @@ -469,13 +490,16 @@ def on_validation_epoch_end(self, *args, **kwargs) -> None: self.log_dict(results, prog_bar=False) val_metrics.append(results) - average_results = {"val_" + k.split("_")[-1]: [] for k in val_metrics[0].keys()} + average_results = { + 'val_' + k.split('_')[-1]: [] for k in val_metrics[0].keys() + } for i in range(len(val_metrics)): for k, v in val_metrics[i].items(): - average_results["val_" + k.split("_")[-1]].append(v) + average_results['val_' + k.split('_')[-1]].append(v) self.log_dict( - {k: sum(v) / len(v) for k, v in average_results.items()}, prog_bar=True + {k: sum(v) / len(v) for k, v in average_results.items()}, + prog_bar=True, ) def setup(self, stage: str) -> None: @@ -483,11 +507,12 @@ def setup(self, stage: str) -> None: stage (str): either 'fit', 'validate', 'test', or 'predict' """ - if stage in (None, "fit"): + if stage in (None, 'fit'): train_dataset = self.read_training_data(self.hparams.train_data[0]) self.validation_sets = [ - self.read_validation_data(d) for d in self.hparams.validation_data + self.read_validation_data(d) + for d in self.hparams.validation_data ] self.first_epoch_total_steps = len(train_dataset) // ( @@ -495,7 +520,8 @@ def setup(self, stage: str) -> None: ) # Always validate the model with part of training. train_subset = np.random.choice( - a=len(train_dataset), size=min(1000, int(len(train_dataset) * 0.2)) + a=len(train_dataset), + size=min(1000, int(len(train_dataset) * 0.2)), ) self.train_subset = Subset(train_dataset, train_subset) @@ -508,13 +534,13 @@ def train_dataloader(self) -> DataLoader: self.current_epoch % len(self.hparams.train_data) ] train_dataset = self.read_training_data(data_path) - logger.info(f"Loading {data_path}.") + logger.info(f'Loading {data_path}.') return DataLoader( dataset=train_dataset, sampler=RandomSampler(train_dataset), batch_size=self.hparams.batch_size, - collate_fn=lambda s: self.prepare_sample(s, stage="fit"), + collate_fn=lambda s: self.prepare_sample(s, stage='fit'), num_workers=2 * self.trainer.num_devices, ) @@ -524,7 +550,7 @@ def val_dataloader(self) -> DataLoader: DataLoader( dataset=self.train_subset, batch_size=self.hparams.batch_size, - collate_fn=lambda s: self.prepare_sample(s, stage="validate"), + collate_fn=lambda s: self.prepare_sample(s, stage='validate'), num_workers=2 * self.trainer.num_devices, ) ] @@ -533,7 +559,9 @@ def val_dataloader(self) -> DataLoader: DataLoader( dataset=validation_set, batch_size=self.hparams.batch_size, - collate_fn=lambda s: self.prepare_sample(s, stage="validate"), + collate_fn=lambda s: self.prepare_sample( + s, stage='validate' + ), num_workers=2 * self.trainer.num_devices, ) ) @@ -543,7 +571,7 @@ def prepare_for_inference(self, sample): """This is to avoid having a lamba function inside the predict dataloader `collate_fn=lambda x: self.prepare_sample(x, inference=True)` """ - return self.prepare_sample(sample, stage="predict") + return self.prepare_sample(sample, stage='predict') def predict( self, @@ -553,7 +581,7 @@ def predict( devices: Union[List[int], str, int] = None, mc_dropout: int = 0, progress_bar: bool = True, - accelerator: str = "auto", + accelerator: str = 'auto', num_workers: int = None, length_batching: bool = True, ) -> Prediction: @@ -587,30 +615,34 @@ def predict( if gpus > 0 and devices is not None: assert len(devices) == gpus, AssertionError( - "List of devices must be same size as `gpus` or None if `gpus=0`" + 'List of devices must be same size as `gpus` or None if `gpus=0`' ) elif gpus > 0: devices = gpus else: # gpu = 0 - devices = "auto" + devices = 'auto' sampler = SequentialSampler(samples) if length_batching and gpus < 2: try: - sort_ids = np.argsort([len(sample["src"]) for sample in samples]) + sort_ids = np.argsort( + [len(sample['src']) for sample in samples] + ) except KeyError: - sort_ids = np.argsort([len(sample["ref"]) for sample in samples]) + sort_ids = np.argsort( + [len(sample['ref']) for sample in samples] + ) sampler = OrderedSampler(sort_ids) # On Windows, only num_workers=0 is supported. - is_windows = os.name == "nt" + is_windows = os.name == 'nt' if num_workers is None: # Guideline for workers that typically works well. num_workers = 0 if is_windows else 2 * gpus elif is_windows and num_workers != 0: logger.warning( - "Due to limits of multiprocessing on Windows, it is likely that setting num_workers > 0 will result" - " in scores of 0. It is therefore recommended to set num_workers=0 or leave it to None (default)." + 'Due to limits of multiprocessing on Windows, it is likely that setting num_workers > 0 will result' + ' in scores of 0. It is therefore recommended to set num_workers=0 or leave it to None (default).' ) self.eval() @@ -621,7 +653,7 @@ def predict( collate_fn=self.prepare_for_inference, num_workers=num_workers, multiprocessing_context=( - "fork" if torch.backends.mps.is_available() else None + 'fork' if torch.backends.mps.is_available() else None ), ) if gpus > 1: @@ -639,16 +671,16 @@ def predict( enable_progress_bar = False warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer = ptl.Trainer( devices=devices, logger=False, callbacks=callbacks, - accelerator=accelerator if gpus > 0 else "cpu", - strategy="auto" if gpus < 2 else "ddp", + accelerator=accelerator if gpus > 0 else 'cpu', + strategy='auto' if gpus < 2 else 'ddp', enable_progress_bar=enable_progress_bar, ) return_predictions = False if gpus > 1 else True @@ -669,25 +701,34 @@ def predict( # If we are not in the GLOBAL RANK we will return None exit() - scores = torch.cat([pred["scores"] for pred in predictions], dim=0).tolist() - if "metadata" in predictions[0]: - metadata = flatten_metadata([pred["metadata"] for pred in predictions]) + scores = torch.cat( + [pred['scores'] for pred in predictions], dim=0 + ).tolist() + if 'metadata' in predictions[0]: + metadata = flatten_metadata( + [pred['metadata'] for pred in predictions] + ) else: metadata = [] - output = Prediction(scores=scores, system_score=sum(scores) / len(scores)) + output = Prediction( + scores=scores, system_score=sum(scores) / len(scores) + ) # Restore order of samples! if length_batching and gpus < 2: - output["scores"] = restore_list_order(scores, sort_ids) + output['scores'] = restore_list_order(scores, sort_ids) if metadata: - output["metadata"] = Prediction( - **{k: restore_list_order(v, sort_ids) for k, v in metadata.items()} + output['metadata'] = Prediction( + **{ + k: restore_list_order(v, sort_ids) + for k, v in metadata.items() + } ) return output else: # Add metadata to output if metadata: - output["metadata"] = metadata + output['metadata'] = metadata return output diff --git a/comet/models/download_utils.py b/comet/models/download_utils.py index d0ebad23..45535464 100644 --- a/comet/models/download_utils.py +++ b/comet/models/download_utils.py @@ -29,25 +29,25 @@ available_legacy_metrics = { # WMT20 Models - "emnlp20-comet-rank": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/emnlp20-comet-rank.tar.gz", - "wmt20-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-da.tar.gz", - "wmt20-comet-qe-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da.tar.gz", - "wmt20-comet-qe-da-v2": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da-v2.tar.gz", + 'emnlp20-comet-rank': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/emnlp20-comet-rank.tar.gz', + 'wmt20-comet-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-da.tar.gz', + 'wmt20-comet-qe-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da.tar.gz', + 'wmt20-comet-qe-da-v2': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da-v2.tar.gz', # WMT21 Models - "wmt21-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-da.tar.gz", - "wmt21-comet-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-mqm.tar.gz", - "wmt21-cometinho-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-mqm.tar.gz", - "wmt21-cometinho-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-da.tar.gz", - "wmt21-comet-qe-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-mqm.tar.gz", - "wmt21-comet-qe-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-da.tar.gz", + 'wmt21-comet-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-da.tar.gz', + 'wmt21-comet-mqm': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-mqm.tar.gz', + 'wmt21-cometinho-mqm': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-mqm.tar.gz', + 'wmt21-cometinho-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-da.tar.gz', + 'wmt21-comet-qe-mqm': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-mqm.tar.gz', + 'wmt21-comet-qe-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-da.tar.gz', # EAMT22 Models - "eamt22-cometinho-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-cometinho-da.tar.gz", - "eamt22-prune-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-prune-comet-da.tar.gz", + 'eamt22-cometinho-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-cometinho-da.tar.gz', + 'eamt22-prune-comet-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-prune-comet-da.tar.gz', } def get_cache_folder(): - cache_directory = Path.home() / ".cache" / "torch" / "unbabel_comet" + cache_directory = Path.home() / '.cache' / 'torch' / 'unbabel_comet' if not cache_directory.exists(): cache_directory.mkdir(exist_ok=True, parents=True) @@ -81,7 +81,9 @@ def inner(b: int = 1, bsize: int = 1, tsize: int = None): return inner -def _maybe_extract(compressed_filename: str, directory: str, extension: str = None): +def _maybe_extract( + compressed_filename: str, directory: str, extension: str = None +): """Extract a compressed file to ``directory``. :param compressed_filename: Compressed file. @@ -89,30 +91,32 @@ def _maybe_extract(compressed_filename: str, directory: str, extension: str = No :param extension: Extension of the file; Otherwise, attempts to extract extension from the filename. """ - logger.info("Extracting {}".format(compressed_filename)) + logger.info('Extracting {}'.format(compressed_filename)) if extension is None: basename = os.path.basename(compressed_filename) - extension = basename.split(".", 1)[1] + extension = basename.split('.', 1)[1] - if "zip" in extension: - with zipfile.ZipFile(compressed_filename, "r") as zip_: + if 'zip' in extension: + with zipfile.ZipFile(compressed_filename, 'r') as zip_: zip_.extractall(directory) - elif "tar.gz" in extension or "tgz" in extension: + elif 'tar.gz' in extension or 'tgz' in extension: # `tar` is much faster than python's `tarfile` implementation - with open(os.devnull, "w") as devnull: + with open(os.devnull, 'w') as devnull: subprocess.call( - ["tar", "-C", directory, "-zxvf", compressed_filename], stdout=devnull + ['tar', '-C', directory, '-zxvf', compressed_filename], + stdout=devnull, ) - elif "tar" in extension: - with open(os.devnull, "w") as devnull: + elif 'tar' in extension: + with open(os.devnull, 'w') as devnull: subprocess.call( - ["tar", "-C", directory, "-xvf", compressed_filename], stdout=devnull + ['tar', '-C', directory, '-xvf', compressed_filename], + stdout=devnull, ) - logger.info("Extracted {}".format(compressed_filename)) + logger.info('Extracted {}'.format(compressed_filename)) def _get_filename_from_url(url): @@ -176,18 +180,20 @@ def download_file_maybe_extract( if not os.path.isdir(directory): os.makedirs(directory) - logger.info("Downloading {}".format(filename)) + logger.info('Downloading {}'.format(filename)) # Download - with tqdm(unit="B", unit_scale=True, miniters=1, desc=filename) as t: - urllib.request.urlretrieve(url, filename=filepath, reporthook=_reporthook(t)) + with tqdm(unit='B', unit_scale=True, miniters=1, desc=filename) as t: + urllib.request.urlretrieve( + url, filename=filepath, reporthook=_reporthook(t) + ) _maybe_extract( compressed_filename=filepath, directory=directory, extension=extension ) if not _check_download(*check_files): - raise ValueError("[DOWNLOAD FAILED] `*check_files` not found") + raise ValueError('[DOWNLOAD FAILED] `*check_files` not found') return filepath @@ -207,42 +213,44 @@ def download_model_legacy(model: str, saving_directory: str = None) -> str: if saving_directory is None: saving_directory = get_cache_folder() - if not saving_directory.endswith("/"): - saving_directory += "/" + if not saving_directory.endswith('/'): + saving_directory += '/' if not os.path.exists(saving_directory): os.makedirs(saving_directory) if os.path.isdir(saving_directory + model): - logger.info(f"{model} is already in cache.") - if not model.endswith("/"): - model += "/" + logger.info(f'{model} is already in cache.') + if not model.endswith('/'): + model += '/' elif model not in available_legacy_metrics.keys(): raise Exception( - f"{model} is not in the `available_legacy_metrics` or is a valid checkpoint folder." + f'{model} is not in the `available_legacy_metrics` or is a valid checkpoint folder.' ) - elif available_legacy_metrics[model].startswith("https://"): + elif available_legacy_metrics[model].startswith('https://'): download_file_maybe_extract( available_legacy_metrics[model], directory=saving_directory ) else: - raise Exception("Invalid model name!") + raise Exception('Invalid model name!') # CLEAN Cache - if os.path.exists(saving_directory + model + ".zip"): - os.remove(saving_directory + model + ".zip") - if os.path.exists(saving_directory + model + ".tar.gz"): - os.remove(saving_directory + model + ".tar.gz") - if os.path.exists(saving_directory + model + ".tar"): - os.remove(saving_directory + model + ".tar") - - checkpoints_folder = saving_directory + model + "/checkpoints" + if os.path.exists(saving_directory + model + '.zip'): + os.remove(saving_directory + model + '.zip') + if os.path.exists(saving_directory + model + '.tar.gz'): + os.remove(saving_directory + model + '.tar.gz') + if os.path.exists(saving_directory + model + '.tar'): + os.remove(saving_directory + model + '.tar') + + checkpoints_folder = saving_directory + model + '/checkpoints' checkpoints = [ - file for file in os.listdir(checkpoints_folder) if file.endswith(".ckpt") + file + for file in os.listdir(checkpoints_folder) + if file.endswith('.ckpt') ] checkpoint = checkpoints[-1] - checkpoint_path = checkpoints_folder + "/" + checkpoint + checkpoint_path = checkpoints_folder + '/' + checkpoint return checkpoint_path diff --git a/comet/models/lru_cache.py b/comet/models/lru_cache.py index 6b876c08..33cec77a 100644 --- a/comet/models/lru_cache.py +++ b/comet/models/lru_cache.py @@ -21,6 +21,7 @@ Our modification modifies the _make_key function to use tensor str representation intead of the object reference. Other than that we use the original implementation """ + from _thread import RLock from functools import _CacheInfo, _HashedSeq, update_wrapper @@ -52,10 +53,10 @@ def _make_key( for x in args: if torch.is_tensor(x): if len(x.size()) == 0: - raise Exception("Tensor needs to be at least 1-Dimensional.") + raise Exception('Tensor needs to be at least 1-Dimensional.') if len(x.size()) == 1: - new_args.append("\n".join([repr(x), repr(x.shape)])) + new_args.append('\n'.join([repr(x), repr(x.shape)])) else: new_args.append( # HACK: Tensor representations omit some tensor content. @@ -63,9 +64,9 @@ def _make_key( # The current solution is an approximation to the actual tensor # full representation. This can still lead to `false` cache hits! x.__repr__() - + "\n" + + '\n' + x.diagonal().__repr__() - + "\n" + + '\n' + x.shape.__repr__() ) else: @@ -99,14 +100,16 @@ def tensor_lru_cache(maxsize=128, typed=False): # The user_function was passed in directly via the maxsize argument user_function, maxsize = maxsize, 128 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) - wrapper.cache_parameters = lambda: {"maxsize": maxsize, "typed": typed} + wrapper.cache_parameters = lambda: {'maxsize': maxsize, 'typed': typed} return update_wrapper(wrapper, user_function) elif maxsize is not None: - raise TypeError("Expected first argument to be an integer, a callable, or None") + raise TypeError( + 'Expected first argument to be an integer, a callable, or None' + ) def decorating_function(user_function): wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) - wrapper.cache_parameters = lambda: {"maxsize": maxsize, "typed": typed} + wrapper.cache_parameters = lambda: {'maxsize': maxsize, 'typed': typed} return update_wrapper(wrapper, user_function) return decorating_function diff --git a/comet/models/metrics.py b/comet/models/metrics.py index ae047f0f..e29cb339 100644 --- a/comet/models/metrics.py +++ b/comet/models/metrics.py @@ -19,6 +19,7 @@ Regression and Ranking metrics to be used during training to measure correlations with human judgements """ + from itertools import combinations from typing import Any, Callable, List, Optional @@ -29,7 +30,9 @@ from torchmetrics.classification import MulticlassMatthewsCorrCoef -def system_accuracy(y_hat: List[float], y: List[float], system: List[str]) -> float: +def system_accuracy( + y_hat: List[float], y: List[float], system: List[str] +) -> float: """Implementation of system-level accuracy proposed in [To Ship not to Ship](https://aclanthology.org/2021.wmt-1.57/) @@ -42,23 +45,23 @@ def system_accuracy(y_hat: List[float], y: List[float], system: List[str]) -> fl Float: System-level accuracy. """ try: - data = pd.DataFrame({"y_hat": y_hat, "y": y, "system": system}) + data = pd.DataFrame({'y_hat': y_hat, 'y': y, 'system': system}) except ValueError: raise Exception( - "The program will be interrupted, followed by a series of errors." + 'The program will be interrupted, followed by a series of errors.' " This probably happens because you're using ddp strategy in the" - " trainer config. System accuracy computation does not currently" - " work with ddp. Please make sure your VALIDATION data DOES NOT" + ' trainer config. System accuracy computation does not currently' + ' work with ddp. Please make sure your VALIDATION data DOES NOT' " include a 'system' column, and try again." ) - data = data.groupby("system").mean() + data = data.groupby('system').mean() pairs = list(combinations(data.index.tolist(), 2)) tp = 0 for system_a, system_b in pairs: - human_delta = data.loc[system_a]["y"] - data.loc[system_b]["y"] - model_delta = data.loc[system_a]["y_hat"] - data.loc[system_b]["y_hat"] + human_delta = data.loc[system_a]['y'] - data.loc[system_b]['y'] + model_delta = data.loc[system_a]['y_hat'] - data.loc[system_b]['y_hat'] if (human_delta >= 0) ^ (model_delta < 0): tp += 1 @@ -67,14 +70,14 @@ def system_accuracy(y_hat: List[float], y: List[float], system: List[str]) -> fl class MCCMetric(MulticlassMatthewsCorrCoef): - def __init__(self, prefix: str = "", **kwargs) -> None: + def __init__(self, prefix: str = '', **kwargs) -> None: super().__init__(**kwargs) self.prefix = prefix def compute(self) -> torch.Tensor: """Computes matthews correlation coefficient.""" mcc = super(MCCMetric, self).compute() - return {self.prefix + "_mcc": mcc} + return {self.prefix + '_mcc': mcc} class RegressionMetrics(Metric): @@ -86,7 +89,7 @@ class RegressionMetrics(Metric): def __init__( self, - prefix: str = "", + prefix: str = '', dist_sync_on_step: bool = False, process_group: Optional[Any] = None, dist_sync_fn: Optional[Callable] = None, @@ -96,9 +99,9 @@ def __init__( process_group=process_group, dist_sync_fn=dist_sync_fn, ) - self.add_state("preds", default=[], dist_reduce_fx="cat") - self.add_state("target", default=[], dist_reduce_fx="cat") - self.add_state("systems", default=[], dist_reduce_fx=None) + self.add_state('preds', default=[], dist_reduce_fx='cat') + self.add_state('target', default=[], dist_reduce_fx='cat') + self.add_state('systems', default=[], dist_reduce_fx=None) self.prefix = prefix def update( @@ -131,16 +134,16 @@ def compute(self) -> torch.Tensor: spearman, _ = stats.spearmanr(preds.tolist(), target.tolist()) pearson, _ = stats.pearsonr(preds.tolist(), target.tolist()) report = { - self.prefix + "_kendall": kendall, - self.prefix + "_spearman": spearman, - self.prefix + "_pearson": pearson, + self.prefix + '_kendall': kendall, + self.prefix + '_spearman': spearman, + self.prefix + '_pearson': pearson, } if len(self.systems) > 0: system_acc = system_accuracy( preds.cpu().tolist(), target.cpu().tolist(), self.systems ) - report["system_acc"] = system_acc + report['system_acc'] = system_acc return report @@ -150,7 +153,7 @@ class WMTKendall(Metric): def __init__( self, - prefix: str = "", + prefix: str = '', dist_sync_on_step: bool = False, process_group: Optional[Any] = None, dist_sync_fn: Optional[Callable] = None, @@ -160,8 +163,12 @@ def __init__( process_group=process_group, dist_sync_fn=dist_sync_fn, ) - self.add_state("concordance", default=torch.tensor(0), dist_reduce_fx="sum") - self.add_state("discordance", default=torch.tensor(0), dist_reduce_fx="sum") + self.add_state( + 'concordance', default=torch.tensor(0), dist_reduce_fx='sum' + ) + self.add_state( + 'discordance', default=torch.tensor(0), dist_reduce_fx='sum' + ) self.prefix = prefix def update(self, distance_pos: torch.Tensor, distance_neg: torch.Tensor): @@ -176,6 +183,6 @@ def update(self, distance_pos: torch.Tensor, distance_neg: torch.Tensor): def compute(self): return { self.prefix - + "_kendall": (self.concordance - self.discordance) + + '_kendall': (self.concordance - self.discordance) / (self.concordance + self.discordance) } diff --git a/comet/models/multitask/unified_metric.py b/comet/models/multitask/unified_metric.py index 911e66eb..edaf90b6 100644 --- a/comet/models/multitask/unified_metric.py +++ b/comet/models/multitask/unified_metric.py @@ -22,14 +22,17 @@ Inspired on [UniTE](https://arxiv.org/pdf/2204.13346.pdf) """ + from collections import OrderedDict from typing import Dict, List, Optional, Tuple, Union import pandas as pd import torch from torch import nn -from transformers.optimization import (Adafactor, - get_constant_schedule_with_warmup) +from transformers.optimization import ( + Adafactor, + get_constant_schedule_with_warmup, +) from comet.models.base import CometModel from comet.models.metrics import MCCMetric, RegressionMetrics @@ -96,29 +99,29 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.9, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 3.0e-06, learning_rate: float = 3.0e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "microsoft/infoxlm-large", - sent_layer: Union[str, int] = "mix", - layer_transformation: str = "sparsemax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'microsoft/infoxlm-large', + sent_layer: Union[str, int] = 'mix', + layer_transformation: str = 'sparsemax', layer_norm: bool = True, word_layer: int = 24, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [3072, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, - input_segments: List[str] = ["mt", "src", "ref"], + input_segments: List[str] = ['mt', 'src', 'ref'], word_level_training: bool = False, loss_lambda: float = 0.65, - error_labels: List[str] = ["minor", "major"], + error_labels: List[str] = ['minor', 'major'], cross_entropy_weights: Optional[List[float]] = None, load_pretrained_weights: bool = True, local_files_only: bool = False, @@ -141,7 +144,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="unified_metric", + class_identifier='unified_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -156,7 +159,9 @@ def __init__( self.word_level = word_level_training if word_level_training: self.encoder.labelset = self.label_encoder - self.hidden2tag = nn.Linear(self.encoder.output_units, self.num_classes) + self.hidden2tag = nn.Linear( + self.encoder.output_units, self.num_classes + ) if len(self.hparams.input_segments) == 3: # By default 3rd input [mt:src:ref] has 50% weight, @@ -191,7 +196,7 @@ def set_decoding_threshold(self, threshold: float = 0.5): def init_metrics(self): """Initializes training and validation metrics""" # Train and Dev correlation metrics - self.train_corr = RegressionMetrics(prefix="train") + self.train_corr = RegressionMetrics(prefix='train') self.val_corr = nn.ModuleList( [RegressionMetrics(prefix=d) for d in self.hparams.validation_data] ) @@ -199,7 +204,9 @@ def init_metrics(self): self.label_encoder = LabelSet(self.hparams.error_labels) self.num_classes = len(self.label_encoder.labels_to_id) # Train and Dev MCC - self.train_mcc = MCCMetric(num_classes=self.num_classes, prefix="train") + self.train_mcc = MCCMetric( + num_classes=self.num_classes, prefix='train' + ) self.val_mcc = nn.ModuleList( [ MCCMetric(num_classes=self.num_classes, prefix=d) @@ -212,13 +219,15 @@ def init_losses(self) -> None: self.sentloss = nn.MSELoss() if self.word_level: if self.hparams.cross_entropy_weights: - assert len(self.hparams.cross_entropy_weights) == self.num_classes + assert ( + len(self.hparams.cross_entropy_weights) == self.num_classes + ) loss_weights = torch.tensor(self.hparams.cross_entropy_weights) else: loss_weights = None self.wordloss = nn.CrossEntropyLoss( - reduction="mean", ignore_index=-1, weight=loss_weights + reduction='mean', ignore_index=-1, weight=loss_weights ) def requires_references(self) -> bool: @@ -229,13 +238,15 @@ def requires_references(self) -> bool: Return: [bool]: True if the model was trained to work exclusively with references. """ - if self.hparams.input_segments == ["mt", "ref"]: + if self.hparams.input_segments == ['mt', 'ref']: return True return False def configure_optimizers( self, - ) -> Tuple[List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR]]: + ) -> Tuple[ + List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR] + ]: """Pytorch Lightning method to initialize a training Optimizer and learning rate scheduler. @@ -247,25 +258,28 @@ def configure_optimizers( self.hparams.encoder_learning_rate, self.hparams.layerwise_decay ) params += [ - {"params": self.estimator.parameters(), "lr": self.hparams.learning_rate} + { + 'params': self.estimator.parameters(), + 'lr': self.hparams.learning_rate, + } ] if self.word_level: params += [ { - "params": self.hidden2tag.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.hidden2tag.parameters(), + 'lr': self.hparams.learning_rate, }, ] if self.layerwise_attention: params += [ { - "params": self.layerwise_attention.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.layerwise_attention.parameters(), + 'lr': self.hparams.learning_rate, } ] - if self.hparams.optimizer == "Adafactor": + if self.hparams.optimizer == 'Adafactor': optimizer = Adafactor( params, lr=self.hparams.learning_rate, @@ -300,10 +314,10 @@ def read_training_data(self, path: str) -> List[dict]: # Make sure everything except score is str type for col in columns: df[col] = df[col].astype(str) - columns.append("score") - df["score"] = df["score"].astype("float16") + columns.append('score') + df['score'] = df['score'].astype('float16') df = df[columns] - return df.to_dict("records") + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Reads a csv file with validation data. @@ -318,15 +332,15 @@ def read_validation_data(self, path: str) -> List[dict]: # Deep copy input segments columns = self.hparams.input_segments[:] # If system in columns we will use this to calculate system-level accuracy - if "system" in df.columns: - columns.append("system") + if 'system' in df.columns: + columns.append('system') # Make sure everything except score is str type for col in columns: df[col] = df[col].astype(str) - columns.append("score") - df["score"] = df["score"].astype("float16") + columns.append('score') + df['score'] = df['score'].astype('float16') df = df[columns] - return df.to_dict("records") + return df.to_dict('records') def concat_inputs( self, @@ -359,23 +373,27 @@ def concat_inputs( full_input, _, _ = self.encoder.concat_sequences( input_sequences, return_label_ids=self.word_level ) - model_inputs["inputs"] = (src_input, ref_input, full_input) - model_inputs["mt_length"] = input_sequences[0]["attention_mask"].sum(dim=1) + model_inputs['inputs'] = (src_input, ref_input, full_input) + model_inputs['mt_length'] = input_sequences[0][ + 'attention_mask' + ].sum(dim=1) return model_inputs # Otherwise we will have one single input sequence that concatenates the MT # with SRC/REF. else: - model_inputs["inputs"] = ( + model_inputs['inputs'] = ( self.encoder.concat_sequences( input_sequences, return_label_ids=self.word_level )[0], ) - model_inputs["mt_length"] = input_sequences[0]["attention_mask"].sum(dim=1) + model_inputs['mt_length'] = input_sequences[0][ + 'attention_mask' + ].sum(dim=1) return model_inputs def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "fit" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'fit' ) -> Union[Tuple[Dict[str, torch.Tensor]], Dict[str, torch.Tensor]]: """Tokenizes input data and prepares targets for training. @@ -389,45 +407,47 @@ def prepare_sample( """ inputs = {k: [d[k] for d in sample] for k in sample[0]} input_sequences = [ - self.encoder.prepare_sample(inputs["mt"], self.word_level, None), + self.encoder.prepare_sample(inputs['mt'], self.word_level, None), ] src_input, ref_input = False, False - if ("src" in inputs) and ("src" in self.hparams.input_segments): - input_sequences.append(self.encoder.prepare_sample(inputs["src"])) + if ('src' in inputs) and ('src' in self.hparams.input_segments): + input_sequences.append(self.encoder.prepare_sample(inputs['src'])) src_input = True - if ("ref" in inputs) and ("ref" in self.hparams.input_segments): - input_sequences.append(self.encoder.prepare_sample(inputs["ref"])) + if ('ref' in inputs) and ('ref' in self.hparams.input_segments): + input_sequences.append(self.encoder.prepare_sample(inputs['ref'])) ref_input = True unified_input = src_input and ref_input model_inputs = self.concat_inputs(input_sequences, unified_input) - if stage == "predict": - return model_inputs["inputs"] + if stage == 'predict': + return model_inputs['inputs'] - scores = [float(s) for s in inputs["score"]] + scores = [float(s) for s in inputs['score']] targets = Target(score=torch.tensor(scores, dtype=torch.float)) - if "system" in inputs: - targets["system"] = inputs["system"] + if 'system' in inputs: + targets['system'] = inputs['system'] if self.word_level: # Labels will be the same accross all inputs because we are only # doing sequence tagging on the MT. We will only use the mask corresponding # to the MT segment. - seq_len = model_inputs["mt_length"].max() - targets["mt_length"] = model_inputs["mt_length"] - targets["labels"] = model_inputs["inputs"][0]["label_ids"][:, :seq_len] + seq_len = model_inputs['mt_length'].max() + targets['mt_length'] = model_inputs['mt_length'] + targets['labels'] = model_inputs['inputs'][0]['label_ids'][ + :, :seq_len + ] - return model_inputs["inputs"], targets + return model_inputs['inputs'], targets def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor, token_type_ids: Optional[torch.Tensor] = None, - **kwargs + **kwargs, ) -> Dict[str, torch.Tensor]: """Forward function. @@ -455,36 +475,44 @@ def forward( isinstance(self.hparams.word_layer, int) and 0 <= self.hparams.word_layer < self.encoder.num_layers ): - wordemb = encoder_out["all_layers"][self.hparams.word_layer] + wordemb = encoder_out['all_layers'][self.hparams.word_layer] else: raise Exception( - "Invalid model word layer {}.".format(self.hparams.word_layer) + 'Invalid model word layer {}.'.format( + self.hparams.word_layer + ) ) # embeddings used for the sentence-level regression task if self.layerwise_attention: embeddings = self.layerwise_attention( - encoder_out["all_layers"], attention_mask + encoder_out['all_layers'], attention_mask ) elif ( isinstance(self.hparams.sent_layer, int) and 0 <= self.hparams.sent_layer < self.encoder.num_layers ): - embeddings = encoder_out["all_layers"][self.hparams.sent_layer] + embeddings = encoder_out['all_layers'][self.hparams.sent_layer] else: raise Exception( - "Invalid model sent layer {}.".format(self.hparams.word_layer) + 'Invalid model sent layer {}.'.format(self.hparams.word_layer) ) - sentemb = embeddings[:, 0, :] # We take the CLS token as sentence-embedding + sentemb = embeddings[ + :, 0, : + ] # We take the CLS token as sentence-embedding if self.word_level: sentence_output = self.estimator(sentemb) word_output = self.hidden2tag(wordemb) - return Prediction(score=sentence_output.view(-1), logits=word_output) + return Prediction( + score=sentence_output.view(-1), logits=word_output + ) return Prediction(score=self.estimator(sentemb).view(-1)) - def compute_loss(self, prediction: Prediction, target: Target) -> torch.Tensor: + def compute_loss( + self, prediction: Prediction, target: Target + ) -> torch.Tensor: """Receives model batch prediction and respective targets and computes a loss value @@ -500,9 +528,9 @@ def compute_loss(self, prediction: Prediction, target: Target) -> torch.Tensor: predictions = prediction.logits.reshape(-1, self.num_classes) targets = target.labels.reshape(-1).type(torch.LongTensor).cuda() word_loss = self.wordloss(predictions, targets) - return sentence_loss * (1 - self.hparams.loss_lambda) + word_loss * ( - self.hparams.loss_lambda - ) + return sentence_loss * ( + 1 - self.hparams.loss_lambda + ) + word_loss * (self.hparams.loss_lambda) else: return sentence_loss @@ -539,7 +567,7 @@ def training_step( self._frozen = False self.log( - "train_loss", + 'train_loss', loss_value, on_step=True, on_epoch=True, @@ -549,7 +577,10 @@ def training_step( return loss_value def validation_step( - self, batch: Tuple[Dict[str, torch.Tensor]], batch_nb: int, dataloader_idx: int + self, + batch: Tuple[Dict[str, torch.Tensor]], + batch_nb: int, + dataloader_idx: int, ) -> None: """Pytorch Lightning validation_step. @@ -562,7 +593,9 @@ def validation_step( batch_input, batch_target = batch predictions = [self.forward(**input_seq) for input_seq in batch_input] # Final score is the average of the 3 scores when using references. - scores = torch.stack([pred.score for pred in predictions], dim=0).mean(dim=0) + scores = torch.stack([pred.score for pred in predictions], dim=0).mean( + dim=0 + ) if self.word_level: seq_len = batch_target.mt_length.max() # Final probs for each word is the average of the 3 forward passes. @@ -587,7 +620,7 @@ def validation_step( self.val_corr[dataloader_idx - 1].update( scores, batch_target.score, - batch_target["system"] if "system" in batch_target else None, + batch_target['system'] if 'system' in batch_target else None, ) if self.word_level: self.val_mcc[dataloader_idx - 1].update(probs, targets) @@ -599,7 +632,9 @@ def on_validation_epoch_end(self, *args, **kwargs) -> None: self.train_corr.reset() if self.word_level: - self.log_dict(self.train_mcc.compute(), prog_bar=False, sync_dist=True) + self.log_dict( + self.train_mcc.compute(), prog_bar=False, sync_dist=True + ) self.train_mcc.reset() val_metrics = [] @@ -617,10 +652,12 @@ def on_validation_epoch_end(self, *args, **kwargs) -> None: self.log_dict(results, prog_bar=False, sync_dist=True) val_metrics.append(results) - average_results = {"val_" + k.split("_")[-1]: [] for k in val_metrics[0].keys()} + average_results = { + 'val_' + k.split('_')[-1]: [] for k in val_metrics[0].keys() + } for i in range(len(val_metrics)): for k, v in val_metrics[i].items(): - average_results["val_" + k.split("_")[-1]].append(v) + average_results['val_' + k.split('_')[-1]].append(v) self.log_dict( {k: sum(v) / len(v) for k, v in average_results.items()}, @@ -634,7 +671,7 @@ def set_mc_dropout(self, value: int): Args: value (int): number of runs per sample. """ - raise NotImplementedError("MCD not implemented for this model!") + raise NotImplementedError('MCD not implemented for this model!') def decode( self, @@ -658,7 +695,9 @@ def decode( seq_len = len(mt_offsets[i]) error_spans, in_span, span = [], False, {} for token_id, probs, token_offset in zip( - input_ids[i, :seq_len], subword_probs[i][:seq_len], mt_offsets[i] + input_ids[i, :seq_len], + subword_probs[i][:seq_len], + mt_offsets[i], ): if self.decoding_threshold: if torch.sum(probs[1:]) > self.decoding_threshold: @@ -682,26 +721,26 @@ def decode( # Label set: # O I-minor I-major # Begin of annotation span - if label.startswith("I") and not in_span: + if label.startswith('I') and not in_span: in_span = True - span["tokens"] = [ + span['tokens'] = [ token_id, ] - span["severity"] = label.split("-")[1] - span["offset"] = list(token_offset) - span["confidence"] = [ + span['severity'] = label.split('-')[1] + span['offset'] = list(token_offset) + span['confidence'] = [ probability, ] # Inside an annotation span - elif label.startswith("I") and in_span: - span["tokens"].append(token_id) - span["confidence"].append(probability) + elif label.startswith('I') and in_span: + span['tokens'].append(token_id) + span['confidence'].append(probability) # Update offset end - span["offset"][1] = token_offset[1] + span['offset'][1] = token_offset[1] # annotation span finished. - elif label == "O" and in_span: + elif label == 'O' and in_span: error_spans.append(span) in_span, span = False, {} @@ -709,11 +748,13 @@ def decode( for span in error_spans: sentence_output.append( { - "text": self.encoder.tokenizer.decode(span["tokens"]), - "confidence": torch.concat(span["confidence"]).mean().item(), - "severity": span["severity"], - "start": span["offset"][0], - "end": span["offset"][1], + 'text': self.encoder.tokenizer.decode(span['tokens']), + 'confidence': torch.concat(span['confidence']) + .mean() + .item(), + 'severity': span['severity'], + 'start': span['offset'][0], + 'end': span['offset'][1], } ) decoded_output.append(sentence_output) @@ -740,9 +781,9 @@ def predict_step( if len(batch) == 3: predictions = [self.forward(**input_seq) for input_seq in batch] # Final score is the average of the 3 scores! - avg_scores = torch.stack([pred.score for pred in predictions], dim=0).mean( - dim=0 - ) + avg_scores = torch.stack( + [pred.score for pred in predictions], dim=0 + ).mean(dim=0) batch_prediction = Prediction( scores=avg_scores, metadata=Prediction( @@ -752,7 +793,7 @@ def predict_step( ), ) if self.word_level: - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() subword_probs = [ @@ -761,22 +802,26 @@ def predict_step( ] subword_probs = torch.sum(torch.stack(subword_probs), dim=0) error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, + batch[0]['input_ids'], + batch[0]['mt_offsets'], ) - batch_prediction.metadata["error_spans"] = error_spans + batch_prediction.metadata['error_spans'] = error_spans else: model_output = self.forward(**batch[0]) batch_prediction = Prediction(scores=model_output.score) if self.word_level: - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() - subword_probs = nn.functional.softmax(model_output.logits, dim=2)[ - :, :seq_len, : - ] + subword_probs = nn.functional.softmax( + model_output.logits, dim=2 + )[:, :seq_len, :] error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, + batch[0]['input_ids'], + batch[0]['mt_offsets'], ) batch_prediction = Prediction( scores=model_output.score, diff --git a/comet/models/multitask/xcomet_metric.py b/comet/models/multitask/xcomet_metric.py index 76ccf883..59f7eadf 100644 --- a/comet/models/multitask/xcomet_metric.py +++ b/comet/models/multitask/xcomet_metric.py @@ -43,29 +43,34 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1.0e-06, learning_rate: float = 3.66e-06, layerwise_decay: float = 0.983, - encoder_model: str = "XLM-RoBERTa-XL", - pretrained_model: str = "facebook/xlm-roberta-xl", - sent_layer: Union[str, int] = "mix", - layer_transformation: str = "sparsemax", + encoder_model: str = 'XLM-RoBERTa-XL', + pretrained_model: str = 'facebook/xlm-roberta-xl', + sent_layer: Union[str, int] = 'mix', + layer_transformation: str = 'sparsemax', layer_norm: bool = False, word_layer: int = 36, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [2560, 1280], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, word_level_training: bool = True, - error_labels: List[str] = ["minor", "major", "critical"], + error_labels: List[str] = ['minor', 'major', 'critical'], loss_lambda: float = 0.055, - cross_entropy_weights: Optional[List[float]] = [0.08, 0.486, 0.505, 0.533], + cross_entropy_weights: Optional[List[float]] = [ + 0.08, + 0.486, + 0.505, + 0.533, + ], load_pretrained_weights: bool = True, local_files_only: bool = False, ) -> None: @@ -87,7 +92,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="xcomet_metric", + class_identifier='xcomet_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -98,8 +103,8 @@ def __init__( dropout=self.hparams.dropout, final_activation=self.hparams.final_activation, ) - assert error_labels == ["minor", "major", "critical"] - self.hparams.input_segments = ["mt", "src", "ref"] + assert error_labels == ['minor', 'major', 'critical'] + self.hparams.input_segments = ['mt', 'src', 'ref'] self.word_level = True self.encoder.labelset = self.label_encoder self.hidden2tag = nn.Linear(self.encoder.output_units, self.num_classes) @@ -145,11 +150,11 @@ def _compute_mqm_from_spans(error_spans): for sentence_spans in error_spans: sentence_score = 0 for annotation in sentence_spans: - if annotation["severity"] == "minor": + if annotation['severity'] == 'minor': sentence_score += 1 - elif annotation["severity"] == "major": + elif annotation['severity'] == 'major': sentence_score += 5 - elif annotation["severity"] == "critical": + elif annotation['severity'] == 'critical': sentence_score += 10 if sentence_score > 25: @@ -172,7 +177,7 @@ def _compute_mqm_from_spans(error_spans): ], dim=0, ).sum(dim=0) - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() @@ -183,12 +188,13 @@ def _compute_mqm_from_spans(error_spans): ] subword_probs = torch.sum(torch.stack(subword_probs), dim=0) error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, batch[0]['input_ids'], batch[0]['mt_offsets'] ) mqm_scores = _compute_mqm_from_spans(error_spans) final_scores = ( regression_scores - + mqm_scores.to(regression_scores.device) * self.score_weights[3] + + mqm_scores.to(regression_scores.device) + * self.score_weights[3] ) batch_prediction = Prediction( scores=final_scores, @@ -207,14 +213,14 @@ def _compute_mqm_from_spans(error_spans): regression_score = torch.where( model_output.score > 1.0, 1.0, model_output.score ) - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() subword_probs = nn.functional.softmax(model_output.logits, dim=2)[ :, :seq_len, : ] error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, batch[0]['input_ids'], batch[0]['mt_offsets'] ) mqm_scores = _compute_mqm_from_spans(error_spans) final_scores = ( diff --git a/comet/models/pooling_utils.py b/comet/models/pooling_utils.py index 98fc5328..e57a332b 100644 --- a/comet/models/pooling_utils.py +++ b/comet/models/pooling_utils.py @@ -70,9 +70,13 @@ def average_pooling( start_inds, ctx_mask = find_start_inds_and_mask_tokens( mask, tokens, separator_index ) - wordemb = mask_fill_index(0.0, tokens, embeddings, start_inds, padding_index) + wordemb = mask_fill_index( + 0.0, tokens, embeddings, start_inds, padding_index + ) sentemb = torch.sum(wordemb, 1) - sum_mask = ctx_mask.unsqueeze(-1).expand(embeddings.size()).float().sum(1) + sum_mask = ( + ctx_mask.unsqueeze(-1).expand(embeddings.size()).float().sum(1) + ) else: wordemb = mask_fill(0.0, tokens, embeddings, padding_index) sentemb = torch.sum(wordemb, 1) @@ -94,7 +98,9 @@ def max_pooling( Return: torch.Tensor: Sentence embedding """ - return mask_fill(float("-inf"), tokens, embeddings, padding_index).max(dim=1)[0] + return mask_fill(float('-inf'), tokens, embeddings, padding_index).max( + dim=1 + )[0] # From https://github.com/amazon-science/doc-mt-metrics/blob/5385cc28930aae9924edcb3201645dd3810b12c0/COMET/comet/models/pooling_utils.py#L18 @@ -125,7 +131,11 @@ def mask_fill_index( for i, start in enumerate(start_inds): padding_maks2[i, 1 : start + 1] = True padding_mask = torch.logical_or(padding_mask, padding_maks2.unsqueeze(-1)) - return embeddings.float().masked_fill_(padding_mask, fill_value).type_as(embeddings) + return ( + embeddings.float() + .masked_fill_(padding_mask, fill_value) + .type_as(embeddings) + ) def mask_fill( @@ -147,4 +157,8 @@ def mask_fill( torch.Tensor: Word embeddings [batch_size x seq_length x hidden_size] """ padding_mask = tokens.eq(padding_index).unsqueeze(-1) - return embeddings.float().masked_fill_(padding_mask, fill_value).type_as(embeddings) + return ( + embeddings.float() + .masked_fill_(padding_mask, fill_value) + .type_as(embeddings) + ) diff --git a/comet/models/predict_pbar.py b/comet/models/predict_pbar.py index 30bd0554..7218c17e 100644 --- a/comet/models/predict_pbar.py +++ b/comet/models/predict_pbar.py @@ -23,7 +23,7 @@ class PredictProgressBar(ptl.callbacks.progress.tqdm_progress.TQDMProgressBar): def init_predict_tqdm(self) -> tqdm: bar = tqdm( - desc="Predicting", + desc='Predicting', position=(2 * self.process_position), disable=self.is_disabled, leave=True, diff --git a/comet/models/predict_writer.py b/comet/models/predict_writer.py index 7d95989d..66079864 100644 --- a/comet/models/predict_writer.py +++ b/comet/models/predict_writer.py @@ -33,10 +33,12 @@ class CustomWriter(BasePredictionWriter): write_interval (str): When to perform write operations. Defaults to 'epoch' """ - def __init__(self, write_interval="epoch") -> None: + def __init__(self, write_interval='epoch') -> None: super().__init__(write_interval) - def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): + def write_on_epoch_end( + self, trainer, pl_module, predictions, batch_indices + ): """Saves predictions after running inference on all samples.""" # We need to save predictions in the most secure manner possible to avoid @@ -48,7 +50,7 @@ def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): tempfile.mkdtemp(), ] logger.info( - "Created temporary folder to store predictions: {}.".format( + 'Created temporary folder to store predictions: {}.'.format( output_dir[0] ) ) @@ -67,13 +69,16 @@ def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): # this will create N (num processes) files in `output_dir` each containing # the predictions of it's respective rank torch.save( - predictions, os.path.join(self.output_dir, f"pred_{trainer.global_rank}.pt") + predictions, + os.path.join(self.output_dir, f'pred_{trainer.global_rank}.pt'), ) # optionally, you can also save `batch_indices` to get the information about # the data index from your prediction data torch.save( batch_indices, - os.path.join(self.output_dir, f"batch_indices_{trainer.global_rank}.pt"), + os.path.join( + self.output_dir, f'batch_indices_{trainer.global_rank}.pt' + ), ) def gather_all_predictions(self): @@ -86,10 +91,12 @@ def flatten(list): def flatten_predictions(predictions): flatten_pred = Prediction( - scores=torch.cat([pred["scores"] for pred in predictions], dim=0) + scores=torch.cat( + [pred['scores'] for pred in predictions], dim=0 + ) ) - if "metadata" in predictions[0]: - flatten_pred["metadata"] = flatten_metadata( + if 'metadata' in predictions[0]: + flatten_pred['metadata'] = flatten_metadata( [pred.metadata for pred in predictions] ) return flatten_pred @@ -97,29 +104,34 @@ def flatten_predictions(predictions): files = sorted(os.listdir(self.output_dir)) pred = flatten_predictions( [ - flatten_predictions(torch.load(os.path.join(self.output_dir, f))) + flatten_predictions( + torch.load(os.path.join(self.output_dir, f)) + ) for f in files - if "pred" in f + if 'pred' in f ] ) indices = flatten( [ flatten(torch.load(os.path.join(self.output_dir, f))[0]) for f in files - if "batch_indices" in f + if 'batch_indices' in f ] ) output = Prediction( scores=restore_list_order(pred.scores.tolist(), indices), system_score=sum(pred.scores.tolist()) / len(pred.scores), ) - if "metadata" in pred: - output["metadata"] = Prediction( - **{k: restore_list_order(v, indices) for k, v in pred.metadata.items()} + if 'metadata' in pred: + output['metadata'] = Prediction( + **{ + k: restore_list_order(v, indices) + for k, v in pred.metadata.items() + } ) return output def cleanup(self): """Cleans temporary files.""" - logger.info("Cleanup temporary folder: {}.".format(self.output_dir)) + logger.info('Cleanup temporary folder: {}.'.format(self.output_dir)) shutil.rmtree(self.output_dir) diff --git a/comet/models/ranking/ranking_metric.py b/comet/models/ranking/ranking_metric.py index f5d9825d..a4c18526 100644 --- a/comet/models/ranking/ranking_metric.py +++ b/comet/models/ranking/ranking_metric.py @@ -22,14 +22,17 @@ `good` translations closer to the anchors (source & reference) than `worse` translations. """ + from typing import Dict, List, Optional, Tuple, Union import pandas as pd import torch import torch.nn.functional as F from torch import nn -from transformers.optimization import (Adafactor, - get_constant_schedule_with_warmup) +from transformers.optimization import ( + Adafactor, + get_constant_schedule_with_warmup, +) from comet.models.base import CometModel from comet.models.metrics import WMTKendall @@ -76,18 +79,18 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.1, keep_embeddings_frozen: bool = False, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1e-05, learning_rate: float = 3e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-base", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-base', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "triplet-margin", + loss: str = 'triplet-margin', dropout: float = 0.1, batch_size: int = 8, train_data: List[str] = [], @@ -113,7 +116,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="ranking_metric", + class_identifier='ranking_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -121,7 +124,7 @@ def __init__( def init_metrics(self): """Initializes train/validation metrics.""" - self.train_metrics = WMTKendall(prefix="train") + self.train_metrics = WMTKendall(prefix='train') self.val_metrics = nn.ModuleList( [WMTKendall(prefix=d) for d in self.hparams.validation_data] ) @@ -135,7 +138,9 @@ def loss(self): def configure_optimizers( self, - ) -> Tuple[List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR]]: + ) -> Tuple[ + List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR] + ]: """Pytorch Lightning method to configure optimizers and schedulers.""" layer_parameters = self.encoder.layerwise_lr( self.hparams.encoder_learning_rate, self.hparams.layerwise_decay @@ -143,15 +148,15 @@ def configure_optimizers( if self.layerwise_attention: layerwise_attn_params = [ { - "params": self.layerwise_attention.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.layerwise_attention.parameters(), + 'lr': self.hparams.learning_rate, } ] params = layer_parameters + layerwise_attn_params else: params = layer_parameters - if self.hparams.optimizer == "Adafactor": + if self.hparams.optimizer == 'Adafactor': optimizer = Adafactor( params, lr=self.hparams.learning_rate, @@ -172,7 +177,7 @@ def configure_optimizers( return [optimizer], [scheduler] def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "fit" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'fit' ) -> Dict[str, torch.Tensor]: """This method will be called by dataloaders to prepared data to input to the model. @@ -189,26 +194,26 @@ def prepare_sample( """ sample = {k: [str(dic[k]) for dic in sample] for k in sample[0]} - if stage == "predict": - src_inputs = self.encoder.prepare_sample(sample["src"]) - mt_inputs = self.encoder.prepare_sample(sample["mt"]) - ref_inputs = self.encoder.prepare_sample(sample["ref"]) + if stage == 'predict': + src_inputs = self.encoder.prepare_sample(sample['src']) + mt_inputs = self.encoder.prepare_sample(sample['mt']) + ref_inputs = self.encoder.prepare_sample(sample['ref']) - ref_inputs = {"ref_" + k: v for k, v in ref_inputs.items()} - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - mt_inputs = {"mt_" + k: v for k, v in mt_inputs.items()} + ref_inputs = {'ref_' + k: v for k, v in ref_inputs.items()} + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + mt_inputs = {'mt_' + k: v for k, v in mt_inputs.items()} return {**ref_inputs, **src_inputs, **mt_inputs} - ref_inputs = self.encoder.prepare_sample(sample["ref"]) - src_inputs = self.encoder.prepare_sample(sample["src"]) - pos_inputs = self.encoder.prepare_sample(sample["pos"]) - neg_inputs = self.encoder.prepare_sample(sample["neg"]) + ref_inputs = self.encoder.prepare_sample(sample['ref']) + src_inputs = self.encoder.prepare_sample(sample['src']) + pos_inputs = self.encoder.prepare_sample(sample['pos']) + neg_inputs = self.encoder.prepare_sample(sample['neg']) - ref_inputs = {"ref_" + k: v for k, v in ref_inputs.items()} - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - pos_inputs = {"pos_" + k: v for k, v in pos_inputs.items()} - neg_inputs = {"neg_" + k: v for k, v in neg_inputs.items()} + ref_inputs = {'ref_' + k: v for k, v in ref_inputs.items()} + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + pos_inputs = {'pos_' + k: v for k, v in pos_inputs.items()} + neg_inputs = {'neg_' + k: v for k, v in neg_inputs.items()} return {**ref_inputs, **src_inputs, **pos_inputs, **neg_inputs} @@ -241,10 +246,18 @@ def forward( Dictionary with triplet loss, distance between anchors and positive samples and distance between anchors and negative samples. """ - src_sentemb = self.get_sentence_embedding(src_input_ids, src_attention_mask) - ref_sentemb = self.get_sentence_embedding(ref_input_ids, ref_attention_mask) - pos_sentemb = self.get_sentence_embedding(pos_input_ids, pos_attention_mask) - neg_sentemb = self.get_sentence_embedding(neg_input_ids, neg_attention_mask) + src_sentemb = self.get_sentence_embedding( + src_input_ids, src_attention_mask + ) + ref_sentemb = self.get_sentence_embedding( + ref_input_ids, ref_attention_mask + ) + pos_sentemb = self.get_sentence_embedding( + pos_input_ids, pos_attention_mask + ) + neg_sentemb = self.get_sentence_embedding( + neg_input_ids, neg_attention_mask + ) loss = self.loss(src_sentemb, pos_sentemb, neg_sentemb) + self.loss( ref_sentemb, pos_sentemb, neg_sentemb @@ -265,9 +278,9 @@ def forward( ) return { - "loss": loss, - "distance_pos": distance_pos, - "distance_neg": distance_neg, + 'loss': loss, + 'distance_pos': distance_pos, + 'distance_neg': distance_neg, } def read_training_data(self, path: str) -> List[dict]: @@ -278,12 +291,12 @@ def read_training_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - df = df[["src", "pos", "neg", "ref"]] - df["src"] = df["src"].astype(str) - df["pos"] = df["pos"].astype(str) - df["neg"] = df["neg"].astype(str) - df["ref"] = df["ref"].astype(str) - return df.to_dict("records") + df = df[['src', 'pos', 'neg', 'ref']] + df['src'] = df['src'].astype(str) + df['pos'] = df['pos'].astype(str) + df['neg'] = df['neg'].astype(str) + df['ref'] = df['ref'].astype(str) + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Method that reads the validation data (a csv file) and returns a list of @@ -309,7 +322,7 @@ def training_step( [torch.Tensor] Loss value """ batch_prediction = self.forward(**batch) - loss_value = batch_prediction["loss"] + loss_value = batch_prediction['loss'] if ( self.nr_frozen_epochs < 1.0 @@ -319,7 +332,7 @@ def training_step( self.unfreeze_encoder() self._frozen = False - self.log("train_loss", loss_value, on_step=True, on_epoch=True) + self.log('train_loss', loss_value, on_step=True, on_epoch=True) return loss_value def validation_step( @@ -335,16 +348,18 @@ def validation_step( batch_idx (int): Integer displaying which batch this is. """ batch_prediction = self.forward(**batch) - loss_value = batch_prediction["loss"] - self.log("val_loss", loss_value, on_step=True, on_epoch=True) + loss_value = batch_prediction['loss'] + self.log('val_loss', loss_value, on_step=True, on_epoch=True) if dataloader_idx == 0: self.train_metrics.update( - batch_prediction["distance_pos"], batch_prediction["distance_neg"] + batch_prediction['distance_pos'], + batch_prediction['distance_neg'], ) elif dataloader_idx > 0: self.val_metrics[dataloader_idx - 1].update( - batch_prediction["distance_pos"], batch_prediction["distance_neg"] + batch_prediction['distance_pos'], + batch_prediction['distance_neg'], ) def predict_step( @@ -367,13 +382,13 @@ def predict_step( def _predict_forward(batch): src_sentemb = self.get_sentence_embedding( - batch["src_input_ids"], batch["src_attention_mask"] + batch['src_input_ids'], batch['src_attention_mask'] ) ref_sentemb = self.get_sentence_embedding( - batch["ref_input_ids"], batch["ref_attention_mask"] + batch['ref_input_ids'], batch['ref_attention_mask'] ) mt_sentemb = self.get_sentence_embedding( - batch["mt_input_ids"], batch["mt_attention_mask"] + batch['mt_input_ids'], batch['mt_attention_mask'] ) src_distance = F.pairwise_distance(mt_sentemb, src_sentemb) ref_distance = F.pairwise_distance(mt_sentemb, ref_sentemb) @@ -389,7 +404,7 @@ def _predict_forward(batch): ) if self.mc_dropout: - raise NotImplementedError("MCD not implemented for this model!") + raise NotImplementedError('MCD not implemented for this model!') else: return _predict_forward(batch) diff --git a/comet/models/regression/referenceless.py b/comet/models/regression/referenceless.py index b35ac95b..127965e0 100644 --- a/comet/models/regression/referenceless.py +++ b/comet/models/regression/referenceless.py @@ -19,6 +19,7 @@ Referenceless Regression Metric that learns to predict a quality assessment by looking at source and translation. """ + from typing import Dict, List, Optional, Tuple, Union import pandas as pd @@ -72,24 +73,24 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1e-06, learning_rate: float = 1.5e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-large", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-large', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [2048, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, load_pretrained_weights: bool = True, local_files_only: bool = False, @@ -113,7 +114,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="referenceless_regression_metric", + class_identifier='referenceless_regression_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -131,13 +132,14 @@ def requires_references(self) -> bool: return False def enable_context(self): - if self.pool == "avg": + if self.pool == 'avg': self.use_context = True def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "train" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'train' ) -> Union[ - Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], Dict[str, torch.Tensor] + Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], + Dict[str, torch.Tensor], ]: """This method will be called by dataloaders to prepared data to input to the model. @@ -150,22 +152,26 @@ def prepare_sample( Returns: Model inputs and depending on the 'stage' training labels/targets. """ - inputs = {k: [str(dic[k]) for dic in sample] for k in sample[0] if k != "score"} - src_inputs = self.encoder.prepare_sample(inputs["src"]) - mt_inputs = self.encoder.prepare_sample(inputs["mt"]) - - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - mt_inputs = {"mt_" + k: v for k, v in mt_inputs.items()} + inputs = { + k: [str(dic[k]) for dic in sample] + for k in sample[0] + if k != 'score' + } + src_inputs = self.encoder.prepare_sample(inputs['src']) + mt_inputs = self.encoder.prepare_sample(inputs['mt']) + + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + mt_inputs = {'mt_' + k: v for k, v in mt_inputs.items()} model_inputs = {**src_inputs, **mt_inputs} - if stage == "predict": + if stage == 'predict': return model_inputs - scores = [float(s["score"]) for s in sample] + scores = [float(s['score']) for s in sample] targets = Target(score=torch.tensor(scores, dtype=torch.float)) - if "system" in inputs: - targets["system"] = inputs["system"] + if 'system' in inputs: + targets['system'] = inputs['system'] return model_inputs, targets @@ -175,7 +181,7 @@ def forward( src_attention_mask: torch.tensor, mt_input_ids: torch.tensor, mt_attention_mask: torch.tensor, - **kwargs + **kwargs, ) -> Dict[str, torch.Tensor]: """ReferencelessRegression model forward method. @@ -188,8 +194,12 @@ def forward( Return: Prediction object with translation scores. """ - src_sentemb = self.get_sentence_embedding(src_input_ids, src_attention_mask) - mt_sentemb = self.get_sentence_embedding(mt_input_ids, mt_attention_mask) + src_sentemb = self.get_sentence_embedding( + src_input_ids, src_attention_mask + ) + mt_sentemb = self.get_sentence_embedding( + mt_input_ids, mt_attention_mask + ) diff_src = torch.abs(mt_sentemb - src_sentemb) prod_src = mt_sentemb * src_sentemb @@ -207,11 +217,11 @@ def read_training_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - df = df[["src", "mt", "score"]] - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - df["score"] = df["score"].astype("float16") - return df.to_dict("records") + df = df[['src', 'mt', 'score']] + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + df['score'] = df['score'].astype('float16') + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Method that reads the validation data (a csv file) and returns a list of @@ -221,14 +231,14 @@ def read_validation_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - columns = ["src", "mt", "score"] + columns = ['src', 'mt', 'score'] # If system in columns we will use this to calculate system-level accuracy - if "system" in df.columns: - columns.append("system") - df["system"] = df["system"].astype(str) + if 'system' in df.columns: + columns.append('system') + df['system'] = df['system'].astype(str) df = df[columns] - df["score"] = df["score"].astype("float16") - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - return df.to_dict("records") + df['score'] = df['score'].astype('float16') + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + return df.to_dict('records') diff --git a/comet/models/regression/regression_metric.py b/comet/models/regression/regression_metric.py index abaf0fab..6387ad53 100644 --- a/comet/models/regression/regression_metric.py +++ b/comet/models/regression/regression_metric.py @@ -19,13 +19,16 @@ Regression Metric that learns to predict a quality assessment by looking at source, translation and reference. """ + from typing import Dict, List, Optional, Tuple, Union import pandas as pd import torch from torch import nn -from transformers.optimization import (Adafactor, - get_constant_schedule_with_warmup) +from transformers.optimization import ( + Adafactor, + get_constant_schedule_with_warmup, +) from comet.models.base import CometModel from comet.models.metrics import RegressionMetrics @@ -76,24 +79,24 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1e-06, learning_rate: float = 1.5e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-large", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-large', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [3072, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, load_pretrained_weights: bool = True, local_files_only: bool = False, @@ -117,7 +120,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="regression_metric", + class_identifier='regression_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -133,7 +136,7 @@ def __init__( def init_metrics(self): """Initializes train/validation metrics.""" - self.train_metrics = RegressionMetrics(prefix="train") + self.train_metrics = RegressionMetrics(prefix='train') self.val_metrics = nn.ModuleList( [RegressionMetrics(prefix=d) for d in self.hparams.validation_data] ) @@ -143,26 +146,33 @@ def requires_references(self) -> bool: def configure_optimizers( self, - ) -> Tuple[List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR]]: + ) -> Tuple[ + List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR] + ]: """Pytorch Lightning method to configure optimizers and schedulers.""" layer_parameters = self.encoder.layerwise_lr( self.hparams.encoder_learning_rate, self.hparams.layerwise_decay ) top_layers_parameters = [ - {"params": self.estimator.parameters(), "lr": self.hparams.learning_rate} + { + 'params': self.estimator.parameters(), + 'lr': self.hparams.learning_rate, + } ] if self.layerwise_attention: layerwise_attn_params = [ { - "params": self.layerwise_attention.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.layerwise_attention.parameters(), + 'lr': self.hparams.learning_rate, } ] - params = layer_parameters + top_layers_parameters + layerwise_attn_params + params = ( + layer_parameters + top_layers_parameters + layerwise_attn_params + ) else: params = layer_parameters + top_layers_parameters - if self.hparams.optimizer == "Adafactor": + if self.hparams.optimizer == 'Adafactor': optimizer = Adafactor( params, lr=self.hparams.learning_rate, @@ -183,9 +193,10 @@ def configure_optimizers( return [optimizer], [scheduler] def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "train" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'train' ) -> Union[ - Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], Dict[str, torch.Tensor] + Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], + Dict[str, torch.Tensor], ]: """This method will be called by dataloaders to prepared data to input to the model. @@ -198,29 +209,33 @@ def prepare_sample( Returns: Model inputs and depending on the 'stage' training labels/targets. """ - inputs = {k: [str(dic[k]) for dic in sample] for k in sample[0] if k != "score"} - src_inputs = self.encoder.prepare_sample(inputs["src"]) - mt_inputs = self.encoder.prepare_sample(inputs["mt"]) - ref_inputs = self.encoder.prepare_sample(inputs["ref"]) - - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - mt_inputs = {"mt_" + k: v for k, v in mt_inputs.items()} - ref_inputs = {"ref_" + k: v for k, v in ref_inputs.items()} + inputs = { + k: [str(dic[k]) for dic in sample] + for k in sample[0] + if k != 'score' + } + src_inputs = self.encoder.prepare_sample(inputs['src']) + mt_inputs = self.encoder.prepare_sample(inputs['mt']) + ref_inputs = self.encoder.prepare_sample(inputs['ref']) + + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + mt_inputs = {'mt_' + k: v for k, v in mt_inputs.items()} + ref_inputs = {'ref_' + k: v for k, v in ref_inputs.items()} model_inputs = {**src_inputs, **mt_inputs, **ref_inputs} - if stage == "predict": + if stage == 'predict': return model_inputs - scores = [float(s["score"]) for s in sample] + scores = [float(s['score']) for s in sample] targets = Target(score=torch.tensor(scores, dtype=torch.float)) - if "system" in inputs: - targets["system"] = inputs["system"] + if 'system' in inputs: + targets['system'] = inputs['system'] return model_inputs, targets def enable_context(self): - if self.pool == "avg": + if self.pool == 'avg': self.use_context = True def estimate( @@ -260,7 +275,7 @@ def forward( mt_attention_mask: torch.tensor, ref_input_ids: torch.tensor, ref_attention_mask: torch.tensor, - **kwargs + **kwargs, ) -> Prediction: """Regression model forward method. @@ -275,9 +290,15 @@ def forward( Return: Prediction object with translation scores. """ - src_sentemb = self.get_sentence_embedding(src_input_ids, src_attention_mask) - ref_sentemb = self.get_sentence_embedding(ref_input_ids, ref_attention_mask) - mt_sentemb = self.get_sentence_embedding(mt_input_ids, mt_attention_mask) + src_sentemb = self.get_sentence_embedding( + src_input_ids, src_attention_mask + ) + ref_sentemb = self.get_sentence_embedding( + ref_input_ids, ref_attention_mask + ) + mt_sentemb = self.get_sentence_embedding( + mt_input_ids, mt_attention_mask + ) return self.estimate(src_sentemb, mt_sentemb, ref_sentemb) def read_training_data(self, path: str) -> List[dict]: @@ -288,12 +309,12 @@ def read_training_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - df = df[["src", "mt", "ref", "score"]] - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - df["ref"] = df["ref"].astype(str) - df["score"] = df["score"].astype("float16") - return df.to_dict("records") + df = df[['src', 'mt', 'ref', 'score']] + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + df['ref'] = df['ref'].astype(str) + df['score'] = df['score'].astype('float16') + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Method that reads the validation data (a csv file) and returns a list of @@ -303,15 +324,15 @@ def read_validation_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - columns = ["src", "mt", "ref", "score"] + columns = ['src', 'mt', 'ref', 'score'] # If system in columns we will use this to calculate system-level accuracy - if "system" in df.columns: - columns.append("system") - df["system"] = df["system"].astype(str) + if 'system' in df.columns: + columns.append('system') + df['system'] = df['system'].astype(str) df = df[columns] - df["score"] = df["score"].astype("float16") - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - df["ref"] = df["ref"].astype(str) - return df.to_dict("records") + df['score'] = df['score'].astype('float16') + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + df['ref'] = df['ref'].astype(str) + return df.to_dict('records') diff --git a/comet/models/utils.py b/comet/models/utils.py index 8b98c9f4..b9343c1d 100644 --- a/comet/models/utils.py +++ b/comet/models/utils.py @@ -43,10 +43,10 @@ def __post_init__(self): # Safety and consistency checks if not len(class_fields): - raise ValueError(f"{self.__class__.__name__} has no fields.") + raise ValueError(f'{self.__class__.__name__} has no fields.') if not all(field.default is None for field in class_fields[1:]): raise ValueError( - f"{self.__class__.__name__} should not have more than one required field." + f'{self.__class__.__name__} should not have more than one required field.' ) first_field = getattr(self, class_fields[0].name) @@ -88,22 +88,22 @@ def __post_init__(self): def __delitem__(self, *args, **kwargs): raise Exception( - f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance." + f'You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.' ) def setdefault(self, *args, **kwargs): raise Exception( - f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance." + f'You cannot use ``setdefault`` on a {self.__class__.__name__} instance.' ) def pop(self, *args, **kwargs): raise Exception( - f"You cannot use ``pop`` on a {self.__class__.__name__} instance." + f'You cannot use ``pop`` on a {self.__class__.__name__} instance.' ) def update(self, *args, **kwargs): raise Exception( - f"You cannot use ``update`` on a {self.__class__.__name__} instance." + f'You cannot use ``update`` on a {self.__class__.__name__} instance.' ) def __getitem__(self, k): @@ -147,16 +147,16 @@ class Target(ModelOutput): class LabelSet: """Taken from: https://github.com/LightTag/sequence-labeling-with-transformers/""" - def __init__(self, labels: List[str] = ["minor", "major", "critical"]): + def __init__(self, labels: List[str] = ['minor', 'major', 'critical']): self.labels_to_id = {} self.ids_to_label = {} - self.labels_to_id["O"] = 0 - self.ids_to_label[0] = "O" + self.labels_to_id['O'] = 0 + self.ids_to_label[0] = 'O' num = 0 # in case there are no labels # Writing BILU will give us incremntal ids for the labels - for _num, (label, s) in enumerate(itertools.product(labels, "I")): + for _num, (label, s) in enumerate(itertools.product(labels, 'I')): num = _num + 1 # skip 0 - l = f"{s}-{label}" + l = f'{s}-{label}' self.labels_to_id[l] = num self.ids_to_label[num] = l @@ -178,7 +178,9 @@ def flatten_metadata(metadata): """Metadata from the model output can be in various forms and this function will gather all metadata and flatten everything. """ - metadata = Prediction(**{k: [dic[k] for dic in metadata] for k in metadata[0]}) + metadata = Prediction( + **{k: [dic[k] for dic in metadata] for k in metadata[0]} + ) for k, v in metadata.items(): if torch.is_tensor(v[0]): # If we have tensors we can use cat to flatten them. diff --git a/comet/modules/feedforward.py b/comet/modules/feedforward.py index a19a74b3..9471a4dc 100644 --- a/comet/modules/feedforward.py +++ b/comet/modules/feedforward.py @@ -17,6 +17,7 @@ ============ Feed Forward Neural Network module that can be used for classification or regression """ + from typing import List, Optional import torch @@ -41,7 +42,7 @@ def __init__( in_dim: int, out_dim: int = 1, hidden_sizes: List[int] = [3072, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, dropout: float = 0.1, ) -> None: @@ -66,7 +67,7 @@ def build_activation(self, activation: str) -> nn.Module: if hasattr(nn, activation.title()): return getattr(nn, activation.title())() else: - raise Exception(f"{activation} is not a valid activation function!") + raise Exception(f'{activation} is not a valid activation function!') def forward(self, in_features: torch.Tensor) -> torch.Tensor: # When casting models to float 16 self.ff(in_features) was giving some problems reported diff --git a/comet/modules/layerwise_attention.py b/comet/modules/layerwise_attention.py index b22ffba1..47a608f6 100644 --- a/comet/modules/layerwise_attention.py +++ b/comet/modules/layerwise_attention.py @@ -26,6 +26,7 @@ Original implementation: - https://github.com/Hyperparticle/udify """ + from typing import List, Optional import torch @@ -39,7 +40,7 @@ def __init__( layer_norm: bool = False, layer_weights: Optional[List[int]] = None, dropout: float = None, - layer_transformation: str = "softmax", + layer_transformation: str = 'softmax', ) -> None: super(LayerwiseAttention, self).__init__() self.num_layers = num_layers @@ -47,7 +48,7 @@ def __init__( self.dropout = dropout self.transform_fn = torch.softmax - if layer_transformation == "sparsemax": + if layer_transformation == 'sparsemax': from entmax import sparsemax self.transform_fn = sparsemax @@ -55,12 +56,8 @@ def __init__( if layer_weights is None: layer_weights = [0.0] * num_layers elif len(layer_weights) != num_layers: - raise Exception( - "Length of layer_weights {} differs \ - from num_layers {}".format( - layer_weights, num_layers - ) - ) + raise Exception('Length of layer_weights {} differs \ + from num_layers {}'.format(layer_weights, num_layers)) self.scalar_parameters = ParameterList( [ @@ -77,8 +74,8 @@ def __init__( if self.dropout: dropout_mask = torch.zeros(len(self.scalar_parameters)) dropout_fill = torch.empty(len(self.scalar_parameters)).fill_(-1e20) - self.register_buffer("dropout_mask", dropout_mask) - self.register_buffer("dropout_fill", dropout_fill) + self.register_buffer('dropout_mask', dropout_mask) + self.register_buffer('dropout_fill', dropout_fill) def forward( self, @@ -87,10 +84,8 @@ def forward( ) -> torch.Tensor: if len(tensors) != self.num_layers: raise Exception( - "{} tensors were passed, but the module was initialized to \ - mix {} tensors.".format( - len(tensors), self.num_layers - ) + '{} tensors were passed, but the module was initialized to \ + mix {} tensors.'.format(len(tensors), self.num_layers) ) def _layer_norm(tensor, broadcast_mask, mask): @@ -105,23 +100,30 @@ def _layer_norm(tensor, broadcast_mask, mask): variance = (((tensor_masked - mean) * broadcast_mask) ** 2).view( batch_size, -1 ).sum(1) / num_elements_not_masked - normalized_tensor = (tensor - mean) / torch.sqrt(variance + 1e-12).view( - batch_size, 1, 1 - ) + normalized_tensor = (tensor - mean) / torch.sqrt( + variance + 1e-12 + ).view(batch_size, 1, 1) return normalized_tensor # BUG: Pytorch bug fix when Parameters are not well copied across GPUs # https://github.com/pytorch/pytorch/issues/36035 - if len([parameter for parameter in self.scalar_parameters]) != self.num_layers: + if ( + len([parameter for parameter in self.scalar_parameters]) + != self.num_layers + ): weights = torch.tensor(self.weights, device=tensors[0].device) gamma = torch.tensor(self.gamma_value, device=tensors[0].device) else: - weights = torch.cat([parameter for parameter in self.scalar_parameters]) + weights = torch.cat( + [parameter for parameter in self.scalar_parameters] + ) gamma = self.gamma if self.training and self.dropout: weights = torch.where( - self.dropout_mask.uniform_() > self.dropout, weights, self.dropout_fill + self.dropout_mask.uniform_() > self.dropout, + weights, + self.dropout_fill, ) normed_weights = self.transform_fn(weights, dim=0) @@ -139,5 +141,7 @@ def _layer_norm(tensor, broadcast_mask, mask): pieces = [] for weight, tensor in zip(normed_weights, tensors): - pieces.append(weight * _layer_norm(tensor, broadcast_mask, mask_float)) + pieces.append( + weight * _layer_norm(tensor, broadcast_mask, mask_float) + ) return gamma * sum(pieces) diff --git a/docs/source/conf.py b/docs/source/conf.py index cd88a5e6..da29f9a7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,15 +17,15 @@ # -- Project information ----------------------------------------------------- -project = "COMET" +project = 'COMET' copyright = ( - "2020, Unbabel. All rights reserved." - "Source code available under Apache License 2.0" + '2020, Unbabel. All rights reserved.' + 'Source code available under Apache License 2.0' ) -author = "Unbabel" +author = 'Unbabel' # The full version, including alpha/beta/rc tags -release = "0.0.3" +release = '0.0.3' # -- General configuration --------------------------------------------------- @@ -34,32 +34,32 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.autodoc", + 'sphinx.ext.autodoc', # 'sphinx.ext.doctest', - "sphinx.ext.intersphinx", - "sphinx.ext.todo", - "sphinx.ext.coverage", - "sphinx.ext.mathjax", - "sphinx.ext.viewcode", - "sphinx.ext.githubpages", - "sphinx.ext.napoleon", - "recommonmark", + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', + 'recommonmark', #'sphinxarg.ext', #'m2r', # 'sphinx-issues', # 'pytest-sphinx', - "sphinx_markdown_tables", - "sphinx.ext.autosectionlabel", + 'sphinx_markdown_tables', + 'sphinx.ext.autosectionlabel', ] autosectionlabel_prefix_document = True # Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +templates_path = ['_templates'] source_suffix = { - ".rst": "restructuredtext", - ".txt": "markdown", - ".md": "markdown", + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', } # List of patterns, relative to source directory, that match files and @@ -68,12 +68,12 @@ exclude_patterns = [] # The master toctree document. -master_doc = "index" +master_doc = 'index' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -85,14 +85,14 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = ['_static'] def setup(app): - app.add_css_file("css/comet.css") + app.add_css_file('css/comet.css') diff --git a/tests/integration/models/test_ranking_metric.py b/tests/integration/models/test_ranking_metric.py index a4e59f26..16ba9b39 100644 --- a/tests/integration/models/test_ranking_metric.py +++ b/tests/integration/models/test_ranking_metric.py @@ -13,20 +13,20 @@ from comet.models import RankingMetric from tests.data import DATA_PATH -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestRankingMetric(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_training(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=22, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -36,68 +36,68 @@ def test_training(self): model = RankingMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-04, learning_rate=1e-04, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - pool="cls", - layer="mix", - layer_transformation="softmax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + pool='cls', + layer='mix', + layer_transformation='softmax', layer_norm=True, dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "ranking_data.csv")], - validation_data=[os.path.join(DATA_PATH, "ranking_data.csv")], + train_data=[os.path.join(DATA_PATH, 'ranking_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'ranking_data.csv')], ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=21-step=154.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=21-step=154.ckpt') ) ) saved_model = RankingMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=21-step=154.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=21-step=154.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "ranking_data.csv") + os.path.join(DATA_PATH, 'ranking_data.csv') ) # Scores for "superior" translations pos_translations = [ - {"src": s["src"], "mt": s["pos"], "ref": s["ref"]} for s in dataset + {'src': s['src'], 'mt': s['pos'], 'ref': s['ref']} for s in dataset ] dataloader = DataLoader( dataset=pos_translations, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_pos = torch.cat([p["scores"] for p in predictions], dim=0) + y_pos = torch.cat([p['scores'] for p in predictions], dim=0) # Scores for "worse" translations neg_translations = [ - {"src": s["src"], "mt": s["neg"], "ref": s["ref"]} for s in dataset + {'src': s['src'], 'mt': s['neg'], 'ref': s['ref']} for s in dataset ] dataloader = DataLoader( dataset=neg_translations, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_neg = torch.cat([p["scores"] for p in predictions], dim=0) + y_neg = torch.cat([p['scores'] for p in predictions], dim=0) ## This shouldn't break! pearsonr(y_pos, y_neg)[0] diff --git a/tests/integration/models/test_referenceless_regression.py b/tests/integration/models/test_referenceless_regression.py index 06e88cc3..2b023f1d 100644 --- a/tests/integration/models/test_referenceless_regression.py +++ b/tests/integration/models/test_referenceless_regression.py @@ -13,8 +13,8 @@ from comet.models import ReferencelessRegression from tests.data import DATA_PATH -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestReferencelessRegression(unittest.TestCase): @@ -22,13 +22,13 @@ class TestReferencelessRegression(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_training(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=10, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -38,52 +38,52 @@ def test_training(self): model = ReferencelessRegression( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-04, learning_rate=1e-04, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - pool="avg", - layer="mix", - layer_transformation="sparsemax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + pool='avg', + layer='mix', + layer_transformation='sparsemax', layer_norm=True, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) ) saved_model = ReferencelessRegression.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.85 diff --git a/tests/integration/models/test_regression_metric.py b/tests/integration/models/test_regression_metric.py index efadaa79..d32ce499 100644 --- a/tests/integration/models/test_regression_metric.py +++ b/tests/integration/models/test_regression_metric.py @@ -13,8 +13,8 @@ from comet.models import RegressionMetric from tests.data import DATA_PATH -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestRegressionMetric(unittest.TestCase): @@ -22,13 +22,13 @@ class TestRegressionMetric(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_training(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=10, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -38,52 +38,52 @@ def test_training(self): model = RegressionMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-04, learning_rate=1e-04, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - pool="avg", - layer="mix", - layer_transformation="sparsemax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + pool='avg', + layer='mix', + layer_transformation='sparsemax', layer_norm=True, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) ) saved_model = RegressionMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.85 diff --git a/tests/integration/models/test_unified_metric.py b/tests/integration/models/test_unified_metric.py index 1fc455e3..61daf39b 100644 --- a/tests/integration/models/test_unified_metric.py +++ b/tests/integration/models/test_unified_metric.py @@ -14,8 +14,8 @@ from comet.models import UnifiedMetric from tests.data import DATA_PATH -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestUnifiedMetric(unittest.TestCase): @@ -23,13 +23,13 @@ class TestUnifiedMetric(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_regression_with_references(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=8, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -39,62 +39,62 @@ def test_regression_with_references(self): model = UnifiedMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-03, learning_rate=1e-03, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - sent_layer="mix", - layer_transformation="softmax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + sent_layer='mix', + layer_transformation='softmax', layer_norm=False, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, - input_segments=["mt", "src", "ref"], + input_segments=['mt', 'src', 'ref'], word_level_training=False, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=7-step=104.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=7-step=104.ckpt') ) ) saved_model = UnifiedMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=7-step=104.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=7-step=104.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.9 def test_regression_without_references(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=10, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -104,54 +104,54 @@ def test_regression_without_references(self): model = UnifiedMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-03, learning_rate=1e-03, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - sent_layer="mix", - layer_transformation="softmax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + sent_layer='mix', + layer_transformation='softmax', layer_norm=False, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, - input_segments=["mt", "src"], + input_segments=['mt', 'src'], word_level_training=False, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) ) saved_model = UnifiedMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt"), - input_segments=["mt", "src"], + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt'), + input_segments=['mt', 'src'], ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.9 diff --git a/tests/integration/modules/test_feedforward.py b/tests/integration/modules/test_feedforward.py index 3a28a905..da80f899 100644 --- a/tests/integration/modules/test_feedforward.py +++ b/tests/integration/modules/test_feedforward.py @@ -22,9 +22,12 @@ def test_MNIST(self): images = [torch.Tensor(images[i, :]) for i in range(images.shape[0])] labels = torch.tensor(labels, dtype=torch.long) - train_images, test_images, train_labels, test_labels = train_test_split( - images, labels, test_size=0.2, random_state=42 - ) + ( + train_images, + test_images, + train_labels, + test_labels, + ) = train_test_split(images, labels, test_size=0.2, random_state=42) train_dataset = list(zip(train_images, train_labels)) test_dataset = list(zip(test_images, test_labels)) @@ -52,7 +55,7 @@ def test_MNIST(self): in_dim=8 * 8, out_dim=10, hidden_sizes=[100], - activations="Tanh", + activations='Tanh', ) """ diff --git a/tests/unit/encoders/test_bert.py b/tests/unit/encoders/test_bert.py index 51d158d7..1dda6542 100644 --- a/tests/unit/encoders/test_bert.py +++ b/tests/unit/encoders/test_bert.py @@ -5,7 +5,7 @@ class TestBERTEncoder(unittest.TestCase): - bert = BERTEncoder.from_pretrained("google/bert_uncased_L-2_H-128_A-2") + bert = BERTEncoder.from_pretrained('google/bert_uncased_L-2_H-128_A-2') def test_num_layers(self): self.assertEqual(self.bert.num_layers, 3) @@ -17,26 +17,26 @@ def test_max_positions(self): self.assertEqual(self.bert.max_positions, 510) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) model_output = self.bert(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] translations_input = self.bert.prepare_sample( @@ -95,9 +95,14 @@ def test_concat_sequences(self): continuous_input = self.bert.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) self.assertListEqual( - continuous_input[0]["token_type_ids"].tolist(), expected_token_type_ids + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) + self.assertListEqual( + continuous_input[0]['token_type_ids'].tolist(), + expected_token_type_ids, ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/encoders/test_minilm.py b/tests/unit/encoders/test_minilm.py index b98dd31f..0d5b99dc 100644 --- a/tests/unit/encoders/test_minilm.py +++ b/tests/unit/encoders/test_minilm.py @@ -7,7 +7,9 @@ class TestMiniLMEncoder(unittest.TestCase): """MiniLMV2 uses XLM-R tokenizer thus, most tests are copy of minilmEncoder""" - minilm = MiniLMEncoder.from_pretrained("microsoft/Multilingual-MiniLM-L12-H384") + minilm = MiniLMEncoder.from_pretrained( + 'microsoft/Multilingual-MiniLM-L12-H384' + ) def test_num_layers(self): self.assertEqual(self.minilm.num_layers, 13) @@ -19,26 +21,26 @@ def test_max_positions(self): self.assertEqual(self.minilm.max_positions, 510) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.minilm.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.minilm.prepare_sample(sample) model_output = self.minilm(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] source_input = self.minilm.prepare_sample(source) @@ -63,7 +65,23 @@ def test_concat_sequences(self): 38, 2, ], - [0, 40088, 393, 286, 15946, 38, 2, 2, 3293, 83, 142, 27781, 38, 2, 1], + [ + 0, + 40088, + 393, + 286, + 15946, + 38, + 2, + 2, + 3293, + 83, + 142, + 27781, + 38, + 2, + 1, + ], ] expected_labels = [ [0, 0, 0, 0, 0, 2, 2, 0, -1, -1, -1, -1, -1, -1, -1], @@ -73,6 +91,10 @@ def test_concat_sequences(self): continuous_input = self.minilm.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) + self.assertListEqual( + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/encoders/test_rembert.py b/tests/unit/encoders/test_rembert.py index dfe47885..e53cd31e 100644 --- a/tests/unit/encoders/test_rembert.py +++ b/tests/unit/encoders/test_rembert.py @@ -5,7 +5,7 @@ class TestRemBERTEncoder(unittest.TestCase): - bert = RemBERTEncoder.from_pretrained("google/rembert") + bert = RemBERTEncoder.from_pretrained('google/rembert') def test_num_layers(self): self.assertEqual(self.bert.num_layers, 33) @@ -17,26 +17,26 @@ def test_max_positions(self): self.assertEqual(self.bert.max_positions, 510) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) model_output = self.bert(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] translations_input = self.bert.prepare_sample( @@ -60,7 +60,22 @@ def test_concat_sequences(self): 646, 313, ], - [312, 58378, 921, 835, 17293, 646, 313, 1357, 619, 666, 7469, 646, 313, 0], + [ + 312, + 58378, + 921, + 835, + 17293, + 646, + 313, + 1357, + 619, + 666, + 7469, + 646, + 313, + 0, + ], ] expected_labels = [ [0, 0, 0, 0, 0, 2, 2, 0, -1, -1, -1, -1, -1, -1], @@ -74,9 +89,14 @@ def test_concat_sequences(self): continuous_input = self.bert.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) self.assertListEqual( - continuous_input[0]["token_type_ids"].tolist(), expected_token_type_ids + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) + self.assertListEqual( + continuous_input[0]['token_type_ids'].tolist(), + expected_token_type_ids, ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/encoders/test_xlmr.py b/tests/unit/encoders/test_xlmr.py index a0b9497d..43d1282b 100644 --- a/tests/unit/encoders/test_xlmr.py +++ b/tests/unit/encoders/test_xlmr.py @@ -5,7 +5,7 @@ class TestXLMREncoder(unittest.TestCase): - xlmr = XLMREncoder.from_pretrained("xlm-roberta-base") + xlmr = XLMREncoder.from_pretrained('xlm-roberta-base') def test_num_layers(self): self.assertEqual(self.xlmr.num_layers, 13) @@ -17,26 +17,26 @@ def test_max_positions(self): self.assertEqual(self.xlmr.max_positions, 512) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.xlmr.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.xlmr.prepare_sample(sample) model_output = self.xlmr(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] translations_input = self.xlmr.prepare_sample( @@ -61,7 +61,23 @@ def test_concat_sequences(self): 38, 2, ], - [0, 40088, 393, 286, 15946, 38, 2, 2, 3293, 83, 142, 27781, 38, 2, 1], + [ + 0, + 40088, + 393, + 286, + 15946, + 38, + 2, + 2, + 3293, + 83, + 142, + 27781, + 38, + 2, + 1, + ], ] expected_labels = [ [0, 0, 0, 0, 0, 2, 2, 0, -1, -1, -1, -1, -1, -1, -1], @@ -71,6 +87,10 @@ def test_concat_sequences(self): continuous_input = self.xlmr.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) + self.assertListEqual( + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 1301bb14..5e36cfbe 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -38,5 +38,6 @@ def test_cache(self): tmp = self.add(torch.tensor(0), torch.tensor(1)) self.assertTrue( - "Tensor needs to be at least 1-Dimensional." in str(context.exception) + 'Tensor needs to be at least 1-Dimensional.' + in str(context.exception) ) diff --git a/tests/unit/test_download_load.py b/tests/unit/test_download_load.py index 80ba4e61..afba7c4e 100644 --- a/tests/unit/test_download_load.py +++ b/tests/unit/test_download_load.py @@ -11,16 +11,22 @@ class TestDownloadModel(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "models--Unbabel--wmt22-comet-da")) - shutil.rmtree(os.path.join(DATA_PATH, "eamt22-cometinho-da")) + shutil.rmtree( + os.path.join(DATA_PATH, 'models--Unbabel--wmt22-comet-da') + ) + shutil.rmtree(os.path.join(DATA_PATH, 'eamt22-cometinho-da')) def test_download_from_aws(self): - data_path = download_model("eamt22-cometinho-da", saving_directory=DATA_PATH) + data_path = download_model( + 'eamt22-cometinho-da', saving_directory=DATA_PATH + ) load_from_checkpoint(data_path) def test_download_fail(self): - self.assertRaises(KeyError, download_model, "this_model_does_not_exist") + self.assertRaises(KeyError, download_model, 'this_model_does_not_exist') def test_download_from_hf(self): - data_path = download_model("Unbabel/wmt22-comet-da", saving_directory=DATA_PATH) + data_path = download_model( + 'Unbabel/wmt22-comet-da', saving_directory=DATA_PATH + ) load_from_checkpoint(data_path) diff --git a/tests/unit/test_models_predict.py b/tests/unit/test_models_predict.py index f80c27eb..7ee6dd46 100644 --- a/tests/unit/test_models_predict.py +++ b/tests/unit/test_models_predict.py @@ -12,162 +12,162 @@ TEST_SAMPLES = [ { - "lp": "it-en", - "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", - "mt": "In 1884, Tesla accepted a job at the Pacific League of New York City and moved to the United States of America.", - "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", - "annotations": [ + 'lp': 'it-en', + 'src': "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", + 'mt': 'In 1884, Tesla accepted a job at the Pacific League of New York City and moved to the United States of America.', + 'ref': 'In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.', + 'annotations': [ { - "start": 37, - "end": 51, - "text": "Pacific League", - "category": "critical_id9_ne_replaced", - "severity": "major", + 'start': 37, + 'end': 51, + 'text': 'Pacific League', + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', } ], - "score": 0.5833333333333333, + 'score': 0.5833333333333333, }, { - "lp": "it-en", - "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", - "mt": "In 1884, Tesla accepted a job at the Edison Company of New York City and moved to the United States of America.", - "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", - "annotations": [], - "score": 1.0, + 'lp': 'it-en', + 'src': "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", + 'mt': 'In 1884, Tesla accepted a job at the Edison Company of New York City and moved to the United States of America.', + 'ref': 'In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.', + 'annotations': [], + 'score': 1.0, }, { - "lp": "it-en", - "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", - "mt": "The American Civil War was also an inspiration for many other oppressed workers from various countries to start their own revolution.", - "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", - "annotations": [ + 'lp': 'it-en', + 'src': 'La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.', + 'mt': 'The American Civil War was also an inspiration for many other oppressed workers from various countries to start their own revolution.', + 'ref': "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", + 'annotations': [ { - "text": "American Civil War", - "start": 4, - "end": 22, - "category": "critical_id9_ne_replaced", - "severity": "major", + 'text': 'American Civil War', + 'start': 4, + 'end': 22, + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', } ], - "score": 0.5454545454545454, + 'score': 0.5454545454545454, }, { - "lp": "it-en", - "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", - "mt": "The French Revolution was also an inspiration for many other oppressed workers from various countries to start their own revolution.", - "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", - "annotations": [], - "score": 1.0, + 'lp': 'it-en', + 'src': 'La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.', + 'mt': 'The French Revolution was also an inspiration for many other oppressed workers from various countries to start their own revolution.', + 'ref': "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", + 'annotations': [], + 'score': 1.0, }, { - "lp": "it-en", - "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", - "mt": "Lokomotiv Moscow activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", - "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", - "annotations": [ + 'lp': 'it-en', + 'src': "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", + 'mt': 'Lokomotiv Moscow activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.', + 'ref': 'With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.', + 'annotations': [ { - "start": 0, - "end": 16, - "text": "Lokomotiv Moscow", - "category": "critical_id9_ne_replaced", - "severity": "major", + 'start': 0, + 'end': 16, + 'text': 'Lokomotiv Moscow', + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', } ], - "score": 0.6, + 'score': 0.6, }, { - "lp": "it-en", - "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", - "mt": "Kundalini Yoga activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", - "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", - "annotations": [], - "score": 1.0, + 'lp': 'it-en', + 'src': "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", + 'mt': 'Kundalini Yoga activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.', + 'ref': 'With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.', + 'annotations': [], + 'score': 1.0, }, { - "lp": "it-en", - "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", - "mt": "The East African islands are located in the Longxi River, off the east coast of Africa.", - "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", - "annotations": [ + 'lp': 'it-en', + 'src': "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + 'mt': 'The East African islands are located in the Longxi River, off the east coast of Africa.', + 'ref': 'The East African Islands are in the Indian Ocean off the eastern coast of Africa.', + 'annotations': [ { - "start": 44, - "end": 56, - "text": "Longxi River", - "category": "critical_id9_ne_replaced", - "severity": "major", + 'start': 44, + 'end': 56, + 'text': 'Longxi River', + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', } ], - "score": 0.4444444444444444, + 'score': 0.4444444444444444, }, { - "lp": "it-en", - "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", - "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", - "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", - "annotations": [], - "score": 1.0, + 'lp': 'it-en', + 'src': "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + 'mt': 'The East African islands are located in the Indian Ocean, off the east coast of Africa.', + 'ref': 'The East African Islands are in the Indian Ocean off the eastern coast of Africa.', + 'annotations': [], + 'score': 1.0, }, { - "lp": "it-en", - "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", - "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of American Dollar (GBP).", - "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", - "annotations": [ + 'lp': 'it-en', + 'src': 'Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).', + 'mt': 'The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of American Dollar (GBP).', + 'ref': 'The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).', + 'annotations': [ { - "text": "of American Dollar", - "start": 112, - "end": 130, - "category": "critical_id9_ne_replaced", - "severity": "major", + 'text': 'of American Dollar', + 'start': 112, + 'end': 130, + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', } ], - "score": 0.6551724137931034, + 'score': 0.6551724137931034, }, { - "lp": "it-en", - "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", - "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of the British Pound (GBP).", - "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", - "annotations": [], - "score": 1.0, + 'lp': 'it-en', + 'src': 'Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).', + 'mt': 'The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of the British Pound (GBP).', + 'ref': 'The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).', + 'annotations': [], + 'score': 1.0, }, { - "lp": "it-en", - "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", - "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful Indonesian wildlife.", - "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", - "annotations": [ + 'lp': 'it-en', + 'src': 'Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.', + 'mt': 'In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful Indonesian wildlife.', + 'ref': 'The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.', + 'annotations': [ { - "text": "Indonesian", - "start": 115, - "end": 125, - "category": "critical_id9_ne_replaced", - "severity": "major", + 'text': 'Indonesian', + 'start': 115, + 'end': 125, + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', } ], - "score": 0.6153846153846154, + 'score': 0.6153846153846154, }, { - "lp": "it-en", - "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", - "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful African wildlife.", - "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", - "annotations": [], - "score": 1.0, + 'lp': 'it-en', + 'src': 'Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.', + 'mt': 'In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful African wildlife.', + 'ref': 'The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.', + 'annotations': [], + 'score': 1.0, }, ] CONTEXT_TEST_SAMPLES = [ { - "lp": "it-en", - "context_src": None, - "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", - "context_mt": None, - "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", - "context_ref": None, - "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", - "annotations": [], - "score": 1.0, + 'lp': 'it-en', + 'context_src': None, + 'src': "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + 'context_mt': None, + 'mt': 'The East African islands are located in the Indian Ocean, off the east coast of Africa.', + 'context_ref': None, + 'ref': 'The East African Islands are in the Indian Ocean off the eastern coast of Africa.', + 'annotations': [], + 'score': 1.0, }, ] @@ -176,7 +176,7 @@ class TestUnifiedMetricPredict(unittest.TestCase): model = load_from_checkpoint( download_model( - "Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH + 'Unbabel/test-model-whimsical-whisper', saving_directory=DATA_PATH ) ) gpus = 1 if torch.cuda.device_count() > 0 else 0 @@ -184,15 +184,19 @@ class TestUnifiedMetricPredict(unittest.TestCase): @classmethod def tearDownClass(cls): shutil.rmtree( - os.path.join(DATA_PATH, "models--Unbabel--test-model-whimsical-whisper") + os.path.join( + DATA_PATH, 'models--Unbabel--test-model-whimsical-whisper' + ) ) def test_predict(self): - model_output = self.model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) - assert "error_spans" in model_output.metadata - assert "src_scores" in model_output.metadata - assert "ref_scores" in model_output.metadata - assert "unified_scores" in model_output.metadata + model_output = self.model.predict( + TEST_SAMPLES, batch_size=12, gpus=self.gpus + ) + assert 'error_spans' in model_output.metadata + assert 'src_scores' in model_output.metadata + assert 'ref_scores' in model_output.metadata + assert 'unified_scores' in model_output.metadata expected_scores = np.array( [ @@ -222,21 +226,25 @@ def test_length_batching(self): TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=True ) self.assertListEqual( - output_without_length_batching.scores, output_with_length_batching.scores + output_without_length_batching.scores, + output_with_length_batching.scores, ) def test_xcomet_predict(self): model = XCOMETMetric.load_from_checkpoint( checkpoint_path=download_model( - "Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH + 'Unbabel/test-model-whimsical-whisper', + saving_directory=DATA_PATH, ), - map_location=torch.device("cpu"), + map_location=torch.device('cpu'), strict=False, **dict(self.model.hparams), ) model.score_weights = [0.25, 0.25, 0.25, 0.25] - model_output = model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) - assert "mqm_scores" in model_output.metadata + model_output = model.predict( + TEST_SAMPLES, batch_size=12, gpus=self.gpus + ) + assert 'mqm_scores' in model_output.metadata # on XCOMET we cap all scores at 1. and final score is a weighted average of 4 features. expected_scores = np.array( @@ -274,14 +282,20 @@ def test_xcomet_predict(self): # Put all the weight on MQM score. model.score_weights = [0, 0, 0, 1] - model_output = model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) - self.assertListEqual(model_output.scores, model_output.metadata.mqm_scores) + model_output = model.predict( + TEST_SAMPLES, batch_size=12, gpus=self.gpus + ) + self.assertListEqual( + model_output.scores, model_output.metadata.mqm_scores + ) class TestRegressionMetricPredict(unittest.TestCase): model = load_from_checkpoint( - download_model("Unbabel/eamt22-cometinho-da", saving_directory=DATA_PATH) + download_model( + 'Unbabel/eamt22-cometinho-da', saving_directory=DATA_PATH + ) ) gpus = 1 if torch.cuda.device_count() > 0 else 0 From f707efb793104a015b289c8daa5941ede118eb82 Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 13:10:18 +0900 Subject: [PATCH 08/13] Updated code for compatibility with Transformers v5 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ee32177e..c9f0834a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ comet-compare = 'comet.cli.compare:compare_command' comet-mbr = 'comet.cli.mbr:mbr_command' [tool.poetry.dependencies] -python = "^3.10.0" +python = "^3.8.0" sentencepiece = ">=0.2.0" pandas = ">=2.3.3" transformers = ">=4.51.1" From 1c034b745a984cf1afaa957cace7487fbf310387 Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 13:11:09 +0900 Subject: [PATCH 09/13] Updated code for compatibility with Transformers v5 --- comet/cli/train.py | 15 ++++----------- comet/models/base.py | 17 ++++------------- comet/models/multitask/unified_metric.py | 6 ++---- comet/models/ranking/ranking_metric.py | 6 ++---- comet/models/regression/regression_metric.py | 6 ++---- 5 files changed, 14 insertions(+), 36 deletions(-) diff --git a/comet/cli/train.py b/comet/cli/train.py index 854a9705..29d939b7 100644 --- a/comet/cli/train.py +++ b/comet/cli/train.py @@ -36,19 +36,12 @@ import torch from jsonargparse import ActionConfigFile, ArgumentParser, namespace_to_dict from pytorch_lightning import seed_everything -from pytorch_lightning.callbacks import ( - EarlyStopping, - LearningRateMonitor, - ModelCheckpoint, -) +from pytorch_lightning.callbacks import (EarlyStopping, LearningRateMonitor, + ModelCheckpoint) from pytorch_lightning.trainer.trainer import Trainer -from comet.models import ( - RankingMetric, - ReferencelessRegression, - RegressionMetric, - UnifiedMetric, -) +from comet.models import (RankingMetric, ReferencelessRegression, + RegressionMetric, UnifiedMetric) torch.set_float32_matmul_precision('high') diff --git a/comet/models/base.py b/comet/models/base.py index 6bc14b37..f108fd1a 100644 --- a/comet/models/base.py +++ b/comet/models/base.py @@ -29,12 +29,8 @@ import numpy as np import pytorch_lightning as ptl import torch -from torch.utils.data import ( - DataLoader, - RandomSampler, - SequentialSampler, - Subset, -) +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + Subset) from comet.encoders import str2encoder from comet.modules import LayerwiseAttention @@ -43,13 +39,8 @@ from .pooling_utils import average_pooling, max_pooling from .predict_pbar import PredictProgressBar from .predict_writer import CustomWriter -from .utils import ( - OrderedSampler, - Prediction, - Target, - flatten_metadata, - restore_list_order, -) +from .utils import (OrderedSampler, Prediction, Target, flatten_metadata, + restore_list_order) if 'COMET_EMBEDDINGS_CACHE' in os.environ: CACHE_SIZE = int(os.environ['COMET_EMBEDDINGS_CACHE']) diff --git a/comet/models/multitask/unified_metric.py b/comet/models/multitask/unified_metric.py index edaf90b6..cc5375bb 100644 --- a/comet/models/multitask/unified_metric.py +++ b/comet/models/multitask/unified_metric.py @@ -29,10 +29,8 @@ import pandas as pd import torch from torch import nn -from transformers.optimization import ( - Adafactor, - get_constant_schedule_with_warmup, -) +from transformers.optimization import (Adafactor, + get_constant_schedule_with_warmup) from comet.models.base import CometModel from comet.models.metrics import MCCMetric, RegressionMetrics diff --git a/comet/models/ranking/ranking_metric.py b/comet/models/ranking/ranking_metric.py index a4c18526..b252c79a 100644 --- a/comet/models/ranking/ranking_metric.py +++ b/comet/models/ranking/ranking_metric.py @@ -29,10 +29,8 @@ import torch import torch.nn.functional as F from torch import nn -from transformers.optimization import ( - Adafactor, - get_constant_schedule_with_warmup, -) +from transformers.optimization import (Adafactor, + get_constant_schedule_with_warmup) from comet.models.base import CometModel from comet.models.metrics import WMTKendall diff --git a/comet/models/regression/regression_metric.py b/comet/models/regression/regression_metric.py index 6387ad53..930ab991 100644 --- a/comet/models/regression/regression_metric.py +++ b/comet/models/regression/regression_metric.py @@ -25,10 +25,8 @@ import pandas as pd import torch from torch import nn -from transformers.optimization import ( - Adafactor, - get_constant_schedule_with_warmup, -) +from transformers.optimization import (Adafactor, + get_constant_schedule_with_warmup) from comet.models.base import CometModel from comet.models.metrics import RegressionMetrics From daf84e1a2deb4b260fe6583c12cafb1826c6a578 Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 14:15:36 +0900 Subject: [PATCH 10/13] Switched from importlib.metadata to importlib_metadata --- comet/encoders/minilm.py | 7 +++---- comet/models/__init__.py | 5 ++--- pyproject.toml | 14 +++++++------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index baf37f6a..8dfaacdf 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -18,15 +18,14 @@ Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT architecture with an XLMR tokenizer. """ - import importlib_metadata import packaging.version as packaging_version from transformers import BertConfig, BertModel transformers_version = importlib_metadata.distribution('transformers').version -if packaging_version.Version(transformers_version) >= packaging_version.Version( - 'v5.0.0rc0' -): +if packaging_version.Version( + transformers_version +) >= packaging_version.Version('v5.0.0rc0'): from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer else: from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer diff --git a/comet/models/__init__.py b/comet/models/__init__.py index 193617db..652996b5 100644 --- a/comet/models/__init__.py +++ b/comet/models/__init__.py @@ -101,11 +101,10 @@ def load_from_checkpoint( # This is a workaround for the bug reported in version 2.2.4 # issue number #244 try: - from importlib import metadata - + import importlib_metadata import packaging.version as parse_version - comet_version = metadata.distribution('unbabel-comet').version + comet_version = importlib_metadata.distribution('unbabel-comet').version use_softmax = ( parse_version.parse(comet_version) >= parse_version.parse('2.2.4') diff --git a/pyproject.toml b/pyproject.toml index c9f0834a..d43723e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,24 +38,24 @@ comet-mbr = 'comet.cli.mbr:mbr_command' [tool.poetry.dependencies] python = "^3.8.0" sentencepiece = ">=0.2.0" -pandas = ">=2.3.3" -transformers = ">=4.51.1" +pandas = ">=1.4.1" +transformers = ">=4.17" pytorch-lightning = ">=2.0.0" -jsonargparse = ">=3.13.1" -torch = ">=2.6.0" +jsonargparse = ">= 3.13" +torch = ">=1.6.0" numpy = ">=1.20.0" torchmetrics = ">=0.10.2" sacrebleu = ">=2.0.0" -scipy = ">=1.10.0" +scipy = ">=1.5.4" entmax = ">=1.1" -huggingface-hub = ">=0.30.0" -protobuf = ">=4.24.4" +huggingface-hub = ">=0.19.3" importlib-metadata = "^9.0.0" [tool.poetry.dev-dependencies] sphinx-markdown-tables = ">=0.0.15" coverage = ">=5.5" scikit-learn = ">=1.0" +protobuf = ">=4.24.4" [build-system] requires = ["poetry-core>=1.0.0"] From cc113ae71bd70b26ea1a53612fe15d9a58d747bc Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 14:24:45 +0900 Subject: [PATCH 11/13] Dynamic build_inputs_with_special_tokens call --- comet/encoders/base.py | 14 +++++++++++--- comet/encoders/minilm.py | 7 ++++--- comet/models/__init__.py | 4 +++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/comet/encoders/base.py b/comet/encoders/base.py index ebd8c674..35038fdb 100644 --- a/comet/encoders/base.py +++ b/comet/encoders/base.py @@ -308,9 +308,17 @@ def concat_sequences( torch.zeros(len(new_sequence[1:-1]) + 2, dtype=torch.int) ) for j in range(1, len(inputs)): - new_sequence = self.build_inputs_with_special_tokens( - new_sequence[1:-1], concat_input_ids[j][i][1:-1] - ) + if hasattr(self.tokenizer, 'build_inputs_with_special_tokens'): + new_sequence = ( + self.tokenizer.build_inputs_with_special_tokens( + new_sequence[1:-1], concat_input_ids[j][i][1:-1] + ) + ) + else: + new_sequence = self.build_inputs_with_special_tokens( + new_sequence[1:-1], concat_input_ids[j][i][1:-1] + ) + if sum(lengths) > self.max_positions - special_tokens: new_sequence = new_sequence[: self.max_positions] diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index 8dfaacdf..baf37f6a 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -18,14 +18,15 @@ Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT architecture with an XLMR tokenizer. """ + import importlib_metadata import packaging.version as packaging_version from transformers import BertConfig, BertModel transformers_version = importlib_metadata.distribution('transformers').version -if packaging_version.Version( - transformers_version -) >= packaging_version.Version('v5.0.0rc0'): +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer else: from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer diff --git a/comet/models/__init__.py b/comet/models/__init__.py index 652996b5..39b601ed 100644 --- a/comet/models/__init__.py +++ b/comet/models/__init__.py @@ -104,7 +104,9 @@ def load_from_checkpoint( import importlib_metadata import packaging.version as parse_version - comet_version = importlib_metadata.distribution('unbabel-comet').version + comet_version = importlib_metadata.distribution( + 'unbabel-comet' + ).version use_softmax = ( parse_version.parse(comet_version) >= parse_version.parse('2.2.4') From c2fdedc09d1d8d865d4fc0649bc29d634b20341b Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 14:54:09 +0900 Subject: [PATCH 12/13] Updated model call in encoders --- comet/encoders/bert.py | 10 +++++++--- comet/encoders/xlmr.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index 53925573..626eb045 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -58,14 +58,14 @@ def __init__( ) if load_pretrained_weights: self.model = BertModel.from_pretrained( - pretrained_model, add_pooling_layer=True + pretrained_model, add_pooling_layer=False ) else: self.model = BertModel( BertConfig.from_pretrained( pretrained_model, local_files_only=local_files_only ), - add_pooling_layer=True, + add_pooling_layer=False, ) self.model.encoder.output_hidden_states = True @@ -189,7 +189,11 @@ def forward( return_dict=False, ) - last_hidden_states, pooler_output, all_layers = output + if len(output) < 3: + last_hidden_states, all_layers = output + pooler_output = None + else: + last_hidden_states, pooler_output, all_layers = output return { 'sentemb': pooler_output, diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index dca310c4..4419715e 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -112,7 +112,7 @@ def forward( return_dict=False, ) - if len(output) == 2: + if len(output) < 3: last_hidden_states, all_layers = output else: last_hidden_states, _, all_layers = output From 9f23f0ef65817844b230861b9b601cc0e9f6353e Mon Sep 17 00:00:00 2001 From: Dania Moriazi Date: Sun, 24 May 2026 15:14:48 +0900 Subject: [PATCH 13/13] Added comments --- comet/encoders/base.py | 5 +++++ comet/encoders/bert.py | 7 +++++-- comet/encoders/minilm.py | 2 ++ comet/encoders/rembert.py | 1 + comet/encoders/xlmr.py | 3 +++ comet/encoders/xlmr_xl.py | 2 ++ 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/comet/encoders/base.py b/comet/encoders/base.py index 35038fdb..37cefa6e 100644 --- a/comet/encoders/base.py +++ b/comet/encoders/base.py @@ -308,6 +308,8 @@ def concat_sequences( torch.zeros(len(new_sequence[1:-1]) + 2, dtype=torch.int) ) for j in range(1, len(inputs)): + # Handles TokenizersBackend (transformers v5) and PretrainedTokenizerFast (transformers v4). + # Transformers v4 if hasattr(self.tokenizer, 'build_inputs_with_special_tokens'): new_sequence = ( self.tokenizer.build_inputs_with_special_tokens( @@ -315,6 +317,7 @@ def concat_sequences( ) ) else: + # Transformers v5 new_sequence = self.build_inputs_with_special_tokens( new_sequence[1:-1], concat_input_ids[j][i][1:-1] ) @@ -360,6 +363,8 @@ def concat_sequences( return encoder_input, lengths, max_len + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. + # To be overrided def build_inputs_with_special_tokens( self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None ) -> list[int]: diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index 626eb045..12d64648 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -25,13 +25,14 @@ import torch from transformers import BertConfig, BertModel +# Handles tokenizer imports for both transformers v4 and v5. transformers_version = importlib_metadata.distribution('transformers').version if packaging_version.Version(transformers_version) >= packaging_version.Version( 'v5.0.0rc0' ): from transformers import BertTokenizer as BertTokenizer else: - from transformers import BertTokenizerFast as BertTokenizer + from transformers import BertTokenizer as BertTokenizer from comet.encoders.base import Encoder @@ -189,9 +190,10 @@ def forward( return_dict=False, ) + # ModelOutput no longer includes pooler_output if model is initialised with `add_pooling_layer=False` if len(output) < 3: last_hidden_states, all_layers = output - pooler_output = None + pooler_output = None # Since `add_pooling_layer=False`, pooler_output would be None else: last_hidden_states, pooler_output, all_layers = output @@ -202,6 +204,7 @@ def forward( 'attention_mask': attention_mask, } + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index baf37f6a..88eac56f 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -23,6 +23,7 @@ import packaging.version as packaging_version from transformers import BertConfig, BertModel +# Handles tokenizer imports for both transformers v4 and v5. transformers_version = importlib_metadata.distribution('transformers').version if packaging_version.Version(transformers_version) >= packaging_version.Version( 'v5.0.0rc0' @@ -89,6 +90,7 @@ def from_pretrained( pretrained_model, load_pretrained_weights, local_files_only ) + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] diff --git a/comet/encoders/rembert.py b/comet/encoders/rembert.py index ae0e0109..67957cd0 100644 --- a/comet/encoders/rembert.py +++ b/comet/encoders/rembert.py @@ -23,6 +23,7 @@ import packaging.version as packaging_version from transformers import RemBertConfig, RemBertModel +# Handles tokenizer imports for both transformers v4 and v5. transformers_version = importlib_metadata.distribution('transformers').version if packaging_version.Version(transformers_version) >= packaging_version.Version( 'v5.0.0rc0' diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index 4419715e..7b6bdd49 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -25,6 +25,7 @@ import torch from transformers import XLMRobertaConfig, XLMRobertaModel +# Handles tokenizer imports for both transformers v4 and v5. transformers_version = importlib_metadata.distribution('transformers').version if packaging_version.parse(transformers_version) >= packaging_version.parse( 'v5.0.0rc0' @@ -112,6 +113,7 @@ def forward( return_dict=False, ) + # ModelOutput no longer includes pooler_output if model is initialised with `add_pooling_layer=False` if len(output) < 3: last_hidden_states, all_layers = output else: @@ -124,6 +126,7 @@ def forward( 'attention_mask': attention_mask, } + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id] diff --git a/comet/encoders/xlmr_xl.py b/comet/encoders/xlmr_xl.py index 57db132d..ea358a6d 100644 --- a/comet/encoders/xlmr_xl.py +++ b/comet/encoders/xlmr_xl.py @@ -22,6 +22,7 @@ import packaging.version as packaging_version from transformers import XLMRobertaXLConfig, XLMRobertaXLModel +# Handles tokenizer imports for both transformers v4 and v5. transformers_version = importlib_metadata.distribution('transformers').version if packaging_version.Version(transformers_version) >= packaging_version.Version( 'v5.0.0rc0' @@ -89,6 +90,7 @@ def from_pretrained( pretrained_model, load_pretrained_weights, local_files_only ) + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): cls = [self.tokenizer.cls_token_id] sep = [self.tokenizer.sep_token_id]