diff --git a/README.md b/README.md index b90bb73a..33909bcf 100644 --- a/README.md +++ b/README.md @@ -316,4 +316,4 @@ If you use COMET please cite our work **and don't forget to say which model you - [Unbabel's Participation in the WMT20 Metrics Shared Task](https://aclanthology.org/2020.wmt-1.101/) -- [COMET: A Neural Framework for MT Evaluation](https://www.aclweb.org/anthology/2020.emnlp-main.213) +- [COMET: A Neural Framework for MT Evaluation](https://www.aclweb.org/anthology/2020.emnlp-main.213) \ No newline at end of file diff --git a/comet/__init__.py b/comet/__init__.py index 5f8b64b6..dc511745 100644 --- a/comet/__init__.py +++ b/comet/__init__.py @@ -16,11 +16,11 @@ import logging -from .models import load_from_checkpoint, download_model +from .models import download_model, load_from_checkpoint -logging.basicConfig(level=logging.INFO, format="%(message)s") +logging.basicConfig(level=logging.INFO, format='%(message)s') logger = logging.getLogger(__name__) -__version__ = "2.2.7" -__copyright__ = "2020 Unbabel. All rights reserved." +__version__ = '2.2.7' +__copyright__ = '2020 Unbabel. All rights reserved.' diff --git a/comet/cli/compare.py b/comet/cli/compare.py index d154dd8f..1d753509 100644 --- a/comet/cli/compare.py +++ b/comet/cli/compare.py @@ -40,7 +40,7 @@ default: 0.4) --t_test_alternative T_TEST_ALTERNATIVE Alternative hypothesis from scipy.stats.ttest_rel. The - following options are available: 'two-sided', 'less', + following options are available: 'two-sided', 'less', 'greater'. Defaults to 'less' (type: str, default: two-sided) --to_json TO_JSON Exports results to a json file. (type: str, default: "") --model MODEL COMET model to be used. (type: str, default: wmt20-comet-da) @@ -57,6 +57,7 @@ (default: False) --print_cache_info Print information about COMET cache. (default: False) """ + import json import os from itertools import combinations @@ -88,36 +89,38 @@ def display_statistical_results(data: Statistical_test_info) -> None: Args: data (Statistical_test_info): Stats to be printed out. """ - print("==========================") - print("x_name:", data["x_name"].rel_path) - print("y_name:", data["y_name"].rel_path) + print('==========================') + print('x_name:', data['x_name'].rel_path) + print('y_name:', data['y_name'].rel_path) - print("\nBootstrap Resampling Results:") - for k, v in data["bootstrap_resampling"].items(): - print("{}:\t{:.4f}".format(k, v)) + print('\nBootstrap Resampling Results:') + for k, v in data['bootstrap_resampling'].items(): + print('{}:\t{:.4f}'.format(k, v)) - print("\nPaired T-Test Results:") - for k, v in data["paired_t-test"].items(): - print("{}:\t{:.4f}".format(k, v)) + print('\nPaired T-Test Results:') + for k, v in data['paired_t-test'].items(): + print('{}:\t{:.4f}'.format(k, v)) - x_seg_scores = data["bootstrap_resampling"]["x-mean"] - y_seg_scores = data["bootstrap_resampling"]["y-mean"] + x_seg_scores = data['bootstrap_resampling']['x-mean'] + y_seg_scores = data['bootstrap_resampling']['y-mean'] best_system = ( - data["x_name"].rel_path + data['x_name'].rel_path if x_seg_scores > y_seg_scores - else data["y_name"].rel_path + else data['y_name'].rel_path ) worse_system = ( - data["x_name"].rel_path + data['x_name'].rel_path if x_seg_scores < y_seg_scores - else data["y_name"].rel_path + else data['y_name'].rel_path ) - if data["paired_t-test"]["p_value"] <= 0.05: - print("Null hypothesis rejected according to t-test.") - print("Scores differ significantly across samples.") - print(f"{best_system} outperforms {worse_system}.") + if data['paired_t-test']['p_value'] <= 0.05: + print('Null hypothesis rejected according to t-test.') + print('Scores differ significantly across samples.') + print(f'{best_system} outperforms {worse_system}.') else: - print("Null hypothesis can't be rejected.\nBoth systems have equal averages.") + print( + "Null hypothesis can't be rejected.\nBoth systems have equal averages." + ) def t_tests_summary( @@ -136,14 +139,14 @@ def t_tests_summary( name2id = {name: i for i, name in enumerate(translations)} grid = [[None] * n for name in translations] for t_test in t_test_results: - p_value = t_test["paired_t-test"]["p_value"] - x_id = name2id[t_test["x_name"]] - y_id = name2id[t_test["y_name"]] + p_value = t_test['paired_t-test']['p_value'] + x_id = name2id[t_test['x_name']] + y_id = name2id[t_test['y_name']] grid[x_id][y_id] = False grid[y_id][x_id] = False if p_value < threshold_p_value: - x_seg_scores = t_test["bootstrap_resampling"]["x-mean"] - y_seg_scores = t_test["bootstrap_resampling"]["y-mean"] + x_seg_scores = t_test['bootstrap_resampling']['x-mean'] + y_seg_scores = t_test['bootstrap_resampling']['y-mean'] if x_seg_scores > y_seg_scores: grid[x_id][y_id] = True else: @@ -152,17 +155,20 @@ def t_tests_summary( # Add the row's name aka the system's name. grid = [(name,) + tuple(row) for name, row in zip(translations, grid)] - print("Summary") - print("If system_x is better than system_y then:") + print('Summary') + print('If system_x is better than system_y then:') print( - f"Null hypothesis rejected according to t-test with p_value={threshold_p_value}." + f'Null hypothesis rejected according to t-test with p_value={threshold_p_value}.' ) - print("Scores differ significantly across samples.") - print(tabulate(grid, headers=("system_x \ system_y",) + translations)) + print('Scores differ significantly across samples.') + print(tabulate(grid, headers=('system_x \ system_y',) + translations)) def calculate_bootstrap( - x_sys_scores: np.ndarray, y_sys_scores: np.ndarray, x_name: Path_fr, y_name: Path_fr + x_sys_scores: np.ndarray, + y_sys_scores: np.ndarray, + x_name: Path_fr, + y_name: Path_fr, ) -> Statistical_test_info: """Calculate bootstrap score, wins and ties for a system pair. @@ -182,14 +188,14 @@ def calculate_bootstrap( x_wins = float(len(delta[delta >= EPS])) y_wins = float(len(delta[delta <= -EPS])) return { - "x_name": x_name, - "y_name": y_name, - "bootstrap_resampling": { - "x-mean": float(np.mean(x_sys_scores)), - "y-mean": float(np.mean(y_sys_scores)), - "ties (%)": ties / num_splits, - "x_wins (%)": x_wins / num_splits, - "y_wins (%)": y_wins / num_splits, + 'x_name': x_name, + 'y_name': y_name, + 'bootstrap_resampling': { + 'x-mean': float(np.mean(x_sys_scores)), + 'y-mean': float(np.mean(y_sys_scores)), + 'ties (%)': ties / num_splits, + 'x_wins (%)': x_wins / num_splits, + 'y_wins (%)': y_wins / num_splits, }, } @@ -206,7 +212,9 @@ def pairwise_bootstrap( Return: Generator(Statistical_test_info): bootstrap resampling stats between systems. """ - assert sys_scores.shape[0] == len(systems), "Each system should have its sys_score." + assert sys_scores.shape[0] == len( + systems + ), 'Each system should have its sys_score.' pairs = combinations(zip(systems, sys_scores), 2) for (x_name, x_sys_scores), (y_name, y_sys_scores) in pairs: @@ -271,7 +279,7 @@ def score(cfg: Namespace, systems: List[Dict[str, List[str]]]) -> np.ndarray: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="auto", + accelerator='auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) @@ -286,16 +294,18 @@ def score(cfg: Namespace, systems: List[Dict[str, List[str]]]) -> np.ndarray: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="cpu" if cfg.gpus == 0 else "auto", + accelerator='cpu' if cfg.gpus == 0 else 'auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) seg_scores += outputs.scores - n = len(systems[0]["src"]) + n = len(systems[0]['src']) # [grouper](https://docs.python.org/3/library/itertools.html#itertools-recipes) seg_scores = list(zip(*[iter(seg_scores)] * n)) - seg_scores = np.array(seg_scores, dtype="float32") # num_systems x num_translations + seg_scores = np.array( + seg_scores, dtype='float32' + ) # num_systems x num_translations return seg_scores @@ -308,111 +318,115 @@ def get_cfg() -> Namespace: parser = ArgumentParser( description="Command for comparing multiple MT systems' translations." ) - parser.add_argument("-s", "--sources", type=Path_fr) - parser.add_argument("-r", "--references", type=Path_fr) - parser.add_argument("-t", "--translations", nargs="*", type=Path_fr) - parser.add_argument("-d", "--sacrebleu_dataset", type=str) - parser.add_argument("--batch_size", type=int, default=8) - parser.add_argument("--gpus", type=int, default=1) + parser.add_argument('-s', '--sources', type=Path_fr) + parser.add_argument('-r', '--references', type=Path_fr) + parser.add_argument('-t', '--translations', nargs='*', type=Path_fr) + parser.add_argument('-d', '--sacrebleu_dataset', type=str) + parser.add_argument('--batch_size', type=int, default=8) + parser.add_argument('--gpus', type=int, default=1) parser.add_argument( - "--quiet", action="store_true", help="Sets all loggers to ERROR level." + '--quiet', action='store_true', help='Sets all loggers to ERROR level.' ) parser.add_argument( - "--only_system", action="store_true", help="Prints only the final system score." + '--only_system', + action='store_true', + help='Prints only the final system score.', ) parser.add_argument( - "--num_splits", + '--num_splits', type=int, default=300, - help="Number of random partitions used in Bootstrap resampling.", + help='Number of random partitions used in Bootstrap resampling.', ) parser.add_argument( - "--sample_ratio", + '--sample_ratio', type=float, default=0.4, - help="Percentage of the testset to use in each split.", + help='Percentage of the testset to use in each split.', ) parser.add_argument( - "--t_test_alternative", + '--t_test_alternative', type=str, - default="two-sided", + default='two-sided', help=( - "Alternative hypothesis from scipy.stats.ttest_rel. The following options" + 'Alternative hypothesis from scipy.stats.ttest_rel. The following options' + " are available: 'two-sided', 'less', 'greater'. Defaults to 'two-sided'" ), ) parser.add_argument( - "--to_json", + '--to_json', type=str, - default="", - help="Exports results to a json file.", + default='', + help='Exports results to a json file.', ) parser.add_argument( - "--model", + '--model', type=str, required=False, - default="Unbabel/wmt22-comet-da", - help="COMET model to be used.", + default='Unbabel/wmt22-comet-da', + help='COMET model to be used.', ) parser.add_argument( - "--model_storage_path", + '--model_storage_path', help=( - "Path to the directory where models will be stored. " - + "By default its saved in ~/.cache/torch/unbabel_comet/" + 'Path to the directory where models will be stored. ' + + 'By default its saved in ~/.cache/torch/unbabel_comet/' ), default=None, ) parser.add_argument( - "--num_workers", - help="Number of workers to use when loading data.", + '--num_workers', + help='Number of workers to use when loading data.', type=int, default=None, ) parser.add_argument( - "--disable_cache", - action="store_true", + '--disable_cache', + action='store_true', help=( - "Disables sentence embeddings caching." - + " This makes inference slower but saves memory." + 'Disables sentence embeddings caching.' + + ' This makes inference slower but saves memory.' ), ) parser.add_argument( - "--disable_length_batching", - action="store_true", - help="Disables length batching. This makes inference slower.", + '--disable_length_batching', + action='store_true', + help='Disables length batching. This makes inference slower.', ) parser.add_argument( - "--print_cache_info", - action="store_true", - help="Print information about COMET cache.", + '--print_cache_info', + action='store_true', + help='Print information about COMET cache.', ) cfg = parser.parse_args() if cfg.sources is None and cfg.sacrebleu_dataset is None: - parser.error(f"You must specify a source (-s) or a sacrebleu dataset (-d)") + parser.error( + f'You must specify a source (-s) or a sacrebleu dataset (-d)' + ) if cfg.sacrebleu_dataset is not None: if cfg.references is not None or cfg.sources is not None: parser.error( - f"Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)" + f'Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)' ) try: - testset, langpair = cfg.sacrebleu_dataset.rsplit(":", maxsplit=1) + testset, langpair = cfg.sacrebleu_dataset.rsplit(':', maxsplit=1) cfg.sources = Path_fr(get_source_file(testset, langpair)) cfg.references = Path_fr(get_reference_files(testset, langpair)[0]) except ValueError: parser.error( - "SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en" + 'SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en' ) except Exception as e: import sys - print("SacreBLEU error:", e, file=sys.stderr) + print('SacreBLEU error:', e, file=sys.stderr) sys.exit(1) - if cfg.model.endswith(".ckpt") and os.path.exists(cfg.model): + if cfg.model.endswith('.ckpt') and os.path.exists(cfg.model): cfg.model_path = cfg.model else: @@ -434,7 +448,9 @@ def compare_command() -> None: if model.requires_references() and (cfg.references is None): parser.error( - "{} requires -r/--references or -d/--sacrebleu_dataset.".format(cfg.model) + '{} requires -r/--references or -d/--sacrebleu_dataset.'.format( + cfg.model + ) ) references = cfg.references if cfg.references is not None else None if not cfg.disable_cache: @@ -443,24 +459,27 @@ def compare_command() -> None: if cfg.print_cache_info: print(model.retrieve_sentence_embedding.cache_info()) - assert len(cfg.translations) > 1, "You must provide at least 2 translation files" + assert ( + len(cfg.translations) > 1 + ), 'You must provide at least 2 translation files' - with open(cfg.sources(), encoding="utf-8") as fp: + with open(cfg.sources(), encoding='utf-8') as fp: sources = [line.strip() for line in fp.readlines()] translations = [] for system in cfg.translations: - with open(system, mode="r", encoding="utf-8") as fp: + with open(system, mode='r', encoding='utf-8') as fp: translations.append([line.strip() for line in fp.readlines()]) if cfg.references is not None: - with open(cfg.references(), encoding="utf-8") as fp: + with open(cfg.references(), encoding='utf-8') as fp: references = [line.strip() for line in fp.readlines()] systems = [ - {"src": sources, "mt": system, "ref": references} for system in translations + {'src': sources, 'mt': system, 'ref': references} + for system in translations ] else: - systems = [{"src": sources, "mt": system} for system in translations] + systems = [{'src': sources, 'mt': system} for system in translations] seg_scores = score(cfg, systems) population_size = seg_scores.shape[1] @@ -478,28 +497,30 @@ def compare_command() -> None: x_seg_scores, y_seg_scores, alternative=cfg.t_test_alternative ) for res in results: - if res["x_name"] == x_name and res["y_name"] == y_name: - res["paired_t-test"] = { - "statistic": ttest_result.statistic, - "p_value": ttest_result.pvalue, + if res['x_name'] == x_name and res['y_name'] == y_name: + res['paired_t-test'] = { + 'statistic': ttest_result.statistic, + 'p_value': ttest_result.pvalue, } info = { - "model": cfg.model, - "statistical_results": results, - "source": sources, - "translations": [ + 'model': cfg.model, + 'statistical_results': results, + 'source': sources, + 'translations': [ { - "name": name, - "mt": trans, - "scores": scores.tolist(), + 'name': name, + 'mt': trans, + 'scores': scores.tolist(), } - for name, trans, scores in zip(cfg.translations, translations, seg_scores) + for name, trans, scores in zip( + cfg.translations, translations, seg_scores + ) ], } if references is not None: - info["reference"] = references + info['reference'] = references for data in results: display_statistical_results(data) @@ -507,11 +528,11 @@ def compare_command() -> None: print() t_tests_summary(results, tuple(cfg.translations)) - if cfg.to_json != "": - with open(cfg.to_json, "w", encoding="utf-8") as outfile: + if cfg.to_json != '': + with open(cfg.to_json, 'w', encoding='utf-8') as outfile: json.dump(info, outfile, ensure_ascii=False, indent=4) - print("Predictions saved in: {}.".format(cfg.to_json)) + print('Predictions saved in: {}.'.format(cfg.to_json)) -if __name__ == "__main__": +if __name__ == '__main__': compare_command() diff --git a/comet/cli/mbr.py b/comet/cli/mbr.py index f7b61eb0..7fa53fd5 100644 --- a/comet/cli/mbr.py +++ b/comet/cli/mbr.py @@ -36,7 +36,7 @@ applying MBR. Disabled by default. (type: int, default: 0) --qe_model QE_MODEL Reference Free model used for reranking before MBR. (type: str, default: Unbabel/wmt23-cometkiwi-da-xl) - --model MODEL COMET model to be used. + --model MODEL COMET model to be used. (type: str, default: Unbabel/wmt23-comet-da-xl) --model_storage_path MODEL_STORAGE_PATH Path to the directory where models will be stored. By default @@ -44,6 +44,7 @@ -o OUTPUT, --output OUTPUT Best candidates after running MBR decoding. (required, type: str) """ + import os from typing import List, Tuple @@ -57,6 +58,7 @@ torch.set_float32_matmul_precision('high') + def build_embeddings( sources: List[str], translations: List[str], @@ -87,8 +89,8 @@ def build_embeddings( src_embeddings = [] with torch.no_grad(): for batch in src_inputs: - input_ids = batch["input_ids"].to(model.device) - attention_mask = batch["attention_mask"].to(model.device) + input_ids = batch['input_ids'].to(model.device) + attention_mask = batch['attention_mask'].to(model.device) src_embeddings.append( model.get_sentence_embedding(input_ids, attention_mask) ) @@ -96,9 +98,11 @@ def build_embeddings( mt_embeddings = [] with torch.no_grad(): - for batch in tqdm(mt_inputs, desc="Encoding sentences...", dynamic_ncols=True): - input_ids = batch["input_ids"].to(model.device) - attention_mask = batch["attention_mask"].to(model.device) + for batch in tqdm( + mt_inputs, desc='Encoding sentences...', dynamic_ncols=True + ): + input_ids = batch['input_ids'].to(model.device) + attention_mask = batch['attention_mask'].to(model.device) mt_embeddings.append( model.get_sentence_embedding(input_ids, attention_mask) ) @@ -108,7 +112,9 @@ def build_embeddings( def mbr_decoding( - src_embeddings: torch.Tensor, mt_embeddings: torch.Tensor, model: RegressionMetric + src_embeddings: torch.Tensor, + mt_embeddings: torch.Tensor, + model: RegressionMetric, ) -> torch.Tensor: """Performs MBR Decoding for each translation for a given source. @@ -127,7 +133,9 @@ def mbr_decoding( with torch.no_grad(): # Loop over all source sentences for i in tqdm( - range(mbr_matrix.shape[0]), desc="MBR Scores...", dynamic_ncols=True + range(mbr_matrix.shape[0]), + desc='MBR Scores...', + dynamic_ncols=True, ): source = src_embeddings[i, :].repeat(num_samples, 1) # Loop over all hypothesis @@ -135,12 +143,15 @@ def mbr_decoding( translation = mt_embeddings[i, j, :].repeat(num_samples, 1) # Score current hypothesis against all others pseudo_refs = mt_embeddings[i, :] - scores = model.estimate(source, translation, pseudo_refs)["score"] + scores = model.estimate(source, translation, pseudo_refs)[ + 'score' + ] scores = torch.cat([scores[0:j], scores[j + 1 :]]) mbr_matrix[i, j] = scores.mean() return mbr_matrix + def rerank_top_k( sources: List[str], translations: List[str], @@ -171,77 +182,81 @@ def rerank_top_k( data = [] for i in range(len(sources)): for j in range(num_samples): - data.append({"src": sources[i], "mt": translations[i][j]}) + data.append({'src': sources[i], 'mt': translations[i][j]}) model_output = qe_model.predict(data, batch_size=batch_size, gpus=gpus) - seg_scores = np.array(model_output.scores).reshape(len(sources), num_samples) + seg_scores = np.array(model_output.scores).reshape( + len(sources), num_samples + ) topk_indices = np.argsort(seg_scores, axis=1) topk_translations = [] for i in range(len(sources)): - topk_translations += [translations[i][idx] for idx in topk_indices[i][::-1][:topk]] + topk_translations += [ + translations[i][idx] for idx in topk_indices[i][::-1][:topk] + ] return topk_translations + def mbr_command() -> None: - parser = ArgumentParser(description="Command for Minimum Bayes Risk Decoding.") - parser.add_argument("-s", "--sources", type=Path_fr, required=True) - parser.add_argument("-t", "--translations", type=Path_fr, required=True) - parser.add_argument("--num_samples", type=int, required=True) - parser.add_argument("--batch_size", type=int, default=32) - parser.add_argument("--gpus", type=int, default=1) + parser = ArgumentParser( + description='Command for Minimum Bayes Risk Decoding.' + ) + parser.add_argument('-s', '--sources', type=Path_fr, required=True) + parser.add_argument('-t', '--translations', type=Path_fr, required=True) + parser.add_argument('--num_samples', type=int, required=True) + parser.add_argument('--batch_size', type=int, default=32) + parser.add_argument('--gpus', type=int, default=1) parser.add_argument( - "--rerank_top_k", + '--rerank_top_k', type=int, default=0, help=( - "Chooses the topK candidates according to --qe_model before applying MBR." - + " Disabled by default." + 'Chooses the topK candidates according to --qe_model before applying MBR.' + + ' Disabled by default.' ), ) parser.add_argument( - "--qe_model", + '--qe_model', type=str, required=False, - default="Unbabel/wmt22-cometkiwi-da", - help="Reference Free model used for reranking before MBR.", + default='Unbabel/wmt22-cometkiwi-da', + help='Reference Free model used for reranking before MBR.', ) parser.add_argument( - "--model", + '--model', type=str, required=False, - default="Unbabel/wmt22-comet-da", - help="COMET model to be used.", + default='Unbabel/wmt22-comet-da', + help='COMET model to be used.', ) parser.add_argument( - "--model_storage_path", + '--model_storage_path', help=( - "Path to the directory where models will be stored. " - + "By default its saved in ~/.cache/torch/unbabel_comet/" + 'Path to the directory where models will be stored. ' + + 'By default its saved in ~/.cache/torch/unbabel_comet/' ), default=None, ) parser.add_argument( - "-o", - "--output", + '-o', + '--output', type=str, required=True, - help="Best candidates after running MBR decoding.", + help='Best candidates after running MBR decoding.', ) cfg = parser.parse_args() - with open(cfg.sources(), encoding="utf-8") as fp: + with open(cfg.sources(), encoding='utf-8') as fp: sources = [line.strip() for line in fp.readlines()] - with open(cfg.translations(), encoding="utf-8") as fp: + with open(cfg.translations(), encoding='utf-8') as fp: translations = [line.strip() for line in fp.readlines()] num_samples = cfg.num_samples # Running QE reranking before MBR! if cfg.rerank_top_k > 0: - if ( - cfg.qe_model.endswith(".ckpt") - and os.path.exists(cfg.qe_model) - ): + if cfg.qe_model.endswith('.ckpt') and os.path.exists(cfg.qe_model): qe_model_path = cfg.qe_model else: qe_model_path = download_model( @@ -249,34 +264,42 @@ def mbr_command() -> None: ) assert ( cfg.rerank_top_k < cfg.num_samples - ), "--rerank_top_k needs to be smaller than number of candidates provided!" + ), '--rerank_top_k needs to be smaller than number of candidates provided!' model = load_from_checkpoint(qe_model_path) assert ( not model.requires_references() - ), "--qe_model expects a Reference Free model!" + ), '--qe_model expects a Reference Free model!' translations = rerank_top_k( - sources, translations, model, cfg.batch_size, cfg.gpus, cfg.num_samples, cfg.rerank_top_k + sources, + translations, + model, + cfg.batch_size, + cfg.gpus, + cfg.num_samples, + cfg.rerank_top_k, ) num_samples = cfg.rerank_top_k - if cfg.model.endswith(".ckpt") and os.path.exists(cfg.model): + if cfg.model.endswith('.ckpt') and os.path.exists(cfg.model): model_path = cfg.model else: - model_path = download_model(cfg.model, saving_directory=cfg.model_storage_path) - + model_path = download_model( + cfg.model, saving_directory=cfg.model_storage_path + ) + model = load_from_checkpoint(model_path) model.eval() model.cuda() model.half() - + if not isinstance(model, RegressionMetric): raise Exception( - "Invalid model ({}). MBR command only works with Reference-based Regression models!".format( + 'Invalid model ({}). MBR command only works with Reference-based Regression models!'.format( model.__class__.__name__ ) ) - + src_embeddings, mt_embeddings = build_embeddings( sources, translations, model, cfg.batch_size ) @@ -293,10 +316,10 @@ def mbr_command() -> None: best_cand_idx = torch.argmax(mbr_matrix[i, :]) best_candidates.append(samples[best_cand_idx]) - with open(cfg.output, "w", encoding="utf-8") as fp: + with open(cfg.output, 'w', encoding='utf-8') as fp: for sample in best_candidates: - fp.write(sample + "\n") + fp.write(sample + '\n') -if __name__ == "__main__": +if __name__ == '__main__': mbr_command() diff --git a/comet/cli/score.py b/comet/cli/score.py index 64323674..6b0e1e51 100644 --- a/comet/cli/score.py +++ b/comet/cli/score.py @@ -39,15 +39,16 @@ Path to the directory where models will be stored. By default its saved in ~/.cache/torch/unbabel_comet/ (default: null) --num_workers NUM_WORKERS - Number of workers to use when loading data. (type: int, + Number of workers to use when loading data. (type: int, default: null) --disable_cache Disables sentence embeddings caching. This makes inference slower but saves memory. (default: False) --disable_length_batching - Disables length batching. This makes inference slower. + Disables length batching. This makes inference slower. (default: False) --print_cache_info Print information about COMET cache. (default: False) """ + import itertools import json import logging @@ -65,134 +66,147 @@ torch.set_float32_matmul_precision('high') + def score_command() -> None: - parser = ArgumentParser(description="Command for scoring MT systems.") - parser.add_argument("-s", "--sources", type=Path_fr) - parser.add_argument("-t", "--translations", type=Path_fr, nargs="+") - parser.add_argument("-r", "--references", type=Path_fr) - parser.add_argument("-d", "--sacrebleu_dataset", type=str) - parser.add_argument("--batch_size", type=int, default=16) - parser.add_argument("--gpus", type=int, default=1) + parser = ArgumentParser(description='Command for scoring MT systems.') + parser.add_argument('-s', '--sources', type=Path_fr) + parser.add_argument('-t', '--translations', type=Path_fr, nargs='+') + parser.add_argument('-r', '--references', type=Path_fr) + parser.add_argument('-d', '--sacrebleu_dataset', type=str) + parser.add_argument('--batch_size', type=int, default=16) + parser.add_argument('--gpus', type=int, default=1) parser.add_argument( - "--quiet", action="store_true", help="Sets all loggers to ERROR level." + '--quiet', action='store_true', help='Sets all loggers to ERROR level.' ) parser.add_argument( - "--enable-context", action="store_true", help="Enables contextual extension of COMET on inputs preprocessed with context information." + '--enable-context', + action='store_true', + help='Enables contextual extension of COMET on inputs preprocessed with context information.', ) parser.add_argument( - "--only_system", action="store_true", help="Prints only the final system score." + '--only_system', + action='store_true', + help='Prints only the final system score.', ) parser.add_argument( - "--to_json", + '--to_json', type=str, - default="", - help="Exports results to a json file.", + default='', + help='Exports results to a json file.', ) parser.add_argument( - "--model", + '--model', type=str, required=False, - default="Unbabel/wmt22-comet-da", - help="COMET model to be used.", + default='Unbabel/wmt22-comet-da', + help='COMET model to be used.', ) parser.add_argument( - "--model_storage_path", + '--model_storage_path', help=( - "Path to the directory where models will be stored. " - + "By default its saved in ~/.cache/torch/unbabel_comet/" + 'Path to the directory where models will be stored. ' + + 'By default its saved in ~/.cache/torch/unbabel_comet/' ), default=None, ) parser.add_argument( - "--num_workers", - help="Number of workers to use when loading data.", + '--num_workers', + help='Number of workers to use when loading data.', type=int, default=None, ) parser.add_argument( - "--disable_cache", - action="store_true", - help="Disables sentence embeddings caching. This makes inference slower but saves memory.", + '--disable_cache', + action='store_true', + help='Disables sentence embeddings caching. This makes inference slower but saves memory.', ) parser.add_argument( - "--disable_length_batching", - action="store_true", - help="Disables length batching. This makes inference slower.", + '--disable_length_batching', + action='store_true', + help='Disables length batching. This makes inference slower.', ) parser.add_argument( - "--print_cache_info", - action="store_true", - help="Print information about COMET cache.", + '--print_cache_info', + action='store_true', + help='Print information about COMET cache.', ) cfg = parser.parse_args() if cfg.quiet: - loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] + loggers = [ + logging.getLogger(name) for name in logging.root.manager.loggerDict + ] for logger in loggers: logger.setLevel(logging.ERROR) seed_everything(1) if cfg.sources is None and cfg.sacrebleu_dataset is None: - parser.error(f"You must specify a source (-s) or a sacrebleu dataset (-d)") + parser.error( + f'You must specify a source (-s) or a sacrebleu dataset (-d)' + ) if cfg.sacrebleu_dataset is not None: if cfg.references is not None or cfg.sources is not None: parser.error( - f"Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)" + f'Cannot use sacrebleu datasets (-d) with manually-specified datasets (-s and -r)' ) try: - testset, langpair = cfg.sacrebleu_dataset.rsplit(":", maxsplit=1) + testset, langpair = cfg.sacrebleu_dataset.rsplit(':', maxsplit=1) cfg.sources = Path_fr(get_source_file(testset, langpair)) cfg.references = Path_fr(get_reference_files(testset, langpair)[0]) except ValueError: parser.error( - "SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en" + 'SacreBLEU testset format must be TESTSET:LANGPAIR, e.g., wmt20:de-en' ) except Exception as e: import sys - print("SacreBLEU error:", e, file=sys.stderr) + print('SacreBLEU error:', e, file=sys.stderr) sys.exit(1) - if cfg.model.endswith(".ckpt") and os.path.exists(cfg.model): + if cfg.model.endswith('.ckpt') and os.path.exists(cfg.model): model_path = cfg.model else: - model_path = download_model(cfg.model, saving_directory=cfg.model_storage_path) + model_path = download_model( + cfg.model, saving_directory=cfg.model_storage_path + ) model = load_from_checkpoint(model_path) model.eval() model.half() - + if cfg.enable_context: model.enable_context() if model.requires_references() and (cfg.references is None): parser.error( - "{} requires -r/--references or -d/--sacrebleu_dataset.".format(cfg.model) + '{} requires -r/--references or -d/--sacrebleu_dataset.'.format( + cfg.model + ) ) if not cfg.disable_cache: model.set_embedding_cache() - with open(cfg.sources(), encoding="utf-8") as fp: + with open(cfg.sources(), encoding='utf-8') as fp: sources = [line.strip() for line in fp.readlines()] translations = [] for path_fr in cfg.translations: - with open(path_fr(), encoding="utf-8") as fp: + with open(path_fr(), encoding='utf-8') as fp: translations.append([line.strip() for line in fp.readlines()]) if cfg.references is not None: - with open(cfg.references(), encoding="utf-8") as fp: + with open(cfg.references(), encoding='utf-8') as fp: references = [line.strip() for line in fp.readlines()] data = { - "src": [sources for _ in translations], - "mt": translations, - "ref": [references for _ in translations], + 'src': [sources for _ in translations], + 'mt': translations, + 'ref': [references for _ in translations], } else: - data = {"src": [sources for _ in translations], "mt": translations} + data = {'src': [sources for _ in translations], 'mt': translations} if cfg.gpus > 1: # Flatten all data to score across multiple GPUs @@ -205,16 +219,16 @@ def score_command() -> None: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="auto", + accelerator='auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) seg_scores = outputs.scores - if "metadata" in outputs and "error_spans" in outputs.metadata: + if 'metadata' in outputs and 'error_spans' in outputs.metadata: errors = outputs.metadata.error_spans else: errors = [] - + if len(cfg.translations) > 1: seg_scores = np.array_split(seg_scores, len(cfg.translations)) sys_scores = [sum(split) / len(split) for split in seg_scores] @@ -227,7 +241,9 @@ def score_command() -> None: seg_scores = [ seg_scores, ] - errors = [errors, ] + errors = [ + errors, + ] data = [ np.array(data), ] @@ -245,42 +261,44 @@ def score_command() -> None: batch_size=cfg.batch_size, gpus=cfg.gpus, progress_bar=(not cfg.quiet), - accelerator="cpu" if cfg.gpus == 0 else "auto", + accelerator='cpu' if cfg.gpus == 0 else 'auto', num_workers=cfg.num_workers, length_batching=(not cfg.disable_length_batching), ) seg_scores.append(outputs.scores) sys_scores.append(outputs.system_score) - if "metadata" in outputs and "error_spans" in outputs.metadata: + if 'metadata' in outputs and 'error_spans' in outputs.metadata: errors.append(outputs.metadata.error_spans) data = new_data files = [path_fr.rel_path for path_fr in cfg.translations] - data = {file: system_data.tolist() for file, system_data in zip(files, data)} + data = { + file: system_data.tolist() for file, system_data in zip(files, data) + } for i in range(len(data[files[0]])): # loop over (src, ref) for j in range(len(files)): # loop of system - data[files[j]][i]["COMET"] = seg_scores[j][i] + data[files[j]][i]['COMET'] = seg_scores[j][i] if errors and errors[j] and errors[j][i]: - data[files[j]][i]["errors"] = errors[j][i] - + data[files[j]][i]['errors'] = errors[j][i] + if not cfg.only_system: print( - "{}\tSegment {}\tscore: {:.4f}".format( + '{}\tSegment {}\tscore: {:.4f}'.format( files[j], i, seg_scores[j][i] ) ) for j in range(len(files)): - print("{}\tscore: {:.4f}".format(files[j], sys_scores[j])) + print('{}\tscore: {:.4f}'.format(files[j], sys_scores[j])) - if cfg.to_json != "": - with open(cfg.to_json, "w", encoding="utf-8") as outfile: + if cfg.to_json != '': + with open(cfg.to_json, 'w', encoding='utf-8') as outfile: json.dump(data, outfile, ensure_ascii=False, indent=4) - print("Predictions saved in: {}.".format(cfg.to_json)) + print('Predictions saved in: {}.'.format(cfg.to_json)) if cfg.print_cache_info: print(model.retrieve_sentence_embedding.cache_info()) -if __name__ == "__main__": +if __name__ == '__main__': score_command() diff --git a/comet/cli/train.py b/comet/cli/train.py index 6ccc5193..29d939b7 100644 --- a/comet/cli/train.py +++ b/comet/cli/train.py @@ -28,6 +28,7 @@ comet-train --help ``` """ + import json import logging import warnings @@ -48,31 +49,31 @@ def read_arguments() -> ArgumentParser: - parser = ArgumentParser(description="Command for training COMET models.") + parser = ArgumentParser(description='Command for training COMET models.') parser.add_argument( - "--seed_everything", + '--seed_everything', type=int, default=12, - help="Training Seed.", + help='Training Seed.', ) - parser.add_argument("--cfg", action=ActionConfigFile) - parser.add_subclass_arguments(RegressionMetric, "regression_metric") + parser.add_argument('--cfg', action=ActionConfigFile) + parser.add_subclass_arguments(RegressionMetric, 'regression_metric') parser.add_subclass_arguments( - ReferencelessRegression, "referenceless_regression_metric" + ReferencelessRegression, 'referenceless_regression_metric' ) - parser.add_subclass_arguments(RankingMetric, "ranking_metric") - parser.add_subclass_arguments(UnifiedMetric, "unified_metric") - parser.add_subclass_arguments(EarlyStopping, "early_stopping") - parser.add_subclass_arguments(ModelCheckpoint, "model_checkpoint") - parser.add_subclass_arguments(Trainer, "trainer") + parser.add_subclass_arguments(RankingMetric, 'ranking_metric') + parser.add_subclass_arguments(UnifiedMetric, 'unified_metric') + parser.add_subclass_arguments(EarlyStopping, 'early_stopping') + parser.add_subclass_arguments(ModelCheckpoint, 'model_checkpoint') + parser.add_subclass_arguments(Trainer, 'trainer') parser.add_argument( - "--load_from_checkpoint", - help="Loads a model checkpoint for fine-tuning", + '--load_from_checkpoint', + help='Loads a model checkpoint for fine-tuning', default=None, ) parser.add_argument( - "--strict_load", - action="store_true", + '--strict_load', + action='store_true', help="Strictly enforce that the keys in checkpoint_path match the keys returned by this module's state dict.", ) return parser @@ -86,16 +87,20 @@ def initialize_trainer(configs) -> Trainer: **namespace_to_dict(configs.early_stopping.init_args) ) trainer_args = namespace_to_dict(configs.trainer.init_args) - lr_monitor = LearningRateMonitor(logging_interval="step") - trainer_args["callbacks"] = [early_stop_callback, checkpoint_callback, lr_monitor] - print("TRAINER ARGUMENTS: ") + lr_monitor = LearningRateMonitor(logging_interval='step') + trainer_args['callbacks'] = [ + early_stop_callback, + checkpoint_callback, + lr_monitor, + ] + print('TRAINER ARGUMENTS: ') print(json.dumps(trainer_args, indent=4, default=lambda x: x.__dict__)) trainer = Trainer(**trainer_args) return trainer def initialize_model(configs): - print("MODEL ARGUMENTS: ") + print('MODEL ARGUMENTS: ') if configs.regression_metric is not None: print( json.dumps( @@ -105,7 +110,7 @@ def initialize_model(configs): ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = RegressionMetric.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, @@ -124,48 +129,60 @@ def initialize_model(configs): ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = ReferencelessRegression.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, - **namespace_to_dict(configs.referenceless_regression_metric.init_args), + **namespace_to_dict( + configs.referenceless_regression_metric.init_args + ), ) else: model = ReferencelessRegression( - **namespace_to_dict(configs.referenceless_regression_metric.init_args) + **namespace_to_dict( + configs.referenceless_regression_metric.init_args + ) ) elif configs.ranking_metric is not None: print( json.dumps( - configs.ranking_metric.init_args, indent=4, default=lambda x: x.__dict__ + configs.ranking_metric.init_args, + indent=4, + default=lambda x: x.__dict__, ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = RankingMetric.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, **namespace_to_dict(configs.ranking_metric.init_args), ) else: - model = RankingMetric(**namespace_to_dict(configs.ranking_metric.init_args)) + model = RankingMetric( + **namespace_to_dict(configs.ranking_metric.init_args) + ) elif configs.unified_metric is not None: print( json.dumps( - configs.unified_metric.init_args, indent=4, default=lambda x: x.__dict__ + configs.unified_metric.init_args, + indent=4, + default=lambda x: x.__dict__, ) ) if configs.load_from_checkpoint is not None: - logger.info(f"Loading weights from {configs.load_from_checkpoint}.") + logger.info(f'Loading weights from {configs.load_from_checkpoint}.') model = UnifiedMetric.load_from_checkpoint( checkpoint_path=configs.load_from_checkpoint, strict=configs.strict_load, **namespace_to_dict(configs.unified_metric.init_args), ) else: - model = UnifiedMetric(**namespace_to_dict(configs.unified_metric.init_args)) + model = UnifiedMetric( + **namespace_to_dict(configs.unified_metric.init_args) + ) else: - raise Exception("Model configurations missing!") + raise Exception('Model configurations missing!') return model @@ -181,12 +198,12 @@ def train_command() -> None: # 2 workers per gpu is enough! If set to the number of cpus on this machine # it throws another exception saying its too many workers. warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) -if __name__ == "__main__": +if __name__ == '__main__': train_command() diff --git a/comet/encoders/__init__.py b/comet/encoders/__init__.py index 5e41e879..27b04b8f 100644 --- a/comet/encoders/__init__.py +++ b/comet/encoders/__init__.py @@ -13,14 +13,14 @@ # limitations under the License. from .bert import BERTEncoder from .minilm import MiniLMEncoder -from .xlmr import XLMREncoder from .rembert import RemBERTEncoder +from .xlmr import XLMREncoder from .xlmr_xl import XLMRXLEncoder str2encoder = { - "BERT": BERTEncoder, - "XLM-RoBERTa": XLMREncoder, - "MiniLM": MiniLMEncoder, - "XLM-RoBERTa-XL": XLMRXLEncoder, - "RemBERT": RemBERTEncoder, + 'BERT': BERTEncoder, + 'XLM-RoBERTa': XLMREncoder, + 'MiniLM': MiniLMEncoder, + 'XLM-RoBERTa-XL': XLMRXLEncoder, + 'RemBERT': RemBERTEncoder, } diff --git a/comet/encoders/base.py b/comet/encoders/base.py index 90ed5649..37cefa6e 100644 --- a/comet/encoders/base.py +++ b/comet/encoders/base.py @@ -16,6 +16,7 @@ ==================== Module defining the common interface between all pretrained encoder models. """ + import abc from typing import Dict, List, Optional, Tuple @@ -129,16 +130,16 @@ def align_tokens_and_annotations( ) -> Tuple[List[int], List[int]]: """Inspired by: https://github.com/LightTag/sequence-labeling-with-transformers/""" tokens = tokenized.tokens - aligned_labels = ["O"] * len(tokens) + aligned_labels = ['O'] * len(tokens) for anno in annotations: # A set that stores the token indices of the annotation annotation_token_ix_set = set() - for char_ix in range(anno["start"], anno["end"]): + for char_ix in range(anno['start'], anno['end']): token_ix = tokenized.char_to_token(char_ix) if token_ix is not None: annotation_token_ix_set.add(token_ix) for _, token_ix in enumerate(sorted(annotation_token_ix_set)): - prefix = "I" + prefix = 'I' aligned_labels[token_ix] = f"{prefix}-{anno['severity']}" def get_label_id(item): @@ -148,7 +149,7 @@ def get_label_id(item): label = self.labelset.labels_to_id.get(item) if label is None: raise Exception( - f"{label} does not exist in self.labelset: {self.labelset.labels_to_id.keys()}" + f'{label} does not exist in self.labelset: {self.labelset.labels_to_id.keys()}' ) return label @@ -183,14 +184,16 @@ def subword_tokenize( attention_mask = [[1 for _ in seq] for seq in input_ids] max_length = max([len(l) for l in input_ids]) - input_ids = self.pad_list(input_ids, max_length, self.tokenizer.pad_token_id) + input_ids = self.pad_list( + input_ids, max_length, self.tokenizer.pad_token_id + ) label_ids = self.pad_list(label_ids, max_length, -1) attention_mask = self.pad_list(attention_mask, max_length, 0) return { - "input_ids": torch.tensor(input_ids), - "label_ids": torch.tensor(label_ids), - "attention_mask": torch.tensor(attention_mask), - "offsets": offsets, # Used during inference + 'input_ids': torch.tensor(input_ids), + 'label_ids': torch.tensor(label_ids), + 'attention_mask': torch.tensor(attention_mask), + 'offsets': offsets, # Used during inference } def prepare_sample( @@ -217,7 +220,7 @@ def prepare_sample( else: tokenizer_output = self.tokenizer( sample, - return_tensors="pt", + return_tensors='pt', padding=True, truncation=True, max_length=self.max_positions - 2, @@ -277,9 +280,10 @@ def concat_sequences( concatenated into a single input. """ concat_input_ids = [] + # Remove padding before concatenation for encoder_input in inputs: - input_ids = encoder_input["input_ids"] + input_ids = encoder_input['input_ids'] input_ids = [ x.masked_select(x.ne(self.tokenizer.pad_token_id)).tolist() for x in input_ids.unbind(dim=0) @@ -295,7 +299,7 @@ def concat_sequences( # because that method adds them again lengths = tuple(len(x[i][1:-1]) for x in concat_input_ids) - # self.max_positions = 512 but we need to remove 4 aditional tokens + # self.max_positions = 512 but we need to remove 4 additional tokens # [CLS]...[SEP]...[SEP]...[SEP] special_tokens = 1 + len(inputs) * self.size_separator new_sequence = concat_input_ids[0][i] @@ -304,9 +308,20 @@ def concat_sequences( torch.zeros(len(new_sequence[1:-1]) + 2, dtype=torch.int) ) for j in range(1, len(inputs)): - new_sequence = self.tokenizer.build_inputs_with_special_tokens( - new_sequence[1:-1], concat_input_ids[j][i][1:-1] - ) + # Handles TokenizersBackend (transformers v5) and PretrainedTokenizerFast (transformers v4). + # Transformers v4 + if hasattr(self.tokenizer, 'build_inputs_with_special_tokens'): + new_sequence = ( + self.tokenizer.build_inputs_with_special_tokens( + new_sequence[1:-1], concat_input_ids[j][i][1:-1] + ) + ) + else: + # Transformers v5 + new_sequence = self.build_inputs_with_special_tokens( + new_sequence[1:-1], concat_input_ids[j][i][1:-1] + ) + if sum(lengths) > self.max_positions - special_tokens: new_sequence = new_sequence[: self.max_positions] @@ -315,29 +330,44 @@ def concat_sequences( lengths = [t.shape[0] for t in batch] max_len = max(lengths) padded = [ - self.pad_tensor(t, max_len, self.tokenizer.pad_token_id) for t in batch + self.pad_tensor(t, max_len, self.tokenizer.pad_token_id) + for t in batch ] lengths = torch.tensor(lengths, dtype=torch.long) padded = torch.stack(padded, dim=0).contiguous() attention_mask = torch.arange(max_len)[None, :] < lengths[:, None] if return_label_ids: label_ids = [ - self.pad_tensor(t, max_len, -1) for t in inputs[0]["label_ids"] + self.pad_tensor(t, max_len, -1) for t in inputs[0]['label_ids'] ] label_ids = torch.stack(label_ids, dim=0).contiguous() encoder_input = { - "input_ids": padded, - "attention_mask": attention_mask, - "label_ids": label_ids, - "mt_offsets": inputs[0]["offsets"], # Used during inference + 'input_ids': padded, + 'attention_mask': attention_mask, + 'label_ids': label_ids, + 'mt_offsets': inputs[0]['offsets'], # Used during inference } else: - encoder_input = {"input_ids": padded, "attention_mask": attention_mask} + encoder_input = { + 'input_ids': padded, + 'attention_mask': attention_mask, + } if self.uses_token_type_ids: - token_type_ids = [self.pad_tensor(t, max_len, 1) for t in token_type_ids] + token_type_ids = [ + self.pad_tensor(t, max_len, 1) for t in token_type_ids + ] token_type_ids = torch.stack(token_type_ids, dim=0).contiguous() - encoder_input["token_type_ids"] = token_type_ids + encoder_input['token_type_ids'] = token_type_ids return encoder_input, lengths, max_len + + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. + # To be overrided + def build_inputs_with_special_tokens( + self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None + ) -> list[int]: + if token_ids_1 is None: + return token_ids_0 + return token_ids_0 + token_ids_1 diff --git a/comet/encoders/bert.py b/comet/encoders/bert.py index 753dc70d..12d64648 100644 --- a/comet/encoders/bert.py +++ b/comet/encoders/bert.py @@ -17,10 +17,22 @@ ============== Pretrained BERT encoder from Hugging Face. """ + from typing import Dict, Optional +import importlib_metadata +import packaging.version as packaging_version import torch -from transformers import BertConfig, BertModel, BertTokenizerFast +from transformers import BertConfig, BertModel + +# Handles tokenizer imports for both transformers v4 and v5. +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): + from transformers import BertTokenizer as BertTokenizer +else: + from transformers import BertTokenizer as BertTokenizer from comet.encoders.base import Encoder @@ -42,7 +54,7 @@ def __init__( local_files_only: bool = False, ) -> None: super().__init__() - self.tokenizer = BertTokenizerFast.from_pretrained( + self.tokenizer = BertTokenizer.from_pretrained( pretrained_model, use_fast=True, local_files_only=local_files_only ) if load_pretrained_weights: @@ -108,7 +120,9 @@ def from_pretrained( Returns: Encoder: XLMREncoder object. """ - return BERTEncoder(pretrained_model, load_pretrained_weights, local_files_only) + return BERTEncoder( + pretrained_model, load_pretrained_weights, local_files_only + ) def freeze_embeddings(self) -> None: """Frezees the embedding layer.""" @@ -128,23 +142,23 @@ def layerwise_lr(self, lr: float, decay: float): # Last layer keeps LR opt_parameters = [ { - "params": self.model.encoder.layer[-1].parameters(), - "lr": lr, + 'params': self.model.encoder.layer[-1].parameters(), + 'lr': lr, } ] # Decay at each layer. for i in range(2, self.num_layers): opt_parameters.append( { - "params": self.model.encoder.layer[-i].parameters(), - "lr": lr * decay ** (i - 1), + 'params': self.model.encoder.layer[-i].parameters(), + 'lr': lr * decay ** (i - 1), } ) # Embedding Layer opt_parameters.append( { - "params": self.model.embeddings.parameters(), - "lr": lr * decay ** (self.num_layers), + 'params': self.model.embeddings.parameters(), + 'lr': lr * decay ** (self.num_layers), } ) return opt_parameters @@ -154,7 +168,7 @@ def forward( input_ids: torch.Tensor, attention_mask: torch.Tensor, token_type_ids: Optional[torch.tensor] = None, - **kwargs + **kwargs, ) -> Dict[str, torch.Tensor]: """BERT model forward @@ -168,16 +182,32 @@ def forward( Dict[str, torch.Tensor]: dictionary with 'sentemb', 'wordemb', 'all_layers' and 'attention_mask'. """ - last_hidden_states, pooler_output, all_layers = self.model( + output = self.model( input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask, output_hidden_states=True, return_dict=False, ) + + # ModelOutput no longer includes pooler_output if model is initialised with `add_pooling_layer=False` + if len(output) < 3: + last_hidden_states, all_layers = output + pooler_output = None # Since `add_pooling_layer=False`, pooler_output would be None + else: + last_hidden_states, pooler_output, all_layers = output + return { - "sentemb": pooler_output, - "wordemb": last_hidden_states, - "all_layers": all_layers, - "attention_mask": attention_mask, + 'sentemb': pooler_output, + 'wordemb': last_hidden_states, + 'all_layers': all_layers, + 'attention_mask': attention_mask, } + + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep diff --git a/comet/encoders/minilm.py b/comet/encoders/minilm.py index 35a460d9..88eac56f 100644 --- a/comet/encoders/minilm.py +++ b/comet/encoders/minilm.py @@ -15,10 +15,22 @@ r""" MiniLM Encoder ============== - Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT + Pretrained MiniLM encoder from Microsoft. This encoder uses a BERT architecture with an XLMR tokenizer. """ -from transformers import BertConfig, BertModel, XLMRobertaTokenizerFast + +import importlib_metadata +import packaging.version as packaging_version +from transformers import BertConfig, BertModel + +# Handles tokenizer imports for both transformers v4 and v5. +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): + from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer +else: + from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer from comet.encoders.xlmr import Encoder, XLMREncoder @@ -40,8 +52,10 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = XLMRobertaTokenizerFast.from_pretrained( - "xlm-roberta-base", use_fast=True, local_files_only=local_files_only + self.tokenizer = XLMRobertaTokenizer.from_pretrained( + 'xlm-roberta-base', + use_fast=True, + local_files_only=local_files_only, ) if load_pretrained_weights: self.model = BertModel.from_pretrained(pretrained_model) @@ -75,3 +89,12 @@ def from_pretrained( return MiniLMEncoder( pretrained_model, load_pretrained_weights, local_files_only ) + + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/comet/encoders/rembert.py b/comet/encoders/rembert.py index 003eb857..67957cd0 100644 --- a/comet/encoders/rembert.py +++ b/comet/encoders/rembert.py @@ -15,10 +15,22 @@ r""" RemBERT Encoder =============== - Pretrained RemBERT encoder from Google. This encoder is similar to BERT but uses + Pretrained RemBERT encoder from Google. This encoder is similar to BERT but uses sentencepiece like XLMR. """ -from transformers import RemBertConfig, RemBertModel, RemBertTokenizerFast + +import importlib_metadata +import packaging.version as packaging_version +from transformers import RemBertConfig, RemBertModel + +# Handles tokenizer imports for both transformers v4 and v5. +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): + from transformers import RemBertTokenizer as RemBertTokenizer +else: + from transformers import RemBertTokenizerFast as RemBertTokenizer from comet.encoders.xlmr import Encoder, XLMREncoder @@ -40,7 +52,7 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = RemBertTokenizerFast.from_pretrained( + self.tokenizer = RemBertTokenizer.from_pretrained( pretrained_model, use_fast=True, local_files_only=local_files_only ) if load_pretrained_weights: @@ -84,3 +96,10 @@ def from_pretrained( return RemBERTEncoder( pretrained_model, load_pretrained_weights, local_files_only ) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep diff --git a/comet/encoders/xlmr.py b/comet/encoders/xlmr.py index 13cc15af..7b6bdd49 100644 --- a/comet/encoders/xlmr.py +++ b/comet/encoders/xlmr.py @@ -17,10 +17,22 @@ ============== Pretrained XLM-RoBERTa encoder from Hugging Face. """ + from typing import Dict +import importlib_metadata +import packaging.version as packaging_version import torch -from transformers import XLMRobertaConfig, XLMRobertaModel, XLMRobertaTokenizerFast +from transformers import XLMRobertaConfig, XLMRobertaModel + +# Handles tokenizer imports for both transformers v4 and v5. +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.parse(transformers_version) >= packaging_version.parse( + 'v5.0.0rc0' +): + from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer +else: + from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer from comet.encoders.base import Encoder from comet.encoders.bert import BERTEncoder @@ -43,7 +55,7 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = XLMRobertaTokenizerFast.from_pretrained( + self.tokenizer = XLMRobertaTokenizer.from_pretrained( pretrained_model, local_files_only=local_files_only ) if load_pretrained_weights: @@ -87,20 +99,38 @@ def from_pretrained( Returns: Encoder: XLMREncoder object. """ - return XLMREncoder(pretrained_model, load_pretrained_weights, local_files_only) + return XLMREncoder( + pretrained_model, load_pretrained_weights, local_files_only + ) def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor, **kwargs ) -> Dict[str, torch.Tensor]: - last_hidden_states, _, all_layers = self.model( + output = self.model( input_ids=input_ids, attention_mask=attention_mask, output_hidden_states=True, return_dict=False, ) + + # ModelOutput no longer includes pooler_output if model is initialised with `add_pooling_layer=False` + if len(output) < 3: + last_hidden_states, all_layers = output + else: + last_hidden_states, _, all_layers = output + return { - "sentemb": last_hidden_states[:, 0, :], - "wordemb": last_hidden_states, - "all_layers": all_layers, - "attention_mask": attention_mask, + 'sentemb': last_hidden_states[:, 0, :], + 'wordemb': last_hidden_states, + 'all_layers': all_layers, + 'attention_mask': attention_mask, } + + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/comet/encoders/xlmr_xl.py b/comet/encoders/xlmr_xl.py index edb64ac4..ea358a6d 100644 --- a/comet/encoders/xlmr_xl.py +++ b/comet/encoders/xlmr_xl.py @@ -17,7 +17,19 @@ ============== Pretrained XLM-RoBERTa-XL encoder from Hugging Face. """ -from transformers import XLMRobertaTokenizerFast, XLMRobertaXLConfig, XLMRobertaXLModel + +import importlib_metadata +import packaging.version as packaging_version +from transformers import XLMRobertaXLConfig, XLMRobertaXLModel + +# Handles tokenizer imports for both transformers v4 and v5. +transformers_version = importlib_metadata.distribution('transformers').version +if packaging_version.Version(transformers_version) >= packaging_version.Version( + 'v5.0.0rc0' +): + from transformers import XLMRobertaTokenizer as XLMRobertaTokenizer +else: + from transformers import XLMRobertaTokenizerFast as XLMRobertaTokenizer from comet.encoders.base import Encoder from comet.encoders.xlmr import XLMREncoder @@ -40,7 +52,7 @@ def __init__( local_files_only: bool = False, ) -> None: super(Encoder, self).__init__() - self.tokenizer = XLMRobertaTokenizerFast.from_pretrained( + self.tokenizer = XLMRobertaTokenizer.from_pretrained( pretrained_model, local_files_only=local_files_only ) if load_pretrained_weights: @@ -77,3 +89,12 @@ def from_pretrained( return XLMRXLEncoder( pretrained_model, load_pretrained_weights, local_files_only ) + + # TokenizersBackend does not have a built-in build_inputs_with_special_tokens method. + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + cls = [self.tokenizer.cls_token_id] + sep = [self.tokenizer.sep_token_id] + + if token_ids_1 is None: + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + sep + token_ids_1 + sep diff --git a/comet/models/__init__.py b/comet/models/__init__.py index 8c1835b0..39b601ed 100644 --- a/comet/models/__init__.py +++ b/comet/models/__init__.py @@ -22,20 +22,19 @@ from huggingface_hub import snapshot_download from .base import CometModel +from .download_utils import download_model_legacy from .multitask.unified_metric import UnifiedMetric from .multitask.xcomet_metric import XCOMETMetric from .ranking.ranking_metric import RankingMetric from .regression.referenceless import ReferencelessRegression from .regression.regression_metric import RegressionMetric -from .download_utils import download_model_legacy - str2model = { - "referenceless_regression_metric": ReferencelessRegression, - "regression_metric": RegressionMetric, - "ranking_metric": RankingMetric, - "unified_metric": UnifiedMetric, - "xcomet_metric": XCOMETMetric, + 'referenceless_regression_metric': ReferencelessRegression, + 'regression_metric': RegressionMetric, + 'ranking_metric': RankingMetric, + 'unified_metric': UnifiedMetric, + 'xcomet_metric': XCOMETMetric, } @@ -46,7 +45,9 @@ def download_model( ) -> str: try: model_path = snapshot_download( - repo_id=model, cache_dir=saving_directory, local_files_only=local_files_only + repo_id=model, + cache_dir=saving_directory, + local_files_only=local_files_only, ) except Exception: try: @@ -54,7 +55,9 @@ def download_model( except Exception: raise KeyError(f"Model '{model}' not supported by COMET.") else: - checkpoint_path = os.path.join(*[model_path, "checkpoints", "model.ckpt"]) + checkpoint_path = os.path.join( + *[model_path, 'checkpoints', 'model.ckpt'] + ) return checkpoint_path @@ -83,41 +86,48 @@ def load_from_checkpoint( checkpoint_path = Path(checkpoint_path) if not checkpoint_path.is_file(): - raise Exception(f"Invalid checkpoint path: {checkpoint_path}") + raise Exception(f'Invalid checkpoint path: {checkpoint_path}') parent_folder = checkpoint_path.parents[1] # .parent.parent - hparams_file = parent_folder / "hparams.yaml" + hparams_file = parent_folder / 'hparams.yaml' if hparams_file.is_file(): with open(hparams_file) as yaml_file: hparams = yaml.load(yaml_file.read(), Loader=yaml.FullLoader) - model_class = str2model[hparams["class_identifier"]] - + model_class = str2model[hparams['class_identifier']] + # Check comet version and hparams for layer_transformation # This is a workaround for the bug reported in version 2.2.4 # issue number #244 try: - import pkg_resources - comet_version = pkg_resources.get_distribution("unbabel-comet").version - use_softmax = (pkg_resources.parse_version(comet_version) >= pkg_resources.parse_version("2.2.4") and - hparams.get("layer_transformation") == "sparsemax_patch") + import importlib_metadata + import packaging.version as parse_version + + comet_version = importlib_metadata.distribution( + 'unbabel-comet' + ).version + use_softmax = ( + parse_version.parse(comet_version) + >= parse_version.parse('2.2.4') + and hparams.get('layer_transformation') == 'sparsemax_patch' + ) except: use_softmax = False # Add the override parameter only if needed kwargs = { - "checkpoint_path": checkpoint_path, - "load_pretrained_weights": False, - "hparams_file": hparams_file if reload_hparams else None, - "map_location": torch.device("cpu"), - "strict": strict, - "local_files_only": local_files_only, + 'checkpoint_path': checkpoint_path, + 'load_pretrained_weights': False, + 'hparams_file': hparams_file if reload_hparams else None, + 'map_location': torch.device('cpu'), + 'strict': strict, + 'local_files_only': local_files_only, } if use_softmax: - kwargs["layer_transformation"] = "softmax" - + kwargs['layer_transformation'] = 'softmax' + model = model_class.load_from_checkpoint(**kwargs) return model else: - raise Exception(f"hparams.yaml file is missing from {parent_folder}!") + raise Exception(f'hparams.yaml file is missing from {parent_folder}!') diff --git a/comet/models/base.py b/comet/models/base.py index 9872d2b7..f108fd1a 100644 --- a/comet/models/base.py +++ b/comet/models/base.py @@ -18,6 +18,7 @@ Abstract Model class that implements some of the Pytorch Lightning logic. Extend this class to create new model and metrics within COMET. """ + import abc import logging import os @@ -28,7 +29,8 @@ import numpy as np import pytorch_lightning as ptl import torch -from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Subset +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + Subset) from comet.encoders import str2encoder from comet.modules import LayerwiseAttention @@ -37,16 +39,11 @@ from .pooling_utils import average_pooling, max_pooling from .predict_pbar import PredictProgressBar from .predict_writer import CustomWriter -from .utils import ( - OrderedSampler, - Prediction, - Target, - flatten_metadata, - restore_list_order, -) - -if "COMET_EMBEDDINGS_CACHE" in os.environ: - CACHE_SIZE = int(os.environ["COMET_EMBEDDINGS_CACHE"]) +from .utils import (OrderedSampler, Prediction, Target, flatten_metadata, + restore_list_order) + +if 'COMET_EMBEDDINGS_CACHE' in os.environ: + CACHE_SIZE = int(os.environ['COMET_EMBEDDINGS_CACHE']) else: CACHE_SIZE = 1024 @@ -96,18 +93,18 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1.0e-06, learning_rate: float = 1.5e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-large", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-large', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], @@ -119,11 +116,13 @@ def __init__( super().__init__() self.save_hyperparameters() self.encoder = str2encoder[self.hparams.encoder_model].from_pretrained( - self.hparams.pretrained_model, load_pretrained_weights, local_files_only + self.hparams.pretrained_model, + load_pretrained_weights, + local_files_only, ) self.epoch_nr = 0 - if self.hparams.layer == "mix": + if self.hparams.layer == 'mix': self.layerwise_attention = LayerwiseAttention( layer_transformation=layer_transformation, num_layers=self.encoder.num_layers, @@ -162,7 +161,9 @@ def set_mc_dropout(self, value: int): def enable_context(self): """Function that extends COMET to use preceding context as described in https://statmt.org/wmt22/pdf/2022.wmt-1.6.pdf.""" - logger.warning("Context should only be enabled for RegressionMetric with Average Pooling.") + logger.warning( + 'Context should only be enabled for RegressionMetric with Average Pooling.' + ) @abc.abstractmethod def read_training_data(self) -> List[dict]: @@ -188,7 +189,7 @@ def read_validation_data(self): def prepare_sample( self, sample: List[dict], - stage: str = "fit", + stage: str = 'fit', *args, **kwargs, ): @@ -227,7 +228,7 @@ def requires_references(self) -> bool: def freeze_encoder(self) -> None: """Deactivates training for encoder model parameters (keeping them frozen)""" - logger.info("Encoder model frozen.") + logger.info('Encoder model frozen.') self.encoder.freeze() @property @@ -235,7 +236,9 @@ def loss(self): """Loss function""" return torch.nn.MSELoss() - def compute_loss(self, prediction: Prediction, target: Target) -> torch.Tensor: + def compute_loss( + self, prediction: Prediction, target: Target + ) -> torch.Tensor: """Computes Loss value between a batch Prediction and respective Target.""" return self.loss(prediction.score, target.score) @@ -243,7 +246,7 @@ def unfreeze_encoder(self) -> None: """Activates fine-tuning of encoder parameters.""" if self._frozen: if self.trainer.is_global_zero: - logger.info("Encoder model fine-tuning") + logger.info('Encoder model fine-tuning') self.encoder.unfreeze() self._frozen = False @@ -329,38 +332,43 @@ def compute_sentence_embedding( ) if self.layerwise_attention: embeddings = self.layerwise_attention( - encoder_out["all_layers"], attention_mask + encoder_out['all_layers'], attention_mask ) - elif self.hparams.layer >= 0 and self.hparams.layer < self.encoder.num_layers: - embeddings = encoder_out["all_layers"][self.hparams.layer] + elif ( + self.hparams.layer >= 0 + and self.hparams.layer < self.encoder.num_layers + ): + embeddings = encoder_out['all_layers'][self.hparams.layer] else: - raise Exception("Invalid model layer {}.".format(self.hparams.layer)) + raise Exception( + 'Invalid model layer {}.'.format(self.hparams.layer) + ) - if self.hparams.pool == "default": - sentemb = encoder_out["sentemb"] + if self.hparams.pool == 'default': + sentemb = encoder_out['sentemb'] - elif self.hparams.pool == "max": + elif self.hparams.pool == 'max': sentemb = max_pooling( input_ids, embeddings, self.encoder.tokenizer.pad_token_id ) - elif self.hparams.pool == "avg": + elif self.hparams.pool == 'avg': sentemb = average_pooling( input_ids, embeddings, attention_mask, self.encoder.tokenizer.pad_token_id, - self.encoder.tokenizer.sep_token_id, + self.encoder.tokenizer.sep_token_id, self.use_context, ) - elif self.hparams.pool == "cls": + elif self.hparams.pool == 'cls': sentemb = embeddings[:, 0, :] else: - raise Exception("Invalid pooling technique.") + raise Exception('Invalid pooling technique.') return sentemb @@ -391,7 +399,7 @@ def training_step( self._frozen = False self.log( - "train_loss", + 'train_loss', loss_value, on_step=True, on_epoch=True, @@ -414,13 +422,15 @@ def validation_step( batch_input, batch_target = batch batch_prediction = self.forward(**batch_input) if dataloader_idx == 0: - self.train_metrics.update(batch_prediction.score, batch_target["score"]) + self.train_metrics.update( + batch_prediction.score, batch_target['score'] + ) elif dataloader_idx > 0: self.val_metrics[dataloader_idx - 1].update( batch_prediction.score, - batch_target["score"], - batch_target["system"] if "system" in batch_target else None, + batch_target['score'], + batch_target['system'] if 'system' in batch_target else None, ) def on_predict_start(self) -> None: @@ -452,7 +462,7 @@ def predict_step( mcd_outputs = torch.stack( [self(**batch).score for _ in range(self.mc_dropout)] ) - model_outputs["metadata"] = Prediction( + model_outputs['metadata'] = Prediction( mcd_scores=mcd_outputs.mean(dim=0), mcd_std=mcd_outputs.std(dim=0), ) @@ -471,13 +481,16 @@ def on_validation_epoch_end(self, *args, **kwargs) -> None: self.log_dict(results, prog_bar=False) val_metrics.append(results) - average_results = {"val_" + k.split("_")[-1]: [] for k in val_metrics[0].keys()} + average_results = { + 'val_' + k.split('_')[-1]: [] for k in val_metrics[0].keys() + } for i in range(len(val_metrics)): for k, v in val_metrics[i].items(): - average_results["val_" + k.split("_")[-1]].append(v) + average_results['val_' + k.split('_')[-1]].append(v) self.log_dict( - {k: sum(v) / len(v) for k, v in average_results.items()}, prog_bar=True + {k: sum(v) / len(v) for k, v in average_results.items()}, + prog_bar=True, ) def setup(self, stage: str) -> None: @@ -485,11 +498,12 @@ def setup(self, stage: str) -> None: stage (str): either 'fit', 'validate', 'test', or 'predict' """ - if stage in (None, "fit"): + if stage in (None, 'fit'): train_dataset = self.read_training_data(self.hparams.train_data[0]) self.validation_sets = [ - self.read_validation_data(d) for d in self.hparams.validation_data + self.read_validation_data(d) + for d in self.hparams.validation_data ] self.first_epoch_total_steps = len(train_dataset) // ( @@ -497,7 +511,8 @@ def setup(self, stage: str) -> None: ) # Always validate the model with part of training. train_subset = np.random.choice( - a=len(train_dataset), size=min(1000, int(len(train_dataset) * 0.2)) + a=len(train_dataset), + size=min(1000, int(len(train_dataset) * 0.2)), ) self.train_subset = Subset(train_dataset, train_subset) @@ -510,13 +525,13 @@ def train_dataloader(self) -> DataLoader: self.current_epoch % len(self.hparams.train_data) ] train_dataset = self.read_training_data(data_path) - logger.info(f"Loading {data_path}.") + logger.info(f'Loading {data_path}.') return DataLoader( dataset=train_dataset, sampler=RandomSampler(train_dataset), batch_size=self.hparams.batch_size, - collate_fn=lambda s: self.prepare_sample(s, stage="fit"), + collate_fn=lambda s: self.prepare_sample(s, stage='fit'), num_workers=2 * self.trainer.num_devices, ) @@ -526,7 +541,7 @@ def val_dataloader(self) -> DataLoader: DataLoader( dataset=self.train_subset, batch_size=self.hparams.batch_size, - collate_fn=lambda s: self.prepare_sample(s, stage="validate"), + collate_fn=lambda s: self.prepare_sample(s, stage='validate'), num_workers=2 * self.trainer.num_devices, ) ] @@ -535,7 +550,9 @@ def val_dataloader(self) -> DataLoader: DataLoader( dataset=validation_set, batch_size=self.hparams.batch_size, - collate_fn=lambda s: self.prepare_sample(s, stage="validate"), + collate_fn=lambda s: self.prepare_sample( + s, stage='validate' + ), num_workers=2 * self.trainer.num_devices, ) ) @@ -545,7 +562,7 @@ def prepare_for_inference(self, sample): """This is to avoid having a lamba function inside the predict dataloader `collate_fn=lambda x: self.prepare_sample(x, inference=True)` """ - return self.prepare_sample(sample, stage="predict") + return self.prepare_sample(sample, stage='predict') def predict( self, @@ -555,7 +572,7 @@ def predict( devices: Union[List[int], str, int] = None, mc_dropout: int = 0, progress_bar: bool = True, - accelerator: str = "auto", + accelerator: str = 'auto', num_workers: int = None, length_batching: bool = True, ) -> Prediction: @@ -589,30 +606,34 @@ def predict( if gpus > 0 and devices is not None: assert len(devices) == gpus, AssertionError( - "List of devices must be same size as `gpus` or None if `gpus=0`" + 'List of devices must be same size as `gpus` or None if `gpus=0`' ) elif gpus > 0: devices = gpus - else: # gpu = 0 - devices = "auto" + else: # gpu = 0 + devices = 'auto' sampler = SequentialSampler(samples) if length_batching and gpus < 2: try: - sort_ids = np.argsort([len(sample["src"]) for sample in samples]) + sort_ids = np.argsort( + [len(sample['src']) for sample in samples] + ) except KeyError: - sort_ids = np.argsort([len(sample["ref"]) for sample in samples]) + sort_ids = np.argsort( + [len(sample['ref']) for sample in samples] + ) sampler = OrderedSampler(sort_ids) # On Windows, only num_workers=0 is supported. - is_windows = os.name == "nt" + is_windows = os.name == 'nt' if num_workers is None: # Guideline for workers that typically works well. num_workers = 0 if is_windows else 2 * gpus elif is_windows and num_workers != 0: logger.warning( - "Due to limits of multiprocessing on Windows, it is likely that setting num_workers > 0 will result" - " in scores of 0. It is therefore recommended to set num_workers=0 or leave it to None (default)." + 'Due to limits of multiprocessing on Windows, it is likely that setting num_workers > 0 will result' + ' in scores of 0. It is therefore recommended to set num_workers=0 or leave it to None (default).' ) self.eval() @@ -622,7 +643,9 @@ def predict( sampler=sampler, collate_fn=self.prepare_for_inference, num_workers=num_workers, - multiprocessing_context="fork" if torch.backends.mps.is_available() else None, + multiprocessing_context=( + 'fork' if torch.backends.mps.is_available() else None + ), ) if gpus > 1: pred_writer = CustomWriter() @@ -639,16 +662,16 @@ def predict( enable_progress_bar = False warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer = ptl.Trainer( devices=devices, logger=False, callbacks=callbacks, - accelerator=accelerator if gpus > 0 else "cpu", - strategy="auto" if gpus < 2 else "ddp", + accelerator=accelerator if gpus > 0 else 'cpu', + strategy='auto' if gpus < 2 else 'ddp', enable_progress_bar=enable_progress_bar, ) return_predictions = False if gpus > 1 else True @@ -669,25 +692,34 @@ def predict( # If we are not in the GLOBAL RANK we will return None exit() - scores = torch.cat([pred["scores"] for pred in predictions], dim=0).tolist() - if "metadata" in predictions[0]: - metadata = flatten_metadata([pred["metadata"] for pred in predictions]) + scores = torch.cat( + [pred['scores'] for pred in predictions], dim=0 + ).tolist() + if 'metadata' in predictions[0]: + metadata = flatten_metadata( + [pred['metadata'] for pred in predictions] + ) else: metadata = [] - output = Prediction(scores=scores, system_score=sum(scores) / len(scores)) + output = Prediction( + scores=scores, system_score=sum(scores) / len(scores) + ) # Restore order of samples! if length_batching and gpus < 2: - output["scores"] = restore_list_order(scores, sort_ids) + output['scores'] = restore_list_order(scores, sort_ids) if metadata: - output["metadata"] = Prediction( - **{k: restore_list_order(v, sort_ids) for k, v in metadata.items()} + output['metadata'] = Prediction( + **{ + k: restore_list_order(v, sort_ids) + for k, v in metadata.items() + } ) return output else: # Add metadata to output if metadata: - output["metadata"] = metadata + output['metadata'] = metadata return output diff --git a/comet/models/download_utils.py b/comet/models/download_utils.py index afd5d84d..45535464 100644 --- a/comet/models/download_utils.py +++ b/comet/models/download_utils.py @@ -29,27 +29,25 @@ available_legacy_metrics = { # WMT20 Models - "emnlp20-comet-rank": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/emnlp20-comet-rank.tar.gz", - "wmt20-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-da.tar.gz", - "wmt20-comet-qe-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da.tar.gz", - "wmt20-comet-qe-da-v2": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da-v2.tar.gz", - + 'emnlp20-comet-rank': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/emnlp20-comet-rank.tar.gz', + 'wmt20-comet-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-da.tar.gz', + 'wmt20-comet-qe-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da.tar.gz', + 'wmt20-comet-qe-da-v2': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt20/wmt20-comet-qe-da-v2.tar.gz', # WMT21 Models - "wmt21-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-da.tar.gz", - "wmt21-comet-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-mqm.tar.gz", - "wmt21-cometinho-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-mqm.tar.gz", - "wmt21-cometinho-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-da.tar.gz", - "wmt21-comet-qe-mqm": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-mqm.tar.gz", - "wmt21-comet-qe-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-da.tar.gz", - + 'wmt21-comet-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-da.tar.gz', + 'wmt21-comet-mqm': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-mqm.tar.gz', + 'wmt21-cometinho-mqm': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-mqm.tar.gz', + 'wmt21-cometinho-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-cometinho-da.tar.gz', + 'wmt21-comet-qe-mqm': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-mqm.tar.gz', + 'wmt21-comet-qe-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/wmt21/wmt21-comet-qe-da.tar.gz', # EAMT22 Models - "eamt22-cometinho-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-cometinho-da.tar.gz", - "eamt22-prune-comet-da": "https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-prune-comet-da.tar.gz", + 'eamt22-cometinho-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-cometinho-da.tar.gz', + 'eamt22-prune-comet-da': 'https://unbabel-experimental-models.s3.amazonaws.com/comet/eamt22/eamt22-prune-comet-da.tar.gz', } def get_cache_folder(): - cache_directory = Path.home() / ".cache" / "torch" / "unbabel_comet" + cache_directory = Path.home() / '.cache' / 'torch' / 'unbabel_comet' if not cache_directory.exists(): cache_directory.mkdir(exist_ok=True, parents=True) @@ -83,7 +81,9 @@ def inner(b: int = 1, bsize: int = 1, tsize: int = None): return inner -def _maybe_extract(compressed_filename: str, directory: str, extension: str = None): +def _maybe_extract( + compressed_filename: str, directory: str, extension: str = None +): """Extract a compressed file to ``directory``. :param compressed_filename: Compressed file. @@ -91,30 +91,32 @@ def _maybe_extract(compressed_filename: str, directory: str, extension: str = No :param extension: Extension of the file; Otherwise, attempts to extract extension from the filename. """ - logger.info("Extracting {}".format(compressed_filename)) + logger.info('Extracting {}'.format(compressed_filename)) if extension is None: basename = os.path.basename(compressed_filename) - extension = basename.split(".", 1)[1] + extension = basename.split('.', 1)[1] - if "zip" in extension: - with zipfile.ZipFile(compressed_filename, "r") as zip_: + if 'zip' in extension: + with zipfile.ZipFile(compressed_filename, 'r') as zip_: zip_.extractall(directory) - elif "tar.gz" in extension or "tgz" in extension: + elif 'tar.gz' in extension or 'tgz' in extension: # `tar` is much faster than python's `tarfile` implementation - with open(os.devnull, "w") as devnull: + with open(os.devnull, 'w') as devnull: subprocess.call( - ["tar", "-C", directory, "-zxvf", compressed_filename], stdout=devnull + ['tar', '-C', directory, '-zxvf', compressed_filename], + stdout=devnull, ) - elif "tar" in extension: - with open(os.devnull, "w") as devnull: + elif 'tar' in extension: + with open(os.devnull, 'w') as devnull: subprocess.call( - ["tar", "-C", directory, "-xvf", compressed_filename], stdout=devnull + ['tar', '-C', directory, '-xvf', compressed_filename], + stdout=devnull, ) - logger.info("Extracted {}".format(compressed_filename)) + logger.info('Extracted {}'.format(compressed_filename)) def _get_filename_from_url(url): @@ -178,18 +180,20 @@ def download_file_maybe_extract( if not os.path.isdir(directory): os.makedirs(directory) - logger.info("Downloading {}".format(filename)) + logger.info('Downloading {}'.format(filename)) # Download - with tqdm(unit="B", unit_scale=True, miniters=1, desc=filename) as t: - urllib.request.urlretrieve(url, filename=filepath, reporthook=_reporthook(t)) + with tqdm(unit='B', unit_scale=True, miniters=1, desc=filename) as t: + urllib.request.urlretrieve( + url, filename=filepath, reporthook=_reporthook(t) + ) _maybe_extract( compressed_filename=filepath, directory=directory, extension=extension ) if not _check_download(*check_files): - raise ValueError("[DOWNLOAD FAILED] `*check_files` not found") + raise ValueError('[DOWNLOAD FAILED] `*check_files` not found') return filepath @@ -209,42 +213,44 @@ def download_model_legacy(model: str, saving_directory: str = None) -> str: if saving_directory is None: saving_directory = get_cache_folder() - if not saving_directory.endswith("/"): - saving_directory += "/" + if not saving_directory.endswith('/'): + saving_directory += '/' if not os.path.exists(saving_directory): os.makedirs(saving_directory) if os.path.isdir(saving_directory + model): - logger.info(f"{model} is already in cache.") - if not model.endswith("/"): - model += "/" + logger.info(f'{model} is already in cache.') + if not model.endswith('/'): + model += '/' elif model not in available_legacy_metrics.keys(): raise Exception( - f"{model} is not in the `available_legacy_metrics` or is a valid checkpoint folder." + f'{model} is not in the `available_legacy_metrics` or is a valid checkpoint folder.' ) - elif available_legacy_metrics[model].startswith("https://"): + elif available_legacy_metrics[model].startswith('https://'): download_file_maybe_extract( available_legacy_metrics[model], directory=saving_directory ) else: - raise Exception("Invalid model name!") + raise Exception('Invalid model name!') # CLEAN Cache - if os.path.exists(saving_directory + model + ".zip"): - os.remove(saving_directory + model + ".zip") - if os.path.exists(saving_directory + model + ".tar.gz"): - os.remove(saving_directory + model + ".tar.gz") - if os.path.exists(saving_directory + model + ".tar"): - os.remove(saving_directory + model + ".tar") - - checkpoints_folder = saving_directory + model + "/checkpoints" + if os.path.exists(saving_directory + model + '.zip'): + os.remove(saving_directory + model + '.zip') + if os.path.exists(saving_directory + model + '.tar.gz'): + os.remove(saving_directory + model + '.tar.gz') + if os.path.exists(saving_directory + model + '.tar'): + os.remove(saving_directory + model + '.tar') + + checkpoints_folder = saving_directory + model + '/checkpoints' checkpoints = [ - file for file in os.listdir(checkpoints_folder) if file.endswith(".ckpt") + file + for file in os.listdir(checkpoints_folder) + if file.endswith('.ckpt') ] checkpoint = checkpoints[-1] - checkpoint_path = checkpoints_folder + "/" + checkpoint + checkpoint_path = checkpoints_folder + '/' + checkpoint return checkpoint_path diff --git a/comet/models/lru_cache.py b/comet/models/lru_cache.py index 5a200fe4..33cec77a 100644 --- a/comet/models/lru_cache.py +++ b/comet/models/lru_cache.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -LRU Cache +LRU Cache ========== LRU Cache function decorator modified to work with tensor arguments. @@ -21,6 +21,7 @@ Our modification modifies the _make_key function to use tensor str representation intead of the object reference. Other than that we use the original implementation """ + from _thread import RLock from functools import _CacheInfo, _HashedSeq, update_wrapper @@ -52,10 +53,10 @@ def _make_key( for x in args: if torch.is_tensor(x): if len(x.size()) == 0: - raise Exception("Tensor needs to be at least 1-Dimensional.") - + raise Exception('Tensor needs to be at least 1-Dimensional.') + if len(x.size()) == 1: - new_args.append("\n".join([repr(x), repr(x.shape)])) + new_args.append('\n'.join([repr(x), repr(x.shape)])) else: new_args.append( # HACK: Tensor representations omit some tensor content. @@ -63,9 +64,9 @@ def _make_key( # The current solution is an approximation to the actual tensor # full representation. This can still lead to `false` cache hits! x.__repr__() - + "\n" + + '\n' + x.diagonal().__repr__() - + "\n" + + '\n' + x.shape.__repr__() ) else: @@ -99,14 +100,16 @@ def tensor_lru_cache(maxsize=128, typed=False): # The user_function was passed in directly via the maxsize argument user_function, maxsize = maxsize, 128 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) - wrapper.cache_parameters = lambda: {"maxsize": maxsize, "typed": typed} + wrapper.cache_parameters = lambda: {'maxsize': maxsize, 'typed': typed} return update_wrapper(wrapper, user_function) elif maxsize is not None: - raise TypeError("Expected first argument to be an integer, a callable, or None") + raise TypeError( + 'Expected first argument to be an integer, a callable, or None' + ) def decorating_function(user_function): wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) - wrapper.cache_parameters = lambda: {"maxsize": maxsize, "typed": typed} + wrapper.cache_parameters = lambda: {'maxsize': maxsize, 'typed': typed} return update_wrapper(wrapper, user_function) return decorating_function diff --git a/comet/models/metrics.py b/comet/models/metrics.py index ae047f0f..e29cb339 100644 --- a/comet/models/metrics.py +++ b/comet/models/metrics.py @@ -19,6 +19,7 @@ Regression and Ranking metrics to be used during training to measure correlations with human judgements """ + from itertools import combinations from typing import Any, Callable, List, Optional @@ -29,7 +30,9 @@ from torchmetrics.classification import MulticlassMatthewsCorrCoef -def system_accuracy(y_hat: List[float], y: List[float], system: List[str]) -> float: +def system_accuracy( + y_hat: List[float], y: List[float], system: List[str] +) -> float: """Implementation of system-level accuracy proposed in [To Ship not to Ship](https://aclanthology.org/2021.wmt-1.57/) @@ -42,23 +45,23 @@ def system_accuracy(y_hat: List[float], y: List[float], system: List[str]) -> fl Float: System-level accuracy. """ try: - data = pd.DataFrame({"y_hat": y_hat, "y": y, "system": system}) + data = pd.DataFrame({'y_hat': y_hat, 'y': y, 'system': system}) except ValueError: raise Exception( - "The program will be interrupted, followed by a series of errors." + 'The program will be interrupted, followed by a series of errors.' " This probably happens because you're using ddp strategy in the" - " trainer config. System accuracy computation does not currently" - " work with ddp. Please make sure your VALIDATION data DOES NOT" + ' trainer config. System accuracy computation does not currently' + ' work with ddp. Please make sure your VALIDATION data DOES NOT' " include a 'system' column, and try again." ) - data = data.groupby("system").mean() + data = data.groupby('system').mean() pairs = list(combinations(data.index.tolist(), 2)) tp = 0 for system_a, system_b in pairs: - human_delta = data.loc[system_a]["y"] - data.loc[system_b]["y"] - model_delta = data.loc[system_a]["y_hat"] - data.loc[system_b]["y_hat"] + human_delta = data.loc[system_a]['y'] - data.loc[system_b]['y'] + model_delta = data.loc[system_a]['y_hat'] - data.loc[system_b]['y_hat'] if (human_delta >= 0) ^ (model_delta < 0): tp += 1 @@ -67,14 +70,14 @@ def system_accuracy(y_hat: List[float], y: List[float], system: List[str]) -> fl class MCCMetric(MulticlassMatthewsCorrCoef): - def __init__(self, prefix: str = "", **kwargs) -> None: + def __init__(self, prefix: str = '', **kwargs) -> None: super().__init__(**kwargs) self.prefix = prefix def compute(self) -> torch.Tensor: """Computes matthews correlation coefficient.""" mcc = super(MCCMetric, self).compute() - return {self.prefix + "_mcc": mcc} + return {self.prefix + '_mcc': mcc} class RegressionMetrics(Metric): @@ -86,7 +89,7 @@ class RegressionMetrics(Metric): def __init__( self, - prefix: str = "", + prefix: str = '', dist_sync_on_step: bool = False, process_group: Optional[Any] = None, dist_sync_fn: Optional[Callable] = None, @@ -96,9 +99,9 @@ def __init__( process_group=process_group, dist_sync_fn=dist_sync_fn, ) - self.add_state("preds", default=[], dist_reduce_fx="cat") - self.add_state("target", default=[], dist_reduce_fx="cat") - self.add_state("systems", default=[], dist_reduce_fx=None) + self.add_state('preds', default=[], dist_reduce_fx='cat') + self.add_state('target', default=[], dist_reduce_fx='cat') + self.add_state('systems', default=[], dist_reduce_fx=None) self.prefix = prefix def update( @@ -131,16 +134,16 @@ def compute(self) -> torch.Tensor: spearman, _ = stats.spearmanr(preds.tolist(), target.tolist()) pearson, _ = stats.pearsonr(preds.tolist(), target.tolist()) report = { - self.prefix + "_kendall": kendall, - self.prefix + "_spearman": spearman, - self.prefix + "_pearson": pearson, + self.prefix + '_kendall': kendall, + self.prefix + '_spearman': spearman, + self.prefix + '_pearson': pearson, } if len(self.systems) > 0: system_acc = system_accuracy( preds.cpu().tolist(), target.cpu().tolist(), self.systems ) - report["system_acc"] = system_acc + report['system_acc'] = system_acc return report @@ -150,7 +153,7 @@ class WMTKendall(Metric): def __init__( self, - prefix: str = "", + prefix: str = '', dist_sync_on_step: bool = False, process_group: Optional[Any] = None, dist_sync_fn: Optional[Callable] = None, @@ -160,8 +163,12 @@ def __init__( process_group=process_group, dist_sync_fn=dist_sync_fn, ) - self.add_state("concordance", default=torch.tensor(0), dist_reduce_fx="sum") - self.add_state("discordance", default=torch.tensor(0), dist_reduce_fx="sum") + self.add_state( + 'concordance', default=torch.tensor(0), dist_reduce_fx='sum' + ) + self.add_state( + 'discordance', default=torch.tensor(0), dist_reduce_fx='sum' + ) self.prefix = prefix def update(self, distance_pos: torch.Tensor, distance_neg: torch.Tensor): @@ -176,6 +183,6 @@ def update(self, distance_pos: torch.Tensor, distance_neg: torch.Tensor): def compute(self): return { self.prefix - + "_kendall": (self.concordance - self.discordance) + + '_kendall': (self.concordance - self.discordance) / (self.concordance + self.discordance) } diff --git a/comet/models/multitask/unified_metric.py b/comet/models/multitask/unified_metric.py index e0a76548..cc5375bb 100644 --- a/comet/models/multitask/unified_metric.py +++ b/comet/models/multitask/unified_metric.py @@ -16,12 +16,13 @@ r""" Unified Metric ============== - Unified Metric is a multitask metric that performs word-level and segment-level - evaluation in a multitask manner. It can also be used with and without reference + Unified Metric is a multitask metric that performs word-level and segment-level + evaluation in a multitask manner. It can also be used with and without reference translations. - + Inspired on [UniTE](https://arxiv.org/pdf/2204.13346.pdf) """ + from collections import OrderedDict from typing import Dict, List, Optional, Tuple, Union @@ -96,29 +97,29 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.9, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 3.0e-06, learning_rate: float = 3.0e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "microsoft/infoxlm-large", - sent_layer: Union[str, int] = "mix", - layer_transformation: str = "sparsemax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'microsoft/infoxlm-large', + sent_layer: Union[str, int] = 'mix', + layer_transformation: str = 'sparsemax', layer_norm: bool = True, word_layer: int = 24, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [3072, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, - input_segments: List[str] = ["mt", "src", "ref"], + input_segments: List[str] = ['mt', 'src', 'ref'], word_level_training: bool = False, loss_lambda: float = 0.65, - error_labels: List[str] = ["minor", "major"], + error_labels: List[str] = ['minor', 'major'], cross_entropy_weights: Optional[List[float]] = None, load_pretrained_weights: bool = True, local_files_only: bool = False, @@ -141,7 +142,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="unified_metric", + class_identifier='unified_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -156,7 +157,9 @@ def __init__( self.word_level = word_level_training if word_level_training: self.encoder.labelset = self.label_encoder - self.hidden2tag = nn.Linear(self.encoder.output_units, self.num_classes) + self.hidden2tag = nn.Linear( + self.encoder.output_units, self.num_classes + ) if len(self.hparams.input_segments) == 3: # By default 3rd input [mt:src:ref] has 50% weight, @@ -191,7 +194,7 @@ def set_decoding_threshold(self, threshold: float = 0.5): def init_metrics(self): """Initializes training and validation metrics""" # Train and Dev correlation metrics - self.train_corr = RegressionMetrics(prefix="train") + self.train_corr = RegressionMetrics(prefix='train') self.val_corr = nn.ModuleList( [RegressionMetrics(prefix=d) for d in self.hparams.validation_data] ) @@ -199,7 +202,9 @@ def init_metrics(self): self.label_encoder = LabelSet(self.hparams.error_labels) self.num_classes = len(self.label_encoder.labels_to_id) # Train and Dev MCC - self.train_mcc = MCCMetric(num_classes=self.num_classes, prefix="train") + self.train_mcc = MCCMetric( + num_classes=self.num_classes, prefix='train' + ) self.val_mcc = nn.ModuleList( [ MCCMetric(num_classes=self.num_classes, prefix=d) @@ -212,13 +217,15 @@ def init_losses(self) -> None: self.sentloss = nn.MSELoss() if self.word_level: if self.hparams.cross_entropy_weights: - assert len(self.hparams.cross_entropy_weights) == self.num_classes + assert ( + len(self.hparams.cross_entropy_weights) == self.num_classes + ) loss_weights = torch.tensor(self.hparams.cross_entropy_weights) else: loss_weights = None self.wordloss = nn.CrossEntropyLoss( - reduction="mean", ignore_index=-1, weight=loss_weights + reduction='mean', ignore_index=-1, weight=loss_weights ) def requires_references(self) -> bool: @@ -229,13 +236,15 @@ def requires_references(self) -> bool: Return: [bool]: True if the model was trained to work exclusively with references. """ - if self.hparams.input_segments == ["mt", "ref"]: + if self.hparams.input_segments == ['mt', 'ref']: return True return False def configure_optimizers( self, - ) -> Tuple[List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR]]: + ) -> Tuple[ + List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR] + ]: """Pytorch Lightning method to initialize a training Optimizer and learning rate scheduler. @@ -247,25 +256,28 @@ def configure_optimizers( self.hparams.encoder_learning_rate, self.hparams.layerwise_decay ) params += [ - {"params": self.estimator.parameters(), "lr": self.hparams.learning_rate} + { + 'params': self.estimator.parameters(), + 'lr': self.hparams.learning_rate, + } ] if self.word_level: params += [ { - "params": self.hidden2tag.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.hidden2tag.parameters(), + 'lr': self.hparams.learning_rate, }, ] if self.layerwise_attention: params += [ { - "params": self.layerwise_attention.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.layerwise_attention.parameters(), + 'lr': self.hparams.learning_rate, } ] - if self.hparams.optimizer == "Adafactor": + if self.hparams.optimizer == 'Adafactor': optimizer = Adafactor( params, lr=self.hparams.learning_rate, @@ -300,10 +312,10 @@ def read_training_data(self, path: str) -> List[dict]: # Make sure everything except score is str type for col in columns: df[col] = df[col].astype(str) - columns.append("score") - df["score"] = df["score"].astype("float16") + columns.append('score') + df['score'] = df['score'].astype('float16') df = df[columns] - return df.to_dict("records") + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Reads a csv file with validation data. @@ -318,15 +330,15 @@ def read_validation_data(self, path: str) -> List[dict]: # Deep copy input segments columns = self.hparams.input_segments[:] # If system in columns we will use this to calculate system-level accuracy - if "system" in df.columns: - columns.append("system") + if 'system' in df.columns: + columns.append('system') # Make sure everything except score is str type for col in columns: df[col] = df[col].astype(str) - columns.append("score") - df["score"] = df["score"].astype("float16") + columns.append('score') + df['score'] = df['score'].astype('float16') df = df[columns] - return df.to_dict("records") + return df.to_dict('records') def concat_inputs( self, @@ -359,23 +371,27 @@ def concat_inputs( full_input, _, _ = self.encoder.concat_sequences( input_sequences, return_label_ids=self.word_level ) - model_inputs["inputs"] = (src_input, ref_input, full_input) - model_inputs["mt_length"] = input_sequences[0]["attention_mask"].sum(dim=1) + model_inputs['inputs'] = (src_input, ref_input, full_input) + model_inputs['mt_length'] = input_sequences[0][ + 'attention_mask' + ].sum(dim=1) return model_inputs # Otherwise we will have one single input sequence that concatenates the MT # with SRC/REF. else: - model_inputs["inputs"] = ( + model_inputs['inputs'] = ( self.encoder.concat_sequences( input_sequences, return_label_ids=self.word_level )[0], ) - model_inputs["mt_length"] = input_sequences[0]["attention_mask"].sum(dim=1) + model_inputs['mt_length'] = input_sequences[0][ + 'attention_mask' + ].sum(dim=1) return model_inputs def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "fit" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'fit' ) -> Union[Tuple[Dict[str, torch.Tensor]], Dict[str, torch.Tensor]]: """Tokenizes input data and prepares targets for training. @@ -389,45 +405,47 @@ def prepare_sample( """ inputs = {k: [d[k] for d in sample] for k in sample[0]} input_sequences = [ - self.encoder.prepare_sample(inputs["mt"], self.word_level, None), + self.encoder.prepare_sample(inputs['mt'], self.word_level, None), ] src_input, ref_input = False, False - if ("src" in inputs) and ("src" in self.hparams.input_segments): - input_sequences.append(self.encoder.prepare_sample(inputs["src"])) + if ('src' in inputs) and ('src' in self.hparams.input_segments): + input_sequences.append(self.encoder.prepare_sample(inputs['src'])) src_input = True - if ("ref" in inputs) and ("ref" in self.hparams.input_segments): - input_sequences.append(self.encoder.prepare_sample(inputs["ref"])) + if ('ref' in inputs) and ('ref' in self.hparams.input_segments): + input_sequences.append(self.encoder.prepare_sample(inputs['ref'])) ref_input = True unified_input = src_input and ref_input model_inputs = self.concat_inputs(input_sequences, unified_input) - if stage == "predict": - return model_inputs["inputs"] + if stage == 'predict': + return model_inputs['inputs'] - scores = [float(s) for s in inputs["score"]] + scores = [float(s) for s in inputs['score']] targets = Target(score=torch.tensor(scores, dtype=torch.float)) - if "system" in inputs: - targets["system"] = inputs["system"] + if 'system' in inputs: + targets['system'] = inputs['system'] if self.word_level: # Labels will be the same accross all inputs because we are only # doing sequence tagging on the MT. We will only use the mask corresponding # to the MT segment. - seq_len = model_inputs["mt_length"].max() - targets["mt_length"] = model_inputs["mt_length"] - targets["labels"] = model_inputs["inputs"][0]["label_ids"][:, :seq_len] + seq_len = model_inputs['mt_length'].max() + targets['mt_length'] = model_inputs['mt_length'] + targets['labels'] = model_inputs['inputs'][0]['label_ids'][ + :, :seq_len + ] - return model_inputs["inputs"], targets + return model_inputs['inputs'], targets def forward( self, input_ids: torch.Tensor, attention_mask: torch.Tensor, token_type_ids: Optional[torch.Tensor] = None, - **kwargs + **kwargs, ) -> Dict[str, torch.Tensor]: """Forward function. @@ -455,36 +473,44 @@ def forward( isinstance(self.hparams.word_layer, int) and 0 <= self.hparams.word_layer < self.encoder.num_layers ): - wordemb = encoder_out["all_layers"][self.hparams.word_layer] + wordemb = encoder_out['all_layers'][self.hparams.word_layer] else: raise Exception( - "Invalid model word layer {}.".format(self.hparams.word_layer) + 'Invalid model word layer {}.'.format( + self.hparams.word_layer + ) ) # embeddings used for the sentence-level regression task if self.layerwise_attention: embeddings = self.layerwise_attention( - encoder_out["all_layers"], attention_mask + encoder_out['all_layers'], attention_mask ) elif ( isinstance(self.hparams.sent_layer, int) and 0 <= self.hparams.sent_layer < self.encoder.num_layers ): - embeddings = encoder_out["all_layers"][self.hparams.sent_layer] + embeddings = encoder_out['all_layers'][self.hparams.sent_layer] else: raise Exception( - "Invalid model sent layer {}.".format(self.hparams.word_layer) + 'Invalid model sent layer {}.'.format(self.hparams.word_layer) ) - sentemb = embeddings[:, 0, :] # We take the CLS token as sentence-embedding - + sentemb = embeddings[ + :, 0, : + ] # We take the CLS token as sentence-embedding + if self.word_level: sentence_output = self.estimator(sentemb) word_output = self.hidden2tag(wordemb) - return Prediction(score=sentence_output.view(-1), logits=word_output) + return Prediction( + score=sentence_output.view(-1), logits=word_output + ) return Prediction(score=self.estimator(sentemb).view(-1)) - def compute_loss(self, prediction: Prediction, target: Target) -> torch.Tensor: + def compute_loss( + self, prediction: Prediction, target: Target + ) -> torch.Tensor: """Receives model batch prediction and respective targets and computes a loss value @@ -500,9 +526,9 @@ def compute_loss(self, prediction: Prediction, target: Target) -> torch.Tensor: predictions = prediction.logits.reshape(-1, self.num_classes) targets = target.labels.reshape(-1).type(torch.LongTensor).cuda() word_loss = self.wordloss(predictions, targets) - return sentence_loss * (1 - self.hparams.loss_lambda) + word_loss * ( - self.hparams.loss_lambda - ) + return sentence_loss * ( + 1 - self.hparams.loss_lambda + ) + word_loss * (self.hparams.loss_lambda) else: return sentence_loss @@ -539,7 +565,7 @@ def training_step( self._frozen = False self.log( - "train_loss", + 'train_loss', loss_value, on_step=True, on_epoch=True, @@ -549,7 +575,10 @@ def training_step( return loss_value def validation_step( - self, batch: Tuple[Dict[str, torch.Tensor]], batch_nb: int, dataloader_idx: int + self, + batch: Tuple[Dict[str, torch.Tensor]], + batch_nb: int, + dataloader_idx: int, ) -> None: """Pytorch Lightning validation_step. @@ -562,7 +591,9 @@ def validation_step( batch_input, batch_target = batch predictions = [self.forward(**input_seq) for input_seq in batch_input] # Final score is the average of the 3 scores when using references. - scores = torch.stack([pred.score for pred in predictions], dim=0).mean(dim=0) + scores = torch.stack([pred.score for pred in predictions], dim=0).mean( + dim=0 + ) if self.word_level: seq_len = batch_target.mt_length.max() # Final probs for each word is the average of the 3 forward passes. @@ -587,7 +618,7 @@ def validation_step( self.val_corr[dataloader_idx - 1].update( scores, batch_target.score, - batch_target["system"] if "system" in batch_target else None, + batch_target['system'] if 'system' in batch_target else None, ) if self.word_level: self.val_mcc[dataloader_idx - 1].update(probs, targets) @@ -599,7 +630,9 @@ def on_validation_epoch_end(self, *args, **kwargs) -> None: self.train_corr.reset() if self.word_level: - self.log_dict(self.train_mcc.compute(), prog_bar=False, sync_dist=True) + self.log_dict( + self.train_mcc.compute(), prog_bar=False, sync_dist=True + ) self.train_mcc.reset() val_metrics = [] @@ -617,10 +650,12 @@ def on_validation_epoch_end(self, *args, **kwargs) -> None: self.log_dict(results, prog_bar=False, sync_dist=True) val_metrics.append(results) - average_results = {"val_" + k.split("_")[-1]: [] for k in val_metrics[0].keys()} + average_results = { + 'val_' + k.split('_')[-1]: [] for k in val_metrics[0].keys() + } for i in range(len(val_metrics)): for k, v in val_metrics[i].items(): - average_results["val_" + k.split("_")[-1]].append(v) + average_results['val_' + k.split('_')[-1]].append(v) self.log_dict( {k: sum(v) / len(v) for k, v in average_results.items()}, @@ -634,7 +669,7 @@ def set_mc_dropout(self, value: int): Args: value (int): number of runs per sample. """ - raise NotImplementedError("MCD not implemented for this model!") + raise NotImplementedError('MCD not implemented for this model!') def decode( self, @@ -658,7 +693,9 @@ def decode( seq_len = len(mt_offsets[i]) error_spans, in_span, span = [], False, {} for token_id, probs, token_offset in zip( - input_ids[i, :seq_len], subword_probs[i][:seq_len], mt_offsets[i] + input_ids[i, :seq_len], + subword_probs[i][:seq_len], + mt_offsets[i], ): if self.decoding_threshold: if torch.sum(probs[1:]) > self.decoding_threshold: @@ -682,26 +719,26 @@ def decode( # Label set: # O I-minor I-major # Begin of annotation span - if label.startswith("I") and not in_span: + if label.startswith('I') and not in_span: in_span = True - span["tokens"] = [ + span['tokens'] = [ token_id, ] - span["severity"] = label.split("-")[1] - span["offset"] = list(token_offset) - span["confidence"] = [ + span['severity'] = label.split('-')[1] + span['offset'] = list(token_offset) + span['confidence'] = [ probability, ] # Inside an annotation span - elif label.startswith("I") and in_span: - span["tokens"].append(token_id) - span["confidence"].append(probability) + elif label.startswith('I') and in_span: + span['tokens'].append(token_id) + span['confidence'].append(probability) # Update offset end - span["offset"][1] = token_offset[1] + span['offset'][1] = token_offset[1] # annotation span finished. - elif label == "O" and in_span: + elif label == 'O' and in_span: error_spans.append(span) in_span, span = False, {} @@ -709,11 +746,13 @@ def decode( for span in error_spans: sentence_output.append( { - "text": self.encoder.tokenizer.decode(span["tokens"]), - "confidence": torch.concat(span["confidence"]).mean().item(), - "severity": span["severity"], - "start": span["offset"][0], - "end": span["offset"][1], + 'text': self.encoder.tokenizer.decode(span['tokens']), + 'confidence': torch.concat(span['confidence']) + .mean() + .item(), + 'severity': span['severity'], + 'start': span['offset'][0], + 'end': span['offset'][1], } ) decoded_output.append(sentence_output) @@ -740,9 +779,9 @@ def predict_step( if len(batch) == 3: predictions = [self.forward(**input_seq) for input_seq in batch] # Final score is the average of the 3 scores! - avg_scores = torch.stack([pred.score for pred in predictions], dim=0).mean( - dim=0 - ) + avg_scores = torch.stack( + [pred.score for pred in predictions], dim=0 + ).mean(dim=0) batch_prediction = Prediction( scores=avg_scores, metadata=Prediction( @@ -752,7 +791,7 @@ def predict_step( ), ) if self.word_level: - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() subword_probs = [ @@ -761,22 +800,26 @@ def predict_step( ] subword_probs = torch.sum(torch.stack(subword_probs), dim=0) error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, + batch[0]['input_ids'], + batch[0]['mt_offsets'], ) - batch_prediction.metadata["error_spans"] = error_spans + batch_prediction.metadata['error_spans'] = error_spans else: model_output = self.forward(**batch[0]) batch_prediction = Prediction(scores=model_output.score) if self.word_level: - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() - subword_probs = nn.functional.softmax(model_output.logits, dim=2)[ - :, :seq_len, : - ] + subword_probs = nn.functional.softmax( + model_output.logits, dim=2 + )[:, :seq_len, :] error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, + batch[0]['input_ids'], + batch[0]['mt_offsets'], ) batch_prediction = Prediction( scores=model_output.score, diff --git a/comet/models/multitask/xcomet_metric.py b/comet/models/multitask/xcomet_metric.py index 76ccf883..59f7eadf 100644 --- a/comet/models/multitask/xcomet_metric.py +++ b/comet/models/multitask/xcomet_metric.py @@ -43,29 +43,34 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1.0e-06, learning_rate: float = 3.66e-06, layerwise_decay: float = 0.983, - encoder_model: str = "XLM-RoBERTa-XL", - pretrained_model: str = "facebook/xlm-roberta-xl", - sent_layer: Union[str, int] = "mix", - layer_transformation: str = "sparsemax", + encoder_model: str = 'XLM-RoBERTa-XL', + pretrained_model: str = 'facebook/xlm-roberta-xl', + sent_layer: Union[str, int] = 'mix', + layer_transformation: str = 'sparsemax', layer_norm: bool = False, word_layer: int = 36, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [2560, 1280], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, word_level_training: bool = True, - error_labels: List[str] = ["minor", "major", "critical"], + error_labels: List[str] = ['minor', 'major', 'critical'], loss_lambda: float = 0.055, - cross_entropy_weights: Optional[List[float]] = [0.08, 0.486, 0.505, 0.533], + cross_entropy_weights: Optional[List[float]] = [ + 0.08, + 0.486, + 0.505, + 0.533, + ], load_pretrained_weights: bool = True, local_files_only: bool = False, ) -> None: @@ -87,7 +92,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="xcomet_metric", + class_identifier='xcomet_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -98,8 +103,8 @@ def __init__( dropout=self.hparams.dropout, final_activation=self.hparams.final_activation, ) - assert error_labels == ["minor", "major", "critical"] - self.hparams.input_segments = ["mt", "src", "ref"] + assert error_labels == ['minor', 'major', 'critical'] + self.hparams.input_segments = ['mt', 'src', 'ref'] self.word_level = True self.encoder.labelset = self.label_encoder self.hidden2tag = nn.Linear(self.encoder.output_units, self.num_classes) @@ -145,11 +150,11 @@ def _compute_mqm_from_spans(error_spans): for sentence_spans in error_spans: sentence_score = 0 for annotation in sentence_spans: - if annotation["severity"] == "minor": + if annotation['severity'] == 'minor': sentence_score += 1 - elif annotation["severity"] == "major": + elif annotation['severity'] == 'major': sentence_score += 5 - elif annotation["severity"] == "critical": + elif annotation['severity'] == 'critical': sentence_score += 10 if sentence_score > 25: @@ -172,7 +177,7 @@ def _compute_mqm_from_spans(error_spans): ], dim=0, ).sum(dim=0) - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() @@ -183,12 +188,13 @@ def _compute_mqm_from_spans(error_spans): ] subword_probs = torch.sum(torch.stack(subword_probs), dim=0) error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, batch[0]['input_ids'], batch[0]['mt_offsets'] ) mqm_scores = _compute_mqm_from_spans(error_spans) final_scores = ( regression_scores - + mqm_scores.to(regression_scores.device) * self.score_weights[3] + + mqm_scores.to(regression_scores.device) + * self.score_weights[3] ) batch_prediction = Prediction( scores=final_scores, @@ -207,14 +213,14 @@ def _compute_mqm_from_spans(error_spans): regression_score = torch.where( model_output.score > 1.0, 1.0, model_output.score ) - mt_mask = batch[0]["label_ids"] != -1 + mt_mask = batch[0]['label_ids'] != -1 mt_length = mt_mask.sum(dim=1) seq_len = mt_length.max() subword_probs = nn.functional.softmax(model_output.logits, dim=2)[ :, :seq_len, : ] error_spans = self.decode( - subword_probs, batch[0]["input_ids"], batch[0]["mt_offsets"] + subword_probs, batch[0]['input_ids'], batch[0]['mt_offsets'] ) mqm_scores = _compute_mqm_from_spans(error_spans) final_scores = ( diff --git a/comet/models/pooling_utils.py b/comet/models/pooling_utils.py index 7c3d8edf..e57a332b 100644 --- a/comet/models/pooling_utils.py +++ b/comet/models/pooling_utils.py @@ -12,16 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import torch from typing import List, Union +import torch + + # From https://github.com/amazon-science/doc-mt-metrics/blob/5385cc28930aae9924edcb3201645dd3810b12c0/COMET/comet/models/pooling_utils.py#L18 def find_start_inds_and_mask_tokens( - mask: torch.Tensor, - tokens: torch.Tensor, - separator_index: int, + mask: torch.Tensor, + tokens: torch.Tensor, + separator_index: int, ) -> Union[List[int], torch.Tensor]: - """Finds the starting indices of each sentence for multi-sentence sequences and + """Finds the starting indices of each sentence for multi-sentence sequences and creates a new mask to omit all context sentences from the pooling function. Args: @@ -38,18 +40,19 @@ def find_start_inds_and_mask_tokens( # if there are more than one find where the last sentence starts ind = separators[-2].cpu().numpy().item() start_inds.append(ind) - ctx_mask[i, 1:ind+1] = 0 + ctx_mask[i, 1 : ind + 1] = 0 else: start_inds.append(0) return start_inds, ctx_mask + def average_pooling( tokens: torch.Tensor, embeddings: torch.Tensor, mask: torch.Tensor, padding_index: int, separator_index: int, - enable_context: bool = False + enable_context: bool = False, ) -> torch.Tensor: """Average pooling method. @@ -64,11 +67,17 @@ def average_pooling( torch.Tensor: Sentence embedding """ if enable_context: - start_inds, ctx_mask = find_start_inds_and_mask_tokens(mask, tokens, separator_index) - wordemb = mask_fill_index(0.0, tokens, embeddings, start_inds, padding_index) + start_inds, ctx_mask = find_start_inds_and_mask_tokens( + mask, tokens, separator_index + ) + wordemb = mask_fill_index( + 0.0, tokens, embeddings, start_inds, padding_index + ) sentemb = torch.sum(wordemb, 1) - sum_mask = ctx_mask.unsqueeze(-1).expand(embeddings.size()).float().sum(1) - else: + sum_mask = ( + ctx_mask.unsqueeze(-1).expand(embeddings.size()).float().sum(1) + ) + else: wordemb = mask_fill(0.0, tokens, embeddings, padding_index) sentemb = torch.sum(wordemb, 1) sum_mask = mask.unsqueeze(-1).expand(embeddings.size()).float().sum(1) @@ -89,15 +98,18 @@ def max_pooling( Return: torch.Tensor: Sentence embedding """ - return mask_fill(float("-inf"), tokens, embeddings, padding_index).max(dim=1)[0] + return mask_fill(float('-inf'), tokens, embeddings, padding_index).max( + dim=1 + )[0] + # From https://github.com/amazon-science/doc-mt-metrics/blob/5385cc28930aae9924edcb3201645dd3810b12c0/COMET/comet/models/pooling_utils.py#L18 def mask_fill_index( - fill_value: float, - tokens: torch.Tensor, - embeddings: torch.Tensor, - start_inds: list, - padding_index: int, + fill_value: float, + tokens: torch.Tensor, + embeddings: torch.Tensor, + start_inds: list, + padding_index: int, ) -> torch.Tensor: """ Masks embeddings representing padded elements and context sentences for multi-sentence sequences. @@ -108,16 +120,23 @@ def mask_fill_index( embeddings: word embeddings [bsz x seq_len x hiddens]. start_inds: Start of sentence indices. padding_index: Index of the padding token. - + Return: torch.Tensor: Sentence embedding """ padding_mask = tokens.eq(padding_index).unsqueeze(-1) - padding_maks2 = torch.zeros(tokens.shape, dtype=torch.bool, device=padding_mask.device) + padding_maks2 = torch.zeros( + tokens.shape, dtype=torch.bool, device=padding_mask.device + ) for i, start in enumerate(start_inds): - padding_maks2[i, 1: start+1] = True + padding_maks2[i, 1 : start + 1] = True padding_mask = torch.logical_or(padding_mask, padding_maks2.unsqueeze(-1)) - return embeddings.float().masked_fill_(padding_mask, fill_value).type_as(embeddings) + return ( + embeddings.float() + .masked_fill_(padding_mask, fill_value) + .type_as(embeddings) + ) + def mask_fill( fill_value: float, @@ -138,4 +157,8 @@ def mask_fill( torch.Tensor: Word embeddings [batch_size x seq_length x hidden_size] """ padding_mask = tokens.eq(padding_index).unsqueeze(-1) - return embeddings.float().masked_fill_(padding_mask, fill_value).type_as(embeddings) + return ( + embeddings.float() + .masked_fill_(padding_mask, fill_value) + .type_as(embeddings) + ) diff --git a/comet/models/predict_pbar.py b/comet/models/predict_pbar.py index 30bd0554..7218c17e 100644 --- a/comet/models/predict_pbar.py +++ b/comet/models/predict_pbar.py @@ -23,7 +23,7 @@ class PredictProgressBar(ptl.callbacks.progress.tqdm_progress.TQDMProgressBar): def init_predict_tqdm(self) -> tqdm: bar = tqdm( - desc="Predicting", + desc='Predicting', position=(2 * self.process_position), disable=self.is_disabled, leave=True, diff --git a/comet/models/predict_writer.py b/comet/models/predict_writer.py index 7d95989d..66079864 100644 --- a/comet/models/predict_writer.py +++ b/comet/models/predict_writer.py @@ -33,10 +33,12 @@ class CustomWriter(BasePredictionWriter): write_interval (str): When to perform write operations. Defaults to 'epoch' """ - def __init__(self, write_interval="epoch") -> None: + def __init__(self, write_interval='epoch') -> None: super().__init__(write_interval) - def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): + def write_on_epoch_end( + self, trainer, pl_module, predictions, batch_indices + ): """Saves predictions after running inference on all samples.""" # We need to save predictions in the most secure manner possible to avoid @@ -48,7 +50,7 @@ def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): tempfile.mkdtemp(), ] logger.info( - "Created temporary folder to store predictions: {}.".format( + 'Created temporary folder to store predictions: {}.'.format( output_dir[0] ) ) @@ -67,13 +69,16 @@ def write_on_epoch_end(self, trainer, pl_module, predictions, batch_indices): # this will create N (num processes) files in `output_dir` each containing # the predictions of it's respective rank torch.save( - predictions, os.path.join(self.output_dir, f"pred_{trainer.global_rank}.pt") + predictions, + os.path.join(self.output_dir, f'pred_{trainer.global_rank}.pt'), ) # optionally, you can also save `batch_indices` to get the information about # the data index from your prediction data torch.save( batch_indices, - os.path.join(self.output_dir, f"batch_indices_{trainer.global_rank}.pt"), + os.path.join( + self.output_dir, f'batch_indices_{trainer.global_rank}.pt' + ), ) def gather_all_predictions(self): @@ -86,10 +91,12 @@ def flatten(list): def flatten_predictions(predictions): flatten_pred = Prediction( - scores=torch.cat([pred["scores"] for pred in predictions], dim=0) + scores=torch.cat( + [pred['scores'] for pred in predictions], dim=0 + ) ) - if "metadata" in predictions[0]: - flatten_pred["metadata"] = flatten_metadata( + if 'metadata' in predictions[0]: + flatten_pred['metadata'] = flatten_metadata( [pred.metadata for pred in predictions] ) return flatten_pred @@ -97,29 +104,34 @@ def flatten_predictions(predictions): files = sorted(os.listdir(self.output_dir)) pred = flatten_predictions( [ - flatten_predictions(torch.load(os.path.join(self.output_dir, f))) + flatten_predictions( + torch.load(os.path.join(self.output_dir, f)) + ) for f in files - if "pred" in f + if 'pred' in f ] ) indices = flatten( [ flatten(torch.load(os.path.join(self.output_dir, f))[0]) for f in files - if "batch_indices" in f + if 'batch_indices' in f ] ) output = Prediction( scores=restore_list_order(pred.scores.tolist(), indices), system_score=sum(pred.scores.tolist()) / len(pred.scores), ) - if "metadata" in pred: - output["metadata"] = Prediction( - **{k: restore_list_order(v, indices) for k, v in pred.metadata.items()} + if 'metadata' in pred: + output['metadata'] = Prediction( + **{ + k: restore_list_order(v, indices) + for k, v in pred.metadata.items() + } ) return output def cleanup(self): """Cleans temporary files.""" - logger.info("Cleanup temporary folder: {}.".format(self.output_dir)) + logger.info('Cleanup temporary folder: {}.'.format(self.output_dir)) shutil.rmtree(self.output_dir) diff --git a/comet/models/ranking/ranking_metric.py b/comet/models/ranking/ranking_metric.py index 111e85f1..b252c79a 100644 --- a/comet/models/ranking/ranking_metric.py +++ b/comet/models/ranking/ranking_metric.py @@ -22,13 +22,15 @@ `good` translations closer to the anchors (source & reference) than `worse` translations. """ + from typing import Dict, List, Optional, Tuple, Union import pandas as pd import torch import torch.nn.functional as F from torch import nn -from transformers.optimization import Adafactor, get_constant_schedule_with_warmup +from transformers.optimization import (Adafactor, + get_constant_schedule_with_warmup) from comet.models.base import CometModel from comet.models.metrics import WMTKendall @@ -75,18 +77,18 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.1, keep_embeddings_frozen: bool = False, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1e-05, learning_rate: float = 3e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-base", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-base', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "triplet-margin", + loss: str = 'triplet-margin', dropout: float = 0.1, batch_size: int = 8, train_data: List[str] = [], @@ -112,7 +114,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="ranking_metric", + class_identifier='ranking_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -120,7 +122,7 @@ def __init__( def init_metrics(self): """Initializes train/validation metrics.""" - self.train_metrics = WMTKendall(prefix="train") + self.train_metrics = WMTKendall(prefix='train') self.val_metrics = nn.ModuleList( [WMTKendall(prefix=d) for d in self.hparams.validation_data] ) @@ -134,7 +136,9 @@ def loss(self): def configure_optimizers( self, - ) -> Tuple[List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR]]: + ) -> Tuple[ + List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR] + ]: """Pytorch Lightning method to configure optimizers and schedulers.""" layer_parameters = self.encoder.layerwise_lr( self.hparams.encoder_learning_rate, self.hparams.layerwise_decay @@ -142,15 +146,15 @@ def configure_optimizers( if self.layerwise_attention: layerwise_attn_params = [ { - "params": self.layerwise_attention.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.layerwise_attention.parameters(), + 'lr': self.hparams.learning_rate, } ] params = layer_parameters + layerwise_attn_params else: params = layer_parameters - if self.hparams.optimizer == "Adafactor": + if self.hparams.optimizer == 'Adafactor': optimizer = Adafactor( params, lr=self.hparams.learning_rate, @@ -171,7 +175,7 @@ def configure_optimizers( return [optimizer], [scheduler] def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "fit" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'fit' ) -> Dict[str, torch.Tensor]: """This method will be called by dataloaders to prepared data to input to the model. @@ -188,26 +192,26 @@ def prepare_sample( """ sample = {k: [str(dic[k]) for dic in sample] for k in sample[0]} - if stage == "predict": - src_inputs = self.encoder.prepare_sample(sample["src"]) - mt_inputs = self.encoder.prepare_sample(sample["mt"]) - ref_inputs = self.encoder.prepare_sample(sample["ref"]) + if stage == 'predict': + src_inputs = self.encoder.prepare_sample(sample['src']) + mt_inputs = self.encoder.prepare_sample(sample['mt']) + ref_inputs = self.encoder.prepare_sample(sample['ref']) - ref_inputs = {"ref_" + k: v for k, v in ref_inputs.items()} - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - mt_inputs = {"mt_" + k: v for k, v in mt_inputs.items()} + ref_inputs = {'ref_' + k: v for k, v in ref_inputs.items()} + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + mt_inputs = {'mt_' + k: v for k, v in mt_inputs.items()} return {**ref_inputs, **src_inputs, **mt_inputs} - ref_inputs = self.encoder.prepare_sample(sample["ref"]) - src_inputs = self.encoder.prepare_sample(sample["src"]) - pos_inputs = self.encoder.prepare_sample(sample["pos"]) - neg_inputs = self.encoder.prepare_sample(sample["neg"]) + ref_inputs = self.encoder.prepare_sample(sample['ref']) + src_inputs = self.encoder.prepare_sample(sample['src']) + pos_inputs = self.encoder.prepare_sample(sample['pos']) + neg_inputs = self.encoder.prepare_sample(sample['neg']) - ref_inputs = {"ref_" + k: v for k, v in ref_inputs.items()} - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - pos_inputs = {"pos_" + k: v for k, v in pos_inputs.items()} - neg_inputs = {"neg_" + k: v for k, v in neg_inputs.items()} + ref_inputs = {'ref_' + k: v for k, v in ref_inputs.items()} + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + pos_inputs = {'pos_' + k: v for k, v in pos_inputs.items()} + neg_inputs = {'neg_' + k: v for k, v in neg_inputs.items()} return {**ref_inputs, **src_inputs, **pos_inputs, **neg_inputs} @@ -240,10 +244,18 @@ def forward( Dictionary with triplet loss, distance between anchors and positive samples and distance between anchors and negative samples. """ - src_sentemb = self.get_sentence_embedding(src_input_ids, src_attention_mask) - ref_sentemb = self.get_sentence_embedding(ref_input_ids, ref_attention_mask) - pos_sentemb = self.get_sentence_embedding(pos_input_ids, pos_attention_mask) - neg_sentemb = self.get_sentence_embedding(neg_input_ids, neg_attention_mask) + src_sentemb = self.get_sentence_embedding( + src_input_ids, src_attention_mask + ) + ref_sentemb = self.get_sentence_embedding( + ref_input_ids, ref_attention_mask + ) + pos_sentemb = self.get_sentence_embedding( + pos_input_ids, pos_attention_mask + ) + neg_sentemb = self.get_sentence_embedding( + neg_input_ids, neg_attention_mask + ) loss = self.loss(src_sentemb, pos_sentemb, neg_sentemb) + self.loss( ref_sentemb, pos_sentemb, neg_sentemb @@ -264,9 +276,9 @@ def forward( ) return { - "loss": loss, - "distance_pos": distance_pos, - "distance_neg": distance_neg, + 'loss': loss, + 'distance_pos': distance_pos, + 'distance_neg': distance_neg, } def read_training_data(self, path: str) -> List[dict]: @@ -277,12 +289,12 @@ def read_training_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - df = df[["src", "pos", "neg", "ref"]] - df["src"] = df["src"].astype(str) - df["pos"] = df["pos"].astype(str) - df["neg"] = df["neg"].astype(str) - df["ref"] = df["ref"].astype(str) - return df.to_dict("records") + df = df[['src', 'pos', 'neg', 'ref']] + df['src'] = df['src'].astype(str) + df['pos'] = df['pos'].astype(str) + df['neg'] = df['neg'].astype(str) + df['ref'] = df['ref'].astype(str) + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Method that reads the validation data (a csv file) and returns a list of @@ -308,7 +320,7 @@ def training_step( [torch.Tensor] Loss value """ batch_prediction = self.forward(**batch) - loss_value = batch_prediction["loss"] + loss_value = batch_prediction['loss'] if ( self.nr_frozen_epochs < 1.0 @@ -318,7 +330,7 @@ def training_step( self.unfreeze_encoder() self._frozen = False - self.log("train_loss", loss_value, on_step=True, on_epoch=True) + self.log('train_loss', loss_value, on_step=True, on_epoch=True) return loss_value def validation_step( @@ -334,16 +346,18 @@ def validation_step( batch_idx (int): Integer displaying which batch this is. """ batch_prediction = self.forward(**batch) - loss_value = batch_prediction["loss"] - self.log("val_loss", loss_value, on_step=True, on_epoch=True) + loss_value = batch_prediction['loss'] + self.log('val_loss', loss_value, on_step=True, on_epoch=True) if dataloader_idx == 0: self.train_metrics.update( - batch_prediction["distance_pos"], batch_prediction["distance_neg"] + batch_prediction['distance_pos'], + batch_prediction['distance_neg'], ) elif dataloader_idx > 0: self.val_metrics[dataloader_idx - 1].update( - batch_prediction["distance_pos"], batch_prediction["distance_neg"] + batch_prediction['distance_pos'], + batch_prediction['distance_neg'], ) def predict_step( @@ -366,13 +380,13 @@ def predict_step( def _predict_forward(batch): src_sentemb = self.get_sentence_embedding( - batch["src_input_ids"], batch["src_attention_mask"] + batch['src_input_ids'], batch['src_attention_mask'] ) ref_sentemb = self.get_sentence_embedding( - batch["ref_input_ids"], batch["ref_attention_mask"] + batch['ref_input_ids'], batch['ref_attention_mask'] ) mt_sentemb = self.get_sentence_embedding( - batch["mt_input_ids"], batch["mt_attention_mask"] + batch['mt_input_ids'], batch['mt_attention_mask'] ) src_distance = F.pairwise_distance(mt_sentemb, src_sentemb) ref_distance = F.pairwise_distance(mt_sentemb, ref_sentemb) @@ -388,7 +402,7 @@ def _predict_forward(batch): ) if self.mc_dropout: - raise NotImplementedError("MCD not implemented for this model!") + raise NotImplementedError('MCD not implemented for this model!') else: return _predict_forward(batch) diff --git a/comet/models/regression/referenceless.py b/comet/models/regression/referenceless.py index 8a95f6e5..127965e0 100644 --- a/comet/models/regression/referenceless.py +++ b/comet/models/regression/referenceless.py @@ -19,6 +19,7 @@ Referenceless Regression Metric that learns to predict a quality assessment by looking at source and translation. """ + from typing import Dict, List, Optional, Tuple, Union import pandas as pd @@ -72,24 +73,24 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1e-06, learning_rate: float = 1.5e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-large", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-large', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [2048, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, load_pretrained_weights: bool = True, local_files_only: bool = False, @@ -113,7 +114,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="referenceless_regression_metric", + class_identifier='referenceless_regression_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -129,15 +130,16 @@ def __init__( def requires_references(self) -> bool: return False - + def enable_context(self): - if self.pool == "avg": + if self.pool == 'avg': self.use_context = True def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "train" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'train' ) -> Union[ - Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], Dict[str, torch.Tensor] + Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], + Dict[str, torch.Tensor], ]: """This method will be called by dataloaders to prepared data to input to the model. @@ -150,22 +152,26 @@ def prepare_sample( Returns: Model inputs and depending on the 'stage' training labels/targets. """ - inputs = {k: [str(dic[k]) for dic in sample] for k in sample[0] if k != "score"} - src_inputs = self.encoder.prepare_sample(inputs["src"]) - mt_inputs = self.encoder.prepare_sample(inputs["mt"]) - - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - mt_inputs = {"mt_" + k: v for k, v in mt_inputs.items()} + inputs = { + k: [str(dic[k]) for dic in sample] + for k in sample[0] + if k != 'score' + } + src_inputs = self.encoder.prepare_sample(inputs['src']) + mt_inputs = self.encoder.prepare_sample(inputs['mt']) + + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + mt_inputs = {'mt_' + k: v for k, v in mt_inputs.items()} model_inputs = {**src_inputs, **mt_inputs} - if stage == "predict": + if stage == 'predict': return model_inputs - - scores = [float(s["score"]) for s in sample] + + scores = [float(s['score']) for s in sample] targets = Target(score=torch.tensor(scores, dtype=torch.float)) - if "system" in inputs: - targets["system"] = inputs["system"] + if 'system' in inputs: + targets['system'] = inputs['system'] return model_inputs, targets @@ -175,7 +181,7 @@ def forward( src_attention_mask: torch.tensor, mt_input_ids: torch.tensor, mt_attention_mask: torch.tensor, - **kwargs + **kwargs, ) -> Dict[str, torch.Tensor]: """ReferencelessRegression model forward method. @@ -188,8 +194,12 @@ def forward( Return: Prediction object with translation scores. """ - src_sentemb = self.get_sentence_embedding(src_input_ids, src_attention_mask) - mt_sentemb = self.get_sentence_embedding(mt_input_ids, mt_attention_mask) + src_sentemb = self.get_sentence_embedding( + src_input_ids, src_attention_mask + ) + mt_sentemb = self.get_sentence_embedding( + mt_input_ids, mt_attention_mask + ) diff_src = torch.abs(mt_sentemb - src_sentemb) prod_src = mt_sentemb * src_sentemb @@ -207,11 +217,11 @@ def read_training_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - df = df[["src", "mt", "score"]] - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - df["score"] = df["score"].astype("float16") - return df.to_dict("records") + df = df[['src', 'mt', 'score']] + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + df['score'] = df['score'].astype('float16') + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Method that reads the validation data (a csv file) and returns a list of @@ -221,14 +231,14 @@ def read_validation_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - columns = ["src", "mt", "score"] + columns = ['src', 'mt', 'score'] # If system in columns we will use this to calculate system-level accuracy - if "system" in df.columns: - columns.append("system") - df["system"] = df["system"].astype(str) + if 'system' in df.columns: + columns.append('system') + df['system'] = df['system'].astype(str) df = df[columns] - df["score"] = df["score"].astype("float16") - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - return df.to_dict("records") + df['score'] = df['score'].astype('float16') + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + return df.to_dict('records') diff --git a/comet/models/regression/regression_metric.py b/comet/models/regression/regression_metric.py index ba810e28..930ab991 100644 --- a/comet/models/regression/regression_metric.py +++ b/comet/models/regression/regression_metric.py @@ -19,12 +19,14 @@ Regression Metric that learns to predict a quality assessment by looking at source, translation and reference. """ + from typing import Dict, List, Optional, Tuple, Union import pandas as pd import torch from torch import nn -from transformers.optimization import Adafactor, get_constant_schedule_with_warmup +from transformers.optimization import (Adafactor, + get_constant_schedule_with_warmup) from comet.models.base import CometModel from comet.models.metrics import RegressionMetrics @@ -75,24 +77,24 @@ def __init__( self, nr_frozen_epochs: Union[float, int] = 0.3, keep_embeddings_frozen: bool = True, - optimizer: str = "AdamW", + optimizer: str = 'AdamW', warmup_steps: int = 0, encoder_learning_rate: float = 1e-06, learning_rate: float = 1.5e-05, layerwise_decay: float = 0.95, - encoder_model: str = "XLM-RoBERTa", - pretrained_model: str = "xlm-roberta-large", - pool: str = "avg", - layer: Union[str, int] = "mix", - layer_transformation: str = "softmax", + encoder_model: str = 'XLM-RoBERTa', + pretrained_model: str = 'xlm-roberta-large', + pool: str = 'avg', + layer: Union[str, int] = 'mix', + layer_transformation: str = 'softmax', layer_norm: bool = True, - loss: str = "mse", + loss: str = 'mse', dropout: float = 0.1, batch_size: int = 4, train_data: List[str] = [], validation_data: List[str] = [], hidden_sizes: List[int] = [3072, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, load_pretrained_weights: bool = True, local_files_only: bool = False, @@ -116,7 +118,7 @@ def __init__( batch_size=batch_size, train_data=train_data, validation_data=validation_data, - class_identifier="regression_metric", + class_identifier='regression_metric', load_pretrained_weights=load_pretrained_weights, local_files_only=local_files_only, ) @@ -132,7 +134,7 @@ def __init__( def init_metrics(self): """Initializes train/validation metrics.""" - self.train_metrics = RegressionMetrics(prefix="train") + self.train_metrics = RegressionMetrics(prefix='train') self.val_metrics = nn.ModuleList( [RegressionMetrics(prefix=d) for d in self.hparams.validation_data] ) @@ -142,26 +144,33 @@ def requires_references(self) -> bool: def configure_optimizers( self, - ) -> Tuple[List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR]]: + ) -> Tuple[ + List[torch.optim.Optimizer], List[torch.optim.lr_scheduler.LambdaLR] + ]: """Pytorch Lightning method to configure optimizers and schedulers.""" layer_parameters = self.encoder.layerwise_lr( self.hparams.encoder_learning_rate, self.hparams.layerwise_decay ) top_layers_parameters = [ - {"params": self.estimator.parameters(), "lr": self.hparams.learning_rate} + { + 'params': self.estimator.parameters(), + 'lr': self.hparams.learning_rate, + } ] if self.layerwise_attention: layerwise_attn_params = [ { - "params": self.layerwise_attention.parameters(), - "lr": self.hparams.learning_rate, + 'params': self.layerwise_attention.parameters(), + 'lr': self.hparams.learning_rate, } ] - params = layer_parameters + top_layers_parameters + layerwise_attn_params + params = ( + layer_parameters + top_layers_parameters + layerwise_attn_params + ) else: params = layer_parameters + top_layers_parameters - if self.hparams.optimizer == "Adafactor": + if self.hparams.optimizer == 'Adafactor': optimizer = Adafactor( params, lr=self.hparams.learning_rate, @@ -182,9 +191,10 @@ def configure_optimizers( return [optimizer], [scheduler] def prepare_sample( - self, sample: List[Dict[str, Union[str, float]]], stage: str = "train" + self, sample: List[Dict[str, Union[str, float]]], stage: str = 'train' ) -> Union[ - Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], Dict[str, torch.Tensor] + Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]], + Dict[str, torch.Tensor], ]: """This method will be called by dataloaders to prepared data to input to the model. @@ -197,29 +207,33 @@ def prepare_sample( Returns: Model inputs and depending on the 'stage' training labels/targets. """ - inputs = {k: [str(dic[k]) for dic in sample] for k in sample[0] if k != "score"} - src_inputs = self.encoder.prepare_sample(inputs["src"]) - mt_inputs = self.encoder.prepare_sample(inputs["mt"]) - ref_inputs = self.encoder.prepare_sample(inputs["ref"]) - - src_inputs = {"src_" + k: v for k, v in src_inputs.items()} - mt_inputs = {"mt_" + k: v for k, v in mt_inputs.items()} - ref_inputs = {"ref_" + k: v for k, v in ref_inputs.items()} + inputs = { + k: [str(dic[k]) for dic in sample] + for k in sample[0] + if k != 'score' + } + src_inputs = self.encoder.prepare_sample(inputs['src']) + mt_inputs = self.encoder.prepare_sample(inputs['mt']) + ref_inputs = self.encoder.prepare_sample(inputs['ref']) + + src_inputs = {'src_' + k: v for k, v in src_inputs.items()} + mt_inputs = {'mt_' + k: v for k, v in mt_inputs.items()} + ref_inputs = {'ref_' + k: v for k, v in ref_inputs.items()} model_inputs = {**src_inputs, **mt_inputs, **ref_inputs} - if stage == "predict": + if stage == 'predict': return model_inputs - scores = [float(s["score"]) for s in sample] + scores = [float(s['score']) for s in sample] targets = Target(score=torch.tensor(scores, dtype=torch.float)) - if "system" in inputs: - targets["system"] = inputs["system"] + if 'system' in inputs: + targets['system'] = inputs['system'] return model_inputs, targets def enable_context(self): - if self.pool == "avg": + if self.pool == 'avg': self.use_context = True def estimate( @@ -259,7 +273,7 @@ def forward( mt_attention_mask: torch.tensor, ref_input_ids: torch.tensor, ref_attention_mask: torch.tensor, - **kwargs + **kwargs, ) -> Prediction: """Regression model forward method. @@ -274,9 +288,15 @@ def forward( Return: Prediction object with translation scores. """ - src_sentemb = self.get_sentence_embedding(src_input_ids, src_attention_mask) - ref_sentemb = self.get_sentence_embedding(ref_input_ids, ref_attention_mask) - mt_sentemb = self.get_sentence_embedding(mt_input_ids, mt_attention_mask) + src_sentemb = self.get_sentence_embedding( + src_input_ids, src_attention_mask + ) + ref_sentemb = self.get_sentence_embedding( + ref_input_ids, ref_attention_mask + ) + mt_sentemb = self.get_sentence_embedding( + mt_input_ids, mt_attention_mask + ) return self.estimate(src_sentemb, mt_sentemb, ref_sentemb) def read_training_data(self, path: str) -> List[dict]: @@ -287,12 +307,12 @@ def read_training_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - df = df[["src", "mt", "ref", "score"]] - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - df["ref"] = df["ref"].astype(str) - df["score"] = df["score"].astype("float16") - return df.to_dict("records") + df = df[['src', 'mt', 'ref', 'score']] + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + df['ref'] = df['ref'].astype(str) + df['score'] = df['score'].astype('float16') + return df.to_dict('records') def read_validation_data(self, path: str) -> List[dict]: """Method that reads the validation data (a csv file) and returns a list of @@ -302,15 +322,15 @@ def read_validation_data(self, path: str) -> List[dict]: List[dict]: List with input samples in the form of a dict """ df = pd.read_csv(path) - columns = ["src", "mt", "ref", "score"] + columns = ['src', 'mt', 'ref', 'score'] # If system in columns we will use this to calculate system-level accuracy - if "system" in df.columns: - columns.append("system") - df["system"] = df["system"].astype(str) + if 'system' in df.columns: + columns.append('system') + df['system'] = df['system'].astype(str) df = df[columns] - df["score"] = df["score"].astype("float16") - df["src"] = df["src"].astype(str) - df["mt"] = df["mt"].astype(str) - df["ref"] = df["ref"].astype(str) - return df.to_dict("records") + df['score'] = df['score'].astype('float16') + df['src'] = df['src'].astype(str) + df['mt'] = df['mt'].astype(str) + df['ref'] = df['ref'].astype(str) + return df.to_dict('records') diff --git a/comet/models/utils.py b/comet/models/utils.py index a360f07a..b9343c1d 100644 --- a/comet/models/utils.py +++ b/comet/models/utils.py @@ -13,26 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. import itertools -from typing import List, Tuple, Any +from collections import OrderedDict +from typing import Any, List, Tuple import torch from torch.utils.data import Sampler -from collections import OrderedDict class ModelOutput(OrderedDict): - """ This was copied from previous versions of HuggingFace Transformers. Latest + """This was copied from previous versions of HuggingFace Transformers. Latest version made some breaking changes into ModelOutputs which impacted Prediction and Target classes defined bellow. Base class for all model outputs as dataclass. Has a `__getitem__` that allows - indexing by integer or slice (like a tuple) or strings (like a dictionary) + indexing by integer or slice (like a tuple) or strings (like a dictionary) that will ignore the `None` attributes. Otherwise behaves like a regular python dictionary. - You can't unpack a `ModelOutput` directly. Use the [`to_tuple`] method to + You can't unpack a `ModelOutput` directly. Use the [`to_tuple`] method to convert it to a tuple before. @@ -43,12 +43,16 @@ def __post_init__(self): # Safety and consistency checks if not len(class_fields): - raise ValueError(f"{self.__class__.__name__} has no fields.") + raise ValueError(f'{self.__class__.__name__} has no fields.') if not all(field.default is None for field in class_fields[1:]): - raise ValueError(f"{self.__class__.__name__} should not have more than one required field.") + raise ValueError( + f'{self.__class__.__name__} should not have more than one required field.' + ) first_field = getattr(self, class_fields[0].name) - other_fields_are_none = all(getattr(self, field.name) is None for field in class_fields[1:]) + other_fields_are_none = all( + getattr(self, field.name) is None for field in class_fields[1:] + ) if other_fields_are_none and not is_tensor(first_field): if isinstance(first_field, dict): @@ -83,16 +87,24 @@ def __post_init__(self): self[field.name] = v def __delitem__(self, *args, **kwargs): - raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + raise Exception( + f'You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.' + ) def setdefault(self, *args, **kwargs): - raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + raise Exception( + f'You cannot use ``setdefault`` on a {self.__class__.__name__} instance.' + ) def pop(self, *args, **kwargs): - raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + raise Exception( + f'You cannot use ``pop`` on a {self.__class__.__name__} instance.' + ) def update(self, *args, **kwargs): - raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + raise Exception( + f'You cannot use ``update`` on a {self.__class__.__name__} instance.' + ) def __getitem__(self, k): if isinstance(k, str): @@ -135,16 +147,16 @@ class Target(ModelOutput): class LabelSet: """Taken from: https://github.com/LightTag/sequence-labeling-with-transformers/""" - def __init__(self, labels: List[str] = ["minor", "major", "critical"]): + def __init__(self, labels: List[str] = ['minor', 'major', 'critical']): self.labels_to_id = {} self.ids_to_label = {} - self.labels_to_id["O"] = 0 - self.ids_to_label[0] = "O" + self.labels_to_id['O'] = 0 + self.ids_to_label[0] = 'O' num = 0 # in case there are no labels # Writing BILU will give us incremntal ids for the labels - for _num, (label, s) in enumerate(itertools.product(labels, "I")): + for _num, (label, s) in enumerate(itertools.product(labels, 'I')): num = _num + 1 # skip 0 - l = f"{s}-{label}" + l = f'{s}-{label}' self.labels_to_id[l] = num self.ids_to_label[num] = l @@ -166,7 +178,9 @@ def flatten_metadata(metadata): """Metadata from the model output can be in various forms and this function will gather all metadata and flatten everything. """ - metadata = Prediction(**{k: [dic[k] for dic in metadata] for k in metadata[0]}) + metadata = Prediction( + **{k: [dic[k] for dic in metadata] for k in metadata[0]} + ) for k, v in metadata.items(): if torch.is_tensor(v[0]): # If we have tensors we can use cat to flatten them. diff --git a/comet/modules/feedforward.py b/comet/modules/feedforward.py index b7958bb6..9471a4dc 100644 --- a/comet/modules/feedforward.py +++ b/comet/modules/feedforward.py @@ -17,6 +17,7 @@ ============ Feed Forward Neural Network module that can be used for classification or regression """ + from typing import List, Optional import torch @@ -41,7 +42,7 @@ def __init__( in_dim: int, out_dim: int = 1, hidden_sizes: List[int] = [3072, 1024], - activations: str = "Tanh", + activations: str = 'Tanh', final_activation: Optional[str] = None, dropout: float = 0.1, ) -> None: @@ -66,7 +67,7 @@ def build_activation(self, activation: str) -> nn.Module: if hasattr(nn, activation.title()): return getattr(nn, activation.title())() else: - raise Exception(f"{activation} is not a valid activation function!") + raise Exception(f'{activation} is not a valid activation function!') def forward(self, in_features: torch.Tensor) -> torch.Tensor: # When casting models to float 16 self.ff(in_features) was giving some problems reported @@ -74,9 +75,9 @@ def forward(self, in_features: torch.Tensor) -> torch.Tensor: # Check the dtype of self.ff parameters ff_dtypes = {param.dtype for param in self.ff.parameters()} - + # If all parameters are float16 and in_features is not, convert it if ff_dtypes == {torch.float16} and in_features.dtype != torch.float16: in_features = in_features.to(torch.float16) - + return self.ff(in_features) diff --git a/comet/modules/layerwise_attention.py b/comet/modules/layerwise_attention.py index 859edb7f..47a608f6 100644 --- a/comet/modules/layerwise_attention.py +++ b/comet/modules/layerwise_attention.py @@ -26,6 +26,7 @@ Original implementation: - https://github.com/Hyperparticle/udify """ + from typing import List, Optional import torch @@ -39,7 +40,7 @@ def __init__( layer_norm: bool = False, layer_weights: Optional[List[int]] = None, dropout: float = None, - layer_transformation: str = "softmax", + layer_transformation: str = 'softmax', ) -> None: super(LayerwiseAttention, self).__init__() self.num_layers = num_layers @@ -47,19 +48,16 @@ def __init__( self.dropout = dropout self.transform_fn = torch.softmax - if layer_transformation == "sparsemax": + if layer_transformation == 'sparsemax': from entmax import sparsemax + self.transform_fn = sparsemax if layer_weights is None: layer_weights = [0.0] * num_layers elif len(layer_weights) != num_layers: - raise Exception( - "Length of layer_weights {} differs \ - from num_layers {}".format( - layer_weights, num_layers - ) - ) + raise Exception('Length of layer_weights {} differs \ + from num_layers {}'.format(layer_weights, num_layers)) self.scalar_parameters = ParameterList( [ @@ -76,8 +74,8 @@ def __init__( if self.dropout: dropout_mask = torch.zeros(len(self.scalar_parameters)) dropout_fill = torch.empty(len(self.scalar_parameters)).fill_(-1e20) - self.register_buffer("dropout_mask", dropout_mask) - self.register_buffer("dropout_fill", dropout_fill) + self.register_buffer('dropout_mask', dropout_mask) + self.register_buffer('dropout_fill', dropout_fill) def forward( self, @@ -86,10 +84,8 @@ def forward( ) -> torch.Tensor: if len(tensors) != self.num_layers: raise Exception( - "{} tensors were passed, but the module was initialized to \ - mix {} tensors.".format( - len(tensors), self.num_layers - ) + '{} tensors were passed, but the module was initialized to \ + mix {} tensors.'.format(len(tensors), self.num_layers) ) def _layer_norm(tensor, broadcast_mask, mask): @@ -104,23 +100,30 @@ def _layer_norm(tensor, broadcast_mask, mask): variance = (((tensor_masked - mean) * broadcast_mask) ** 2).view( batch_size, -1 ).sum(1) / num_elements_not_masked - normalized_tensor = (tensor - mean) / torch.sqrt(variance + 1e-12).view( - batch_size, 1, 1 - ) + normalized_tensor = (tensor - mean) / torch.sqrt( + variance + 1e-12 + ).view(batch_size, 1, 1) return normalized_tensor # BUG: Pytorch bug fix when Parameters are not well copied across GPUs # https://github.com/pytorch/pytorch/issues/36035 - if len([parameter for parameter in self.scalar_parameters]) != self.num_layers: + if ( + len([parameter for parameter in self.scalar_parameters]) + != self.num_layers + ): weights = torch.tensor(self.weights, device=tensors[0].device) gamma = torch.tensor(self.gamma_value, device=tensors[0].device) else: - weights = torch.cat([parameter for parameter in self.scalar_parameters]) + weights = torch.cat( + [parameter for parameter in self.scalar_parameters] + ) gamma = self.gamma if self.training and self.dropout: weights = torch.where( - self.dropout_mask.uniform_() > self.dropout, weights, self.dropout_fill + self.dropout_mask.uniform_() > self.dropout, + weights, + self.dropout_fill, ) normed_weights = self.transform_fn(weights, dim=0) @@ -138,5 +141,7 @@ def _layer_norm(tensor, broadcast_mask, mask): pieces = [] for weight, tensor in zip(normed_weights, tensors): - pieces.append(weight * _layer_norm(tensor, broadcast_mask, mask_float)) + pieces.append( + weight * _layer_norm(tensor, broadcast_mask, mask_float) + ) return gamma * sum(pieces) diff --git a/docs/source/conf.py b/docs/source/conf.py index cd88a5e6..da29f9a7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,15 +17,15 @@ # -- Project information ----------------------------------------------------- -project = "COMET" +project = 'COMET' copyright = ( - "2020, Unbabel. All rights reserved." - "Source code available under Apache License 2.0" + '2020, Unbabel. All rights reserved.' + 'Source code available under Apache License 2.0' ) -author = "Unbabel" +author = 'Unbabel' # The full version, including alpha/beta/rc tags -release = "0.0.3" +release = '0.0.3' # -- General configuration --------------------------------------------------- @@ -34,32 +34,32 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.autodoc", + 'sphinx.ext.autodoc', # 'sphinx.ext.doctest', - "sphinx.ext.intersphinx", - "sphinx.ext.todo", - "sphinx.ext.coverage", - "sphinx.ext.mathjax", - "sphinx.ext.viewcode", - "sphinx.ext.githubpages", - "sphinx.ext.napoleon", - "recommonmark", + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', + 'recommonmark', #'sphinxarg.ext', #'m2r', # 'sphinx-issues', # 'pytest-sphinx', - "sphinx_markdown_tables", - "sphinx.ext.autosectionlabel", + 'sphinx_markdown_tables', + 'sphinx.ext.autosectionlabel', ] autosectionlabel_prefix_document = True # Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] +templates_path = ['_templates'] source_suffix = { - ".rst": "restructuredtext", - ".txt": "markdown", - ".md": "markdown", + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', } # List of patterns, relative to source directory, that match files and @@ -68,12 +68,12 @@ exclude_patterns = [] # The master toctree document. -master_doc = "index" +master_doc = 'index' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -85,14 +85,14 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] +html_static_path = ['_static'] def setup(app): - app.add_css_file("css/comet.css") + app.add_css_file('css/comet.css') diff --git a/poetry.lock b/poetry.lock index aa407bb3..ff91f77b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -6,6 +6,7 @@ version = "2.4.4" description = "Happy Eyeballs for asyncio" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiohappyeyeballs-2.4.4-py3-none-any.whl", hash = "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"}, {file = "aiohappyeyeballs-2.4.4.tar.gz", hash = "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745"}, @@ -17,6 +18,7 @@ version = "3.10.11" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5077b1a5f40ffa3ba1f40d537d3bec4383988ee51fbba6b74aa8fb1bc466599e"}, {file = "aiohttp-3.10.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8d6a14a4d93b5b3c2891fca94fa9d41b2322a68194422bef0dd5ec1e57d7d298"}, @@ -121,7 +123,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.12.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] +speedups = ["Brotli ; platform_python_implementation == \"CPython\"", "aiodns (>=3.2.0) ; sys_platform == \"linux\" or sys_platform == \"darwin\"", "brotlicffi ; platform_python_implementation != \"CPython\""] [[package]] name = "aiosignal" @@ -129,6 +131,7 @@ version = "1.3.1" description = "aiosignal: a list of registered asynchronous callbacks" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"}, {file = "aiosignal-1.3.1.tar.gz", hash = "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc"}, @@ -137,12 +140,46 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "annotated-doc" +version = "0.0.4" +description = "Document parameters, class attributes, return types, and variables inline, with Annotated." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320"}, + {file = "annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4"}, +] + +[[package]] +name = "anyio" +version = "4.13.0" +description = "High-level concurrency and networking framework on top of asyncio or Trio" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708"}, + {file = "anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +trio = ["trio (>=0.32.0)"] + [[package]] name = "async-timeout" version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.11\"" files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, @@ -154,18 +191,19 @@ version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""] [[package]] name = "certifi" @@ -173,131 +211,34 @@ version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] -name = "charset-normalizer" -version = "3.4.0" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, - {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, - {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, +name = "click" +version = "8.3.3" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613"}, + {file = "click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2"}, ] +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -309,6 +250,7 @@ version = "5.5" description = "Code coverage measurement for Python" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +groups = ["dev"] files = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, {file = "coverage-5.5-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b"}, @@ -367,12 +309,112 @@ files = [ [package.extras] toml = ["toml"] +[[package]] +name = "cuda-bindings" +version = "13.2.0" +description = "Python bindings for CUDA" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556"}, + {file = "cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6f3682ec3c4769326aafc67c2ba669d97d688d0b7e63e659d36d2f8b72f32d6"}, + {file = "cuda_bindings-13.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:845025438a1b9e20718b9fb42add3e0eb72e85458bcab3eeb80bfd8f0a9dab33"}, + {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:721104c603f059780d287969be3d194a18d0cc3b713ed9049065a1107706759d"}, + {file = "cuda_bindings-13.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1eba9504ac70667dd48313395fe05157518fd6371b532790e96fbb31bbb5a5e1"}, + {file = "cuda_bindings-13.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:debb51b211d246f8326f6b6e982506a5d0d9906672c91bc478b66addc7ecc60a"}, + {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788"}, + {file = "cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955"}, + {file = "cuda_bindings-13.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:45815daeb595bf3b405c52671a2542b1f8e9329f3b029494acbfcc74aeaa1f2d"}, + {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0"}, + {file = "cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d"}, + {file = "cuda_bindings-13.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8cebe3ce4aeeca5af9c490e175f76c4b569bbf4a35a62294b777bc77bf7ac4d8"}, + {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e"}, + {file = "cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626"}, + {file = "cuda_bindings-13.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd658bb5c0e55b7b3e5dd0ed509c6addb298c665db26a9bfba35e1e626000ba2"}, + {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771"}, + {file = "cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b"}, + {file = "cuda_bindings-13.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6ccf14e0c1def3b7200100aafff3a9f7e210ecb6e409329e92dcf6cd2c00d5c7"}, +] + +[package.dependencies] +cuda-pathfinder = ">=1.1,<2.0" + +[package.extras] +all = ["cuda-toolkit[cufile] (==13.*) ; sys_platform == \"linux\"", "cuda-toolkit[nvfatbin,nvjitlink,nvrtc,nvvm] (==13.*)"] + +[[package]] +name = "cuda-pathfinder" +version = "1.5.4" +description = "Pathfinder for CUDA components" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "cuda_pathfinder-1.5.4-py3-none-any.whl", hash = "sha256:9563d3175ce1828531acf4b94e1c1c7d67208c347ca002493e2654878b26f4b7"}, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +description = "CUDA Toolkit meta-package" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb"}, +] + +[package.dependencies] +nvidia-cuda-cupti = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cupti\""} +nvidia-cuda-nvrtc = {version = "==13.0.88.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"nvrtc\""} +nvidia-cuda-runtime = {version = "==13.0.96.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cudart\""} +nvidia-cufft = {version = "==12.0.0.61.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cufft\""} +nvidia-cufile = {version = "==1.15.1.6.*", optional = true, markers = "sys_platform == \"linux\" and extra == \"cufile\""} +nvidia-curand = {version = "==10.4.0.35.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"curand\""} +nvidia-cusolver = {version = "==12.0.4.66.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cusolver\""} +nvidia-cusparse = {version = "==12.6.3.3.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"cusparse\""} +nvidia-nvjitlink = {version = "==13.0.88.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"nvjitlink\""} +nvidia-nvtx = {version = "==13.0.85.*", optional = true, markers = "(sys_platform == \"linux\" or sys_platform == \"win32\") and extra == \"nvtx\""} + +[package.extras] +all = ["nvidia-cublas (==13.1.0.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\"", "nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\"", "nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvjitlink (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\"", "nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cccl = ["nvidia-cuda-cccl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +crt = ["nvidia-cuda-crt (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cublas = ["nvidia-cublas (==13.1.0.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cudart = ["nvidia-cuda-runtime (==13.0.96.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cufft = ["nvidia-cufft (==12.0.0.61.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cufile = ["nvidia-cufile (==1.15.1.6.*) ; sys_platform == \"linux\""] +culibos = ["nvidia-cuda-culibos (==13.0.85.*) ; sys_platform == \"linux\""] +cupti = ["nvidia-cuda-cupti (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +curand = ["nvidia-curand (==10.4.0.35.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cusolver = ["nvidia-cusolver (==12.0.4.66.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cusparse = ["nvidia-cusparse (==12.6.3.3.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +cuxxfilt = ["nvidia-cuda-cuxxfilt (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +npp = ["nvidia-npp (==13.0.1.2.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvcc = ["nvidia-cuda-nvcc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvfatbin = ["nvidia-nvfatbin (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvjitlink = ["nvidia-nvjitlink (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvjpeg = ["nvidia-nvjpeg (==13.0.1.86.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvml = ["nvidia-nvml-dev (==13.0.87.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvptxcompiler = ["nvidia-nvptxcompiler (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvrtc = ["nvidia-cuda-nvrtc (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvtx = ["nvidia-nvtx (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +nvvm = ["nvidia-nvvm (==13.0.88.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +opencl = ["nvidia-cuda-opencl (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +profiler = ["nvidia-cuda-profiler-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] +sanitizer = ["nvidia-cuda-sanitizer-api (==13.0.85.*) ; sys_platform == \"linux\" or sys_platform == \"win32\""] + [[package]] name = "entmax" version = "1.3" description = "The entmax mapping and its loss, a family of sparse alternatives to softmax." optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "entmax-1.3-py3-none-any.whl", hash = "sha256:41bb1edbd497a1b53b1f0fc80befd543499dc704b4e5436ddf6c5728a2d00546"}, ] @@ -380,12 +422,32 @@ files = [ [package.dependencies] torch = ">=1.3" +[[package]] +name = "exceptiongroup" +version = "1.3.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, + {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "filelock" version = "3.16.1" description = "A platform independent file lock." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0"}, {file = "filelock-3.16.1.tar.gz", hash = "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"}, @@ -394,7 +456,7 @@ files = [ [package.extras] docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4.1)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.2)", "pytest (>=8.3.3)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.4)"] -typing = ["typing-extensions (>=4.12.2)"] +typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""] [[package]] name = "frozenlist" @@ -402,6 +464,7 @@ version = "1.5.0" description = "A list-like structure which implements collections.abc.MutableSequence" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5b6a66c18b5b9dd261ca98dffcb826a525334b2f29e7caa54e182255c5f6a65a"}, {file = "frozenlist-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d1b3eb7b05ea246510b43a7e53ed1653e55c2121019a97e60cad7efb881a97bb"}, @@ -503,6 +566,7 @@ version = "2024.10.0" description = "File-system specification" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "fsspec-2024.10.0-py3-none-any.whl", hash = "sha256:03b9a6785766a4de40368b88906366755e2819e758b83705c88cd7cb5fe81871"}, {file = "fsspec-2024.10.0.tar.gz", hash = "sha256:eda2d8a4116d4f2429db8550f2457da57279247dd930bb12f821b58391359493"}, @@ -539,39 +603,139 @@ test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe, test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"] tqdm = ["tqdm"] +[[package]] +name = "h11" +version = "0.16.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, +] + +[[package]] +name = "hf-xet" +version = "1.5.0" +description = "Fast transfer of large files with the Hugging Face Hub." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\"" +files = [ + {file = "hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c"}, + {file = "hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42"}, + {file = "hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a"}, + {file = "hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480"}, + {file = "hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216"}, + {file = "hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60"}, + {file = "hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d"}, + {file = "hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4"}, + {file = "hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c"}, + {file = "hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73"}, + {file = "hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682"}, + {file = "hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761"}, + {file = "hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded"}, + {file = "hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702"}, + {file = "hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e"}, + {file = "hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0"}, + {file = "hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56"}, + {file = "hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a"}, + {file = "hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949"}, + {file = "hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b"}, + {file = "hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18"}, + {file = "hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690"}, + {file = "hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4"}, + {file = "hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be"}, + {file = "hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948"}, +] + +[package.extras] +tests = ["pytest"] + +[[package]] +name = "httpcore" +version = "1.0.9" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.16" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "huggingface-hub" -version = "0.26.3" +version = "1.14.0" description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub" optional = false -python-versions = ">=3.8.0" +python-versions = ">=3.10.0" +groups = ["main"] files = [ - {file = "huggingface_hub-0.26.3-py3-none-any.whl", hash = "sha256:e66aa99e569c2d5419240a9e553ad07245a5b1300350bfbc5a4945cf7432991b"}, - {file = "huggingface_hub-0.26.3.tar.gz", hash = "sha256:90e1fe62ffc26757a073aaad618422b899ccf9447c2bba8c902a90bef5b42e1d"}, + {file = "huggingface_hub-1.14.0-py3-none-any.whl", hash = "sha256:efe075535c62e130b30e836b138e13785f6f043d1f0539e0a39aa411a99e90b8"}, + {file = "huggingface_hub-1.14.0.tar.gz", hash = "sha256:d6d2c9cd6be1d02ae9ec6672d5587d10a427f377db688e82528f426a041622c2"}, ] [package.dependencies] -filelock = "*" +filelock = ">=3.10.0" fsspec = ">=2023.5.0" +hf-xet = {version = ">=1.4.3,<2.0.0", markers = "platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\""} +httpx = ">=0.23.0,<1" packaging = ">=20.9" pyyaml = ">=5.1" -requests = "*" tqdm = ">=4.42.1" -typing-extensions = ">=3.7.4.3" +typer = ">=0.20.0" +typing-extensions = ">=4.1.0" [package.extras] -all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] -cli = ["InquirerPy (==0.3.4)"] -dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "libcst (==1.4.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +all = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] +dev = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "libcst (>=1.4.0)", "mypy (==1.15.0)", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "ruff (>=0.9.0)", "soundfile", "ty", "types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"] fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"] -hf-transfer = ["hf-transfer (>=0.1.4)"] -inference = ["aiohttp"] -quality = ["libcst (==1.4.0)", "mypy (==1.5.1)", "ruff (>=0.5.0)"] -tensorflow = ["graphviz", "pydot", "tensorflow"] -tensorflow-testing = ["keras (<3.0)", "tensorflow"] -testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio (>=4.0.0)", "jedi", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] +gradio = ["gradio (>=5.0.0)", "requests"] +hf-xet = ["hf-xet (>=1.4.3,<2.0.0)"] +mcp = ["mcp (>=1.8.0)"] +oauth = ["authlib (>=1.3.2)", "fastapi", "httpx", "itsdangerous"] +quality = ["libcst (>=1.4.0)", "mypy (==1.15.0)", "ruff (>=0.9.0)", "ty"] +testing = ["Jinja2", "Pillow", "authlib (>=1.3.2)", "duckdb", "fastapi", "fastapi", "httpx", "itsdangerous", "jedi", "numpy", "pytest (>=8.4.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures (<16.0)", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"] torch = ["safetensors[torch]", "torch"] -typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] +typing = ["types-PyYAML", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] [[package]] name = "idna" @@ -579,6 +743,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -589,26 +754,27 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 [[package]] name = "importlib-metadata" -version = "8.5.0" +version = "9.0.0" description = "Read metadata from Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" +groups = ["main"] files = [ - {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, - {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, + {file = "importlib_metadata-9.0.0-py3-none-any.whl", hash = "sha256:2d21d1cc5a017bd0559e36150c21c830ab1dc304dedd1b7ea85d20f45ef3edd7"}, + {file = "importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc"}, ] [package.dependencies] zipp = ">=3.20" [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] +enabler = ["pytest-enabler (>=3.4)"] perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] -type = ["pytest-mypy"] +test = ["packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [[package]] name = "jinja2" @@ -616,6 +782,7 @@ version = "3.1.4" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, @@ -633,6 +800,7 @@ version = "1.4.2" description = "Lightweight pipelining with Python functions" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"}, {file = "joblib-1.4.2.tar.gz", hash = "sha256:2382c5816b2636fbd20a09e0f4e9dad4736765fdfb7dca582943b9c1366b3f0e"}, @@ -644,6 +812,7 @@ version = "3.13.1" description = "Parsing of command line options, yaml/jsonnet config files and/or environment variables based on argparse." optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "jsonargparse-3.13.1-py3-none-any.whl", hash = "sha256:b58188b98f2ac2b1f5007ece7ea821628883ce3f3ee448eb17c72d9d3b8ec893"}, {file = "jsonargparse-3.13.1.tar.gz", hash = "sha256:705693e9911223bf928fe4a7ed9538f24d9229c55ef5c3d3e9a8bad0e7982cf2"}, @@ -653,15 +822,15 @@ files = [ PyYAML = ">=3.13" [package.extras] -all = ["argcomplete (>=1.12.1)", "docstring-parser (>=0.7.3)", "fsspec (>=0.8.4)", "jsonnet (>=0.13.0)", "jsonschema (>=3.2.0)", "reconplogger (>=4.4.0)", "requests (>=2.18.4)", "ruyaml (>=0.20.0)", "validators (>=0.14.2)"] +all = ["argcomplete (>=1.12.1)", "docstring-parser (>=0.7.3)", "fsspec (>=0.8.4) ; python_version >= \"3.6\"", "jsonnet (>=0.13.0)", "jsonschema (>=3.2.0)", "reconplogger (>=4.4.0)", "requests (>=2.18.4)", "ruyaml (>=0.20.0) ; python_version >= \"3.6\"", "validators (>=0.14.2)"] argcomplete = ["argcomplete (>=1.12.1)"] dev = ["bump2version (>=0.5.11)", "coverage (>=4.5.1)", "mypy (>=0.701)", "pycodestyle (>=2.5.0)", "pylint (>=1.8.3)", "responses (>=0.12.0)", "twine (>=3.1.1)"] doc = ["Sphinx (>=1.7.9)", "autodocsumm (>=0.1.10)", "sphinx-autodoc-typehints (>=1.11.1)", "sphinx-rtd-theme (>=0.4.3)"] -fsspec = ["fsspec (>=0.8.4)"] +fsspec = ["fsspec (>=0.8.4) ; python_version >= \"3.6\""] jsonnet = ["jsonnet (>=0.13.0)"] jsonschema = ["jsonschema (>=3.2.0)"] reconplogger = ["reconplogger (>=4.4.0)"] -ruyaml = ["ruyaml (>=0.20.0)"] +ruyaml = ["ruyaml (>=0.20.0) ; python_version >= \"3.6\""] signatures = ["docstring-parser (>=0.7.3)"] test = ["coverage (>=4.5.1)", "responses (>=0.12.0)"] test-no-urls = ["coverage (>=4.5.1)"] @@ -673,6 +842,7 @@ version = "0.11.9" description = "Lightning toolbox for across the our ecosystem." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "lightning_utilities-0.11.9-py3-none-any.whl", hash = "sha256:ac6d4e9e28faf3ff4be997876750fee10dc604753dbc429bf3848a95c5d7e0d2"}, {file = "lightning_utilities-0.11.9.tar.gz", hash = "sha256:f5052b81344cc2684aa9afd74b7ce8819a8f49a858184ec04548a5a109dfd053"}, @@ -694,6 +864,7 @@ version = "5.3.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656"}, {file = "lxml-5.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d"}, @@ -848,24 +1019,47 @@ version = "3.7" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] -[package.dependencies] -importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} - [package.extras] docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] testing = ["coverage", "pyyaml"] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a"}, + {file = "markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "markdown-it-pyrs", "mistletoe (>=1.0,<2.0)", "mistune (>=3.0,<4.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins (>=0.5.0)"] +profiling = ["gprof2dot"] +rtd = ["ipykernel", "jupyter_sphinx", "mdit-py-plugins (>=0.5.0)", "myst-parser", "pyyaml", "sphinx", "sphinx-book-theme (>=1.0,<2.0)", "sphinx-copybutton", "sphinx-design"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions", "pytest-timeout", "requests"] + [[package]] name = "markupsafe" version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, @@ -929,12 +1123,25 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "mpmath" version = "1.3.0" description = "Python library for arbitrary-precision floating-point arithmetic" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, @@ -943,7 +1150,7 @@ files = [ [package.extras] develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4)"] +gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] tests = ["pytest (>=4.6)"] [[package]] @@ -952,6 +1159,7 @@ version = "6.1.0" description = "multidict implementation" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"}, {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"}, @@ -1056,6 +1264,7 @@ version = "3.1" description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "networkx-3.1-py3-none-any.whl", hash = "sha256:4f33f68cb2afcf86f28a45f43efc27a9386b535d567d2127f8f61d51dec58d36"}, {file = "networkx-3.1.tar.gz", hash = "sha256:de346335408f84de0eada6ff9fafafff9bcda11f0a0dfaa931133debb146ab61"}, @@ -1070,140 +1279,328 @@ test = ["codecov (>=2.1)", "pytest (>=7.2)", "pytest-cov (>=4.0)"] [[package]] name = "numpy" -version = "1.24.4" +version = "1.26.4" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main", "dev"] +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] + +[[package]] +name = "nvidia-cublas" +version = "13.1.1.3" +description = "CUBLAS native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" files = [ - {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, - {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, - {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, - {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, - {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, - {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, - {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, - {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, - {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, - {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, - {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, - {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, - {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, - {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, - {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, - {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, - {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, - {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, - {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, - {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, - {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, - {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, - {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, - {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, - {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, - {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, + {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5"}, + {file = "nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436"}, + {file = "nvidia_cublas-13.1.1.3-py3-none-win_amd64.whl", hash = "sha256:b6cdce694e47ff6aadf0a69df1cab6628d696f5ff56e8d16af50309d855fa20f"}, ] +[package.dependencies] +nvidia-cuda-nvrtc = "*" + [[package]] name = "nvidia-cublas-cu12" -version = "12.4.5.8" +version = "12.6.4.1" description = "CUBLAS native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0f8aa1706812e00b9f19dfe0cdb3999b092ccb8ca168c0db5b8ea712456fd9b3"}, - {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b"}, - {file = "nvidia_cublas_cu12-12.4.5.8-py3-none-win_amd64.whl", hash = "sha256:5a796786da89203a0657eda402bcdcec6180254a8ac22d72213abc42069522dc"}, + {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb"}, + {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:235f728d6e2a409eddf1df58d5b0921cf80cfa9e72b9f2775ccb7b4a87984668"}, + {file = "nvidia_cublas_cu12-12.6.4.1-py3-none-win_amd64.whl", hash = "sha256:9e4fa264f4d8a4eb0cdbd34beadc029f453b3bafae02401e999cf3d5a5af75f8"}, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +description = "CUDA profiling tools runtime libs." +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151"}, + {file = "nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8"}, + {file = "nvidia_cuda_cupti-13.0.85-py3-none-win_amd64.whl", hash = "sha256:683f58d301548deeefcb8f6fac1b8d907691b9d8b18eccab417f51e362102f00"}, ] [[package]] name = "nvidia-cuda-cupti-cu12" -version = "12.4.127" +version = "12.6.80" description = "CUDA profiling tools runtime libs." optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:166ee35a3ff1587f2490364f90eeeb8da06cd867bd5b701bf7f9a02b78bc63fc"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_aarch64.whl", hash = "sha256:358b4a1d35370353d52e12f0a7d1769fc01ff74a191689d3870b2123156184c4"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73"}, + {file = "nvidia_cuda_cupti_cu12-12.6.80-py3-none-win_amd64.whl", hash = "sha256:bbe6ae76e83ce5251b56e8c8e61a964f757175682bbad058b170b136266ab00a"}, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +description = "NVRTC native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" files = [ - {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:79279b35cf6f91da114182a5ce1864997fd52294a87a16179ce275773799458a"}, - {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb"}, - {file = "nvidia_cuda_cupti_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:5688d203301ab051449a2b1cb6690fbe90d2b372f411521c86018b950f3d7922"}, + {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575"}, + {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b"}, + {file = "nvidia_cuda_nvrtc-13.0.88-py3-none-win_amd64.whl", hash = "sha256:6bcd4e7f8e205cbe644f5a98f2f799bef9556fefc89dd786e79a16312ce49872"}, ] [[package]] name = "nvidia-cuda-nvrtc-cu12" -version = "12.4.127" +version = "12.6.77" description = "NVRTC native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5847f1d6e5b757f1d2b3991a01082a44aad6f10ab3c5c0213fa3e25bddc25a13"}, + {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53"}, + {file = "nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:f7007dbd914c56bd80ea31bc43e8e149da38f68158f423ba845fc3292684e45a"}, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +description = "CUDA Runtime native Libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" files = [ - {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0eedf14185e04b76aa05b1fea04133e59f465b6f960c0cbf4e37c3cb6b0ea198"}, - {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338"}, - {file = "nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:a961b2f1d5f17b14867c619ceb99ef6fcec12e46612711bcec78eb05068a60ec"}, + {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55"}, + {file = "nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548"}, + {file = "nvidia_cuda_runtime-13.0.96-py3-none-win_amd64.whl", hash = "sha256:f79298c8a098cec150a597c8eba58ecdab96e3bdc4b9bc4f9983635031740492"}, ] [[package]] name = "nvidia-cuda-runtime-cu12" -version = "12.4.127" +version = "12.6.77" description = "CUDA Runtime native Libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:961fe0e2e716a2a1d967aab7caee97512f71767f852f67432d572e36cb3a11f3"}, - {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5"}, - {file = "nvidia_cuda_runtime_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:09c2e35f48359752dfa822c09918211844a3d93c100a715d79b59591130c5e1e"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6116fad3e049e04791c0256a9778c16237837c08b27ed8c8401e2e45de8d60cd"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d461264ecb429c84c8879a7153499ddc7b19b5f8d84c204307491989a365588e"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba3b56a4f896141e25e19ab287cd71e52a6a0f4b29d0d31609f60e3b4d5219b7"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a84d15d5e1da416dd4774cb42edf5e954a3e60cc945698dc1d5be02321c44dc8"}, + {file = "nvidia_cuda_runtime_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:86c58044c824bf3c173c49a2dbc7a6c8b53cb4e4dca50068be0bf64e9dab3f7f"}, ] [[package]] name = "nvidia-cudnn-cu12" -version = "9.1.0.70" +version = "9.5.1.17" description = "cuDNN runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f"}, - {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-win_amd64.whl", hash = "sha256:6278562929433d68365a07a4a1546c237ba2849852c0d4b2262a486e805b977a"}, + {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9fd4584468533c61873e5fda8ca41bac3a38bcb2d12350830c69b0a96a7e4def"}, + {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2"}, + {file = "nvidia_cudnn_cu12-9.5.1.17-py3-none-win_amd64.whl", hash = "sha256:d7af0f8a4f3b4b9dbb3122f2ef553b45694ed9c384d5a75bab197b8eefb79ab8"}, ] [package.dependencies] nvidia-cublas-cu12 = "*" +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.20.0.48" +description = "cuDNN runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1"}, + {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304"}, + {file = "nvidia_cudnn_cu13-9.20.0.48-py3-none-win_amd64.whl", hash = "sha256:af8139732b99c0118be65ea5aac97f0d46018f8c552889e49d2fb0c6261a4a24"}, +] + +[package.dependencies] +nvidia-cublas = "*" + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +description = "CUFFT native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5"}, + {file = "nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3"}, + {file = "nvidia_cufft-12.0.0.61-py3-none-win_amd64.whl", hash = "sha256:2abce5b39d2f5ae12730fb7e5db6696533e36c26e2d3e8fd1750bdd2853364eb"}, +] + +[package.dependencies] +nvidia-nvjitlink = "*" + [[package]] name = "nvidia-cufft-cu12" -version = "11.2.1.3" +version = "11.3.0.4" description = "CUFFT native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:5dad8008fc7f92f5ddfa2101430917ce2ffacd86824914c82e28990ad7f00399"}, - {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9"}, - {file = "nvidia_cufft_cu12-11.2.1.3-py3-none-win_amd64.whl", hash = "sha256:d802f4954291101186078ccbe22fc285a902136f974d369540fd4a5333d1440b"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d16079550df460376455cba121db6564089176d9bac9e4f360493ca4741b22a6"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8510990de9f96c803a051822618d42bf6cb8f069ff3f48d93a8486efdacb48fb"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.whl", hash = "sha256:768160ac89f6f7b459bee747e8d175dbf53619cfe74b2a5636264163138013ca"}, + {file = "nvidia_cufft_cu12-11.3.0.4-py3-none-win_amd64.whl", hash = "sha256:6048ebddfb90d09d2707efb1fd78d4e3a77cb3ae4dc60e19aab6be0ece2ae464"}, ] [package.dependencies] nvidia-nvjitlink-cu12 = "*" +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +description = "cuFile GPUDirect libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "sys_platform == \"linux\" and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44"}, + {file = "nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1"}, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.11.1.6" +description = "cuFile GPUDirect libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159"}, + {file = "nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8f57a0051dcf2543f6dc2b98a98cb2719c37d3cee1baba8965d57f3bbc90d4db"}, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +description = "CURAND native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a"}, + {file = "nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc"}, + {file = "nvidia_curand-10.4.0.35-py3-none-win_amd64.whl", hash = "sha256:65b1710aa6961d326b411e314b374290904c5ddf41dc3f766ebc3f1d7d4ca69f"}, +] + [[package]] name = "nvidia-curand-cu12" -version = "10.3.5.147" +version = "10.3.7.77" description = "CURAND native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:6e82df077060ea28e37f48a3ec442a8f47690c7499bff392a5938614b56c98d8"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:99f1a32f1ac2bd134897fc7a203f779303261268a65762a623bf30cc9fe79117"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:7b2ed8e95595c3591d984ea3603dd66fe6ce6812b886d59049988a712ed06b6e"}, + {file = "nvidia_curand_cu12-10.3.7.77-py3-none-win_amd64.whl", hash = "sha256:6d6d935ffba0f3d439b7cd968192ff068fafd9018dbf1b85b37261b13cfc9905"}, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +description = "CUDA solver native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" files = [ - {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1f173f09e3e3c76ab084aba0de819c49e56614feae5c12f69883f4ae9bb5fad9"}, - {file = "nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b"}, - {file = "nvidia_curand_cu12-10.3.5.147-py3-none-win_amd64.whl", hash = "sha256:f307cc191f96efe9e8f05a87096abc20d08845a841889ef78cb06924437f6771"}, + {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2"}, + {file = "nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112"}, + {file = "nvidia_cusolver-12.0.4.66-py3-none-win_amd64.whl", hash = "sha256:16515bd33a8e76bb54d024cfa068fa68d30e80fc34b9e1090813ea9362e0cb65"}, ] +[package.dependencies] +nvidia-cublas = "*" +nvidia-cusparse = "*" +nvidia-nvjitlink = "*" + [[package]] name = "nvidia-cusolver-cu12" -version = "11.6.1.9" +version = "11.7.1.2" description = "CUDA solver native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_aarch64.whl", hash = "sha256:d338f155f174f90724bbde3758b7ac375a70ce8e706d70b018dd3375545fc84e"}, - {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260"}, - {file = "nvidia_cusolver_cu12-11.6.1.9-py3-none-win_amd64.whl", hash = "sha256:e77314c9d7b694fcebc84f58989f3aa4fb4cb442f12ca1a9bde50f5e8f6d1b9c"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:0ce237ef60acde1efc457335a2ddadfd7610b892d94efee7b776c64bb1cac9e0"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6cf28f17f64107a0c4d7802be5ff5537b2130bfc112f25d5a30df227058ca0e6"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dbbe4fc38ec1289c7e5230e16248365e375c3673c9c8bac5796e2e20db07f56e"}, + {file = "nvidia_cusolver_cu12-11.7.1.2-py3-none-win_amd64.whl", hash = "sha256:6813f9d8073f555444a8705f3ab0296d3e1cb37a16d694c5fc8b862a0d8706d7"}, ] [package.dependencies] @@ -1211,53 +1608,165 @@ nvidia-cublas-cu12 = "*" nvidia-cusparse-cu12 = "*" nvidia-nvjitlink-cu12 = "*" +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +description = "CUSPARSE native runtime libraries" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" +files = [ + {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c"}, + {file = "nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b"}, + {file = "nvidia_cusparse-12.6.3.3-py3-none-win_amd64.whl", hash = "sha256:cbcf42feb737bd7ec15b4c0a63e62351886bd3f975027b8815d7f720a2b5ea79"}, +] + +[package.dependencies] +nvidia-nvjitlink = "*" + [[package]] name = "nvidia-cusparse-cu12" -version = "12.3.1.170" +version = "12.5.4.2" description = "CUSPARSE native runtime libraries" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_aarch64.whl", hash = "sha256:9d32f62896231ebe0480efd8a7f702e143c98cfaa0e8a76df3386c1ba2b54df3"}, - {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1"}, - {file = "nvidia_cusparse_cu12-12.3.1.170-py3-none-win_amd64.whl", hash = "sha256:9bc90fb087bc7b4c15641521f31c0371e9a612fc2ba12c338d3ae032e6b6797f"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d25b62fb18751758fe3c93a4a08eff08effedfe4edf1c6bb5afd0890fe88f887"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7aa32fa5470cf754f72d1116c7cbc300b4e638d3ae5304cfa4a638a5b87161b1"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f"}, + {file = "nvidia_cusparse_cu12-12.5.4.2-py3-none-win_amd64.whl", hash = "sha256:4acb8c08855a26d737398cba8fb6f8f5045d93f82612b4cfd84645a2332ccf20"}, ] [package.dependencies] nvidia-nvjitlink-cu12 = "*" +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.3" +description = "NVIDIA cuSPARSELt" +optional = false +python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8371549623ba601a06322af2133c4a44350575f5a3108fb75f3ef20b822ad5f1"}, + {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46"}, + {file = "nvidia_cusparselt_cu12-0.6.3-py3-none-win_amd64.whl", hash = "sha256:3b325bcbd9b754ba43df5a311488fca11a6b5dc3d11df4d190c000cf1a0765c7"}, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.1" +description = "NVIDIA cuSPARSELt" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f"}, + {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0"}, + {file = "nvidia_cusparselt_cu13-0.8.1-py3-none-win_amd64.whl", hash = "sha256:dccbd362f91a7b9024d1f55ee9f548ac065027ff15d8c8b0db889ab3a8f31215"}, +] + [[package]] name = "nvidia-nccl-cu12" -version = "2.21.5" +version = "2.26.2" +description = "NVIDIA Collective Communication Library (NCCL) Runtime" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c196e95e832ad30fbbb50381eb3cbd1fadd5675e587a548563993609af19522"}, + {file = "nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6"}, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.29.7" description = "NVIDIA Collective Communication Library (NCCL) Runtime" optional = false python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5"}, + {file = "nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d"}, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.0.88" +description = "Nvidia JIT LTO Library" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" files = [ - {file = "nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0"}, + {file = "nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b"}, + {file = "nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c"}, + {file = "nvidia_nvjitlink-13.0.88-py3-none-win_amd64.whl", hash = "sha256:634e96e3da9ef845ae744097a1f289238ecf946ce0b82e93cdce14b9782e682f"}, ] [[package]] name = "nvidia-nvjitlink-cu12" -version = "12.4.127" +version = "12.6.85" description = "Nvidia JIT LTO Library" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" +files = [ + {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a"}, + {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf4eaa7d4b6b543ffd69d6abfb11efdeb2db48270d94dfd3a452c24150829e41"}, + {file = "nvidia_nvjitlink_cu12-12.6.85-py3-none-win_amd64.whl", hash = "sha256:e61120e52ed675747825cdd16febc6a0730537451d867ee58bee3853b1b13d1c"}, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +description = "NVSHMEM creates a global address space that provides efficient and scalable communication for NVIDIA GPU clusters." +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9"}, + {file = "nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80"}, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +description = "NVIDIA Tools Extension" +optional = false +python-versions = ">=3" +groups = ["main"] +markers = "(sys_platform == \"win32\" or sys_platform == \"linux\") and platform_system == \"Linux\" and python_version < \"3.14\"" files = [ - {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4abe7fef64914ccfa909bc2ba39739670ecc9e820c83ccc7a6ed414122599b83"}, - {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57"}, - {file = "nvidia_nvjitlink_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:fd9020c501d27d135f983c6d3e244b197a7ccad769e34df53a42e276b0e25fa1"}, + {file = "nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4"}, + {file = "nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6"}, + {file = "nvidia_nvtx-13.0.85-py3-none-win_amd64.whl", hash = "sha256:d66ea44254dd3c6eacc300047af6e1288d2269dd072b417e0adffbf479e18519"}, ] [[package]] name = "nvidia-nvtx-cu12" -version = "12.4.127" +version = "12.6.77" description = "NVIDIA Tools Extension" optional = false python-versions = ">=3" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7959ad635db13edf4fc65c06a6e9f9e55fc2f92596db928d169c0bb031e88ef3"}, - {file = "nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a"}, - {file = "nvidia_nvtx_cu12-12.4.127-py3-none-win_amd64.whl", hash = "sha256:641dccaaa1139f3ffb0d3164b4b84f9d253397e38246a4f2f36728b48566d485"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f44f8d86bb7d5629988d61c8d3ae61dddb2015dee142740536bc7481b022fe4b"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:adcaabb9d436c9761fca2b13959a2d237c5f9fd406c8e4b723c695409ff88059"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b90bed3df379fa79afbd21be8e04a0314336b8ae16768b58f2d34cb1d04cd7d2"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6574241a3ec5fdc9334353ab8c479fe75841dbe8f4532a8fc97ce63503330ba1"}, + {file = "nvidia_nvtx_cu12-12.6.77-py3-none-win_amd64.whl", hash = "sha256:2fb11a4af04a5e6c84073e6404d26588a34afd35379f0855a99797897efa75c0"}, ] [[package]] @@ -1266,6 +1775,7 @@ version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, @@ -1273,70 +1783,193 @@ files = [ [[package]] name = "pandas" -version = "2.0.3" +version = "2.3.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false -python-versions = ">=3.8" -files = [ - {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, - {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, - {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, - {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, - {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, - {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, - {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, - {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, - {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, - {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, - {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version < \"3.11\" or python_version >= \"3.14\"" +files = [ + {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, + {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, + {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, + {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, + {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, + {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, + {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, + {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, + {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, ] [package.dependencies] numpy = [ - {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" -tzdata = ">=2022.1" +tzdata = ">=2022.7" [package.extras] -all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] -aws = ["s3fs (>=2021.08.0)"] -clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] -compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] -computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2021.07.0)"] -gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] -hdf5 = ["tables (>=3.6.1)"] -html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] -mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] -spss = ["pyreadstat (>=1.1.2)"] -sql-other = ["SQLAlchemy (>=1.4.16)"] -test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.6.3)"] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + +[[package]] +name = "pandas" +version = "3.0.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.11" +groups = ["main"] +markers = "python_version < \"3.14\" and python_version >= \"3.11\"" +files = [ + {file = "pandas-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:455f6f8139d4282188f526868dbc3c828470e88a3d9d59a891bd46a455f21b98"}, + {file = "pandas-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4e15135e2ee5df1063313e2425ceef8ac0f4ae775893815b0923651b806a5639"}, + {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:05f1f1752b8533ea03f7f39a9c15b1a058d067bb48f4748948e7a8691e0510f2"}, + {file = "pandas-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a1e45c80cceb3b4a21bc5939d52e8cbd8d9b7305309219d59e9754d9ce09e27"}, + {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:14da8316da4d0c5a77618425996bfb1248ca87fc2c1486e6fde4652bd18b5824"}, + {file = "pandas-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a55066a0505dae0ba2b50a46637db34b46f9094c65c5d4800794ef6335010938"}, + {file = "pandas-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6674ab18ad8c57802867264b00e15e7bb904700cdd9046e3b2fa1fce237439ea"}, + {file = "pandas-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:5cc09a68b3120e0f54870dede8287a7bb1fa463907e4fcec1ea77cab6179bf7a"}, + {file = "pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09"}, + {file = "pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4"}, + {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c"}, + {file = "pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9"}, + {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf"}, + {file = "pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c"}, + {file = "pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc"}, + {file = "pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49"}, + {file = "pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa"}, + {file = "pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7"}, + {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8"}, + {file = "pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a"}, + {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb"}, + {file = "pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2"}, + {file = "pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44"}, + {file = "pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e"}, + {file = "pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d"}, + {file = "pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066"}, + {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd"}, + {file = "pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085"}, + {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870"}, + {file = "pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f"}, + {file = "pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13"}, + {file = "pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac"}, + {file = "pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f"}, + {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb"}, + {file = "pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a"}, + {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360"}, + {file = "pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76"}, + {file = "pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5"}, + {file = "pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977"}, + {file = "pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04"}, + {file = "pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6"}, + {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c"}, + {file = "pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028"}, + {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d"}, + {file = "pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a"}, + {file = "pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1"}, + {file = "pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1"}, + {file = "pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version < \"3.14\""} +python-dateutil = ">=2.8.2" +tzdata = {version = "*", markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\""} + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)", "beautifulsoup4 (>=4.12.3)", "bottleneck (>=1.4.2)", "fastparquet (>=2024.11.0)", "fsspec (>=2024.10.0)", "gcsfs (>=2024.10.0)", "html5lib (>=1.1)", "hypothesis (>=6.116.0)", "jinja2 (>=3.1.5)", "lxml (>=5.3.0)", "matplotlib (>=3.9.3)", "numba (>=0.60.0)", "numexpr (>=2.10.2)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "psycopg2 (>=2.9.10)", "pyarrow (>=13.0.0)", "pyiceberg (>=0.8.1)", "pymysql (>=1.1.1)", "pyreadstat (>=1.2.8)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)", "python-calamine (>=0.3.0)", "pytz (>=2020.1)", "pyxlsb (>=1.0.10)", "qtpy (>=2.4.2)", "s3fs (>=2024.10.0)", "scipy (>=1.14.1)", "tables (>=3.10.1)", "tabulate (>=0.9.0)", "xarray (>=2024.10.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)", "zstandard (>=0.23.0)"] +aws = ["s3fs (>=2024.10.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.4.2)"] +compression = ["zstandard (>=0.23.0)"] +computation = ["scipy (>=1.14.1)", "xarray (>=2024.10.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.5)", "python-calamine (>=0.3.0)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.2.0)"] +feather = ["pyarrow (>=13.0.0)"] +fss = ["fsspec (>=2024.10.0)"] +gcp = ["gcsfs (>=2024.10.0)"] +hdf5 = ["tables (>=3.10.1)"] +html = ["beautifulsoup4 (>=4.12.3)", "html5lib (>=1.1)", "lxml (>=5.3.0)"] +iceberg = ["pyiceberg (>=0.8.1)"] +mysql = ["SQLAlchemy (>=2.0.36)", "pymysql (>=1.1.1)"] +output-formatting = ["jinja2 (>=3.1.5)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=13.0.0)"] +performance = ["bottleneck (>=1.4.2)", "numba (>=0.60.0)", "numexpr (>=2.10.2)"] +plot = ["matplotlib (>=3.9.3)"] +postgresql = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "psycopg2 (>=2.9.10)"] +pyarrow = ["pyarrow (>=13.0.0)"] +spss = ["pyreadstat (>=1.2.8)"] +sql-other = ["SQLAlchemy (>=2.0.36)", "adbc-driver-postgresql (>=1.2.0)", "adbc-driver-sqlite (>=1.2.0)"] +test = ["hypothesis (>=6.116.0)", "pytest (>=8.3.4)", "pytest-xdist (>=3.6.1)"] +timezone = ["pytz (>=2020.1)"] +xml = ["lxml (>=5.3.0)"] [[package]] name = "portalocker" @@ -1344,6 +1977,7 @@ version = "3.0.0" description = "Wraps the portalocker recipe for easy usage" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "portalocker-3.0.0-py3-none-any.whl", hash = "sha256:211916b539a0dc3c128a3d9e86893ecfefec5379c4ff684e798f0a00f99db406"}, {file = "portalocker-3.0.0.tar.gz", hash = "sha256:21f535de2e7a82c94c130c054adb5c7421d480d5619d61073996e2f89bcb879b"}, @@ -1363,6 +1997,7 @@ version = "0.2.0" description = "Accelerated property cache" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c5869b8fd70b81835a6f187c5fdbe67917a04d7e52b6e7cc4e5fe39d55c39d58"}, {file = "propcache-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:952e0d9d07609d9c5be361f33b0d6d650cd2bae393aabb11d9b719364521984b"}, @@ -1470,6 +2105,7 @@ version = "4.25.5" description = "" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"}, {file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"}, @@ -1484,12 +2120,28 @@ files = [ {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"}, ] +[[package]] +name = "pygments" +version = "2.20.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, + {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -1504,6 +2156,7 @@ version = "2.4.0" description = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "pytorch-lightning-2.4.0.tar.gz", hash = "sha256:6aa897fd9d6dfa7b7b49f37c2f04e13592861831d08deae584dfda423fdb71c8"}, {file = "pytorch_lightning-2.4.0-py3-none-any.whl", hash = "sha256:9ac7935229ac022ef06994c928217ed37f525ac6700f7d4fc57009624570e655"}, @@ -1520,12 +2173,12 @@ tqdm = ">=4.57.0" typing-extensions = ">=4.4.0" [package.extras] -all = ["bitsandbytes (>=0.42.0)", "deepspeed (>=0.8.2,<=0.9.3)", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "requests (<2.32.0)", "rich (>=12.3.0)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)"] -deepspeed = ["deepspeed (>=0.8.2,<=0.9.3)"] -dev = ["bitsandbytes (>=0.42.0)", "cloudpickle (>=1.3)", "coverage (==7.3.1)", "deepspeed (>=0.8.2,<=0.9.3)", "fastapi", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "numpy (>=1.17.2)", "omegaconf (>=2.2.3)", "onnx (>=1.12.0)", "onnxruntime (>=1.12.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "requests (<2.32.0)", "rich (>=12.3.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)", "uvicorn"] +all = ["bitsandbytes (>=0.42.0)", "deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\"", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "requests (<2.32.0)", "rich (>=12.3.0)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)"] +deepspeed = ["deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\""] +dev = ["bitsandbytes (>=0.42.0)", "cloudpickle (>=1.3)", "coverage (==7.3.1)", "deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\"", "fastapi", "hydra-core (>=1.2.0)", "ipython[all] (<8.15.0)", "jsonargparse[signatures] (>=4.27.7)", "lightning-utilities (>=0.8.0)", "matplotlib (>3.1)", "numpy (>=1.17.2)", "omegaconf (>=2.2.3)", "onnx (>=1.12.0)", "onnxruntime (>=1.12.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "requests (<2.32.0)", "rich (>=12.3.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "tensorboardX (>=2.2)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)", "uvicorn"] examples = ["ipython[all] (<8.15.0)", "lightning-utilities (>=0.8.0)", "requests (<2.32.0)", "torchmetrics (>=0.10.0)", "torchvision (>=0.16.0)"] extra = ["bitsandbytes (>=0.42.0)", "hydra-core (>=1.2.0)", "jsonargparse[signatures] (>=4.27.7)", "matplotlib (>3.1)", "omegaconf (>=2.2.3)", "rich (>=12.3.0)", "tensorboardX (>=2.2)"] -strategies = ["deepspeed (>=0.8.2,<=0.9.3)"] +strategies = ["deepspeed (>=0.8.2,<=0.9.3) ; platform_system != \"Windows\" and platform_system != \"Darwin\""] test = ["cloudpickle (>=1.3)", "coverage (==7.3.1)", "fastapi", "numpy (>=1.17.2)", "onnx (>=1.12.0)", "onnxruntime (>=1.12.0)", "pandas (>1.0)", "psutil (<5.9.6)", "pytest (==7.4.0)", "pytest-cov (==4.1.0)", "pytest-random-order (==1.1.0)", "pytest-rerunfailures (==12.0)", "pytest-timeout (==2.1.0)", "scikit-learn (>0.22.1)", "tensorboard (>=2.9.1)", "uvicorn"] [[package]] @@ -1534,6 +2187,8 @@ version = "2024.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.11\" or python_version >= \"3.14\"" files = [ {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, @@ -1545,6 +2200,8 @@ version = "308" description = "Python for Window Extensions" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Windows\"" files = [ {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, @@ -1572,6 +2229,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1634,6 +2292,7 @@ version = "2024.11.6" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ff590880083d60acc0433f9c3f713c51f7ac6ebb9adf889c79a261ecf541aa91"}, {file = "regex-2024.11.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:658f90550f38270639e83ce492f27d2c8d2cd63805c65a13a14d36ca126753f0"}, @@ -1732,25 +2391,23 @@ files = [ ] [[package]] -name = "requests" -version = "2.32.3" -description = "Python HTTP for Humans." +name = "rich" +version = "15.0.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9.0" +groups = ["main"] files = [ - {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, - {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, + {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, + {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, ] [package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" [package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "sacrebleu" @@ -1758,6 +2415,7 @@ version = "2.4.3" description = "Hassle-free computation of shareable, comparable, and reproducible BLEU, chrF, and TER scores" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "sacrebleu-2.4.3-py3-none-any.whl", hash = "sha256:a976fd6998d8ced267a722120ec7fc47083c8e9745d8808ccee6424464a0aa31"}, {file = "sacrebleu-2.4.3.tar.gz", hash = "sha256:e734b1e0baeaea6ade0fefc9d23bac3df50bf15775d8b78edc108db63654192a"}, @@ -1782,6 +2440,7 @@ version = "0.4.5" description = "" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "safetensors-0.4.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a63eaccd22243c67e4f2b1c3e258b257effc4acd78f3b9d397edc8cf8f1298a7"}, {file = "safetensors-0.4.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:23fc9b4ec7b602915cbb4ec1a7c1ad96d2743c322f20ab709e2c35d1b66dad27"}, @@ -1914,6 +2573,7 @@ version = "1.3.2" description = "A set of python modules for machine learning and data mining" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "scikit-learn-1.3.2.tar.gz", hash = "sha256:a2f54c76accc15a34bfb9066e6c7a56c1e7235dda5762b990792330b52ccfb05"}, {file = "scikit_learn-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e326c0eb5cf4d6ba40f93776a20e9a7a69524c4db0757e7ce24ba222471ee8a1"}, @@ -1957,41 +2617,67 @@ tests = ["black (>=23.3.0)", "matplotlib (>=3.1.3)", "mypy (>=1.3)", "numpydoc ( [[package]] name = "scipy" -version = "1.9.3" +version = "1.15.3" description = "Fundamental algorithms for scientific computing in Python" optional = false -python-versions = ">=3.8" -files = [ - {file = "scipy-1.9.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1884b66a54887e21addf9c16fb588720a8309a57b2e258ae1c7986d4444d3bc0"}, - {file = "scipy-1.9.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:83b89e9586c62e787f5012e8475fbb12185bafb996a03257e9675cd73d3736dd"}, - {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a72d885fa44247f92743fc20732ae55564ff2a519e8302fb7e18717c5355a8b"}, - {file = "scipy-1.9.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d01e1dd7b15bd2449c8bfc6b7cc67d630700ed655654f0dfcf121600bad205c9"}, - {file = "scipy-1.9.3-cp310-cp310-win_amd64.whl", hash = "sha256:68239b6aa6f9c593da8be1509a05cb7f9efe98b80f43a5861cd24c7557e98523"}, - {file = "scipy-1.9.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b41bc822679ad1c9a5f023bc93f6d0543129ca0f37c1ce294dd9d386f0a21096"}, - {file = "scipy-1.9.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:90453d2b93ea82a9f434e4e1cba043e779ff67b92f7a0e85d05d286a3625df3c"}, - {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83c06e62a390a9167da60bedd4575a14c1f58ca9dfde59830fc42e5197283dab"}, - {file = "scipy-1.9.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abaf921531b5aeaafced90157db505e10345e45038c39e5d9b6c7922d68085cb"}, - {file = "scipy-1.9.3-cp311-cp311-win_amd64.whl", hash = "sha256:06d2e1b4c491dc7d8eacea139a1b0b295f74e1a1a0f704c375028f8320d16e31"}, - {file = "scipy-1.9.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a04cd7d0d3eff6ea4719371cbc44df31411862b9646db617c99718ff68d4840"}, - {file = "scipy-1.9.3-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:545c83ffb518094d8c9d83cce216c0c32f8c04aaf28b92cc8283eda0685162d5"}, - {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d54222d7a3ba6022fdf5773931b5d7c56efe41ede7f7128c7b1637700409108"}, - {file = "scipy-1.9.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cff3a5295234037e39500d35316a4c5794739433528310e117b8a9a0c76d20fc"}, - {file = "scipy-1.9.3-cp38-cp38-win_amd64.whl", hash = "sha256:2318bef588acc7a574f5bfdff9c172d0b1bf2c8143d9582e05f878e580a3781e"}, - {file = "scipy-1.9.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d644a64e174c16cb4b2e41dfea6af722053e83d066da7343f333a54dae9bc31c"}, - {file = "scipy-1.9.3-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:da8245491d73ed0a994ed9c2e380fd058ce2fa8a18da204681f2fe1f57f98f95"}, - {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db5b30849606a95dcf519763dd3ab6fe9bd91df49eba517359e450a7d80ce2e"}, - {file = "scipy-1.9.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c68db6b290cbd4049012990d7fe71a2abd9ffbe82c0056ebe0f01df8be5436b0"}, - {file = "scipy-1.9.3-cp39-cp39-win_amd64.whl", hash = "sha256:5b88e6d91ad9d59478fafe92a7c757d00c59e3bdc3331be8ada76a4f8d683f58"}, - {file = "scipy-1.9.3.tar.gz", hash = "sha256:fbc5c05c85c1a02be77b1ff591087c83bc44579c6d2bd9fb798bb64ea5e1a027"}, +python-versions = ">=3.10" +groups = ["main", "dev"] +files = [ + {file = "scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92"}, + {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82"}, + {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40"}, + {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e"}, + {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c"}, + {file = "scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1"}, + {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889"}, + {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982"}, + {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9"}, + {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594"}, + {file = "scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c"}, + {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45"}, + {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49"}, + {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e"}, + {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539"}, + {file = "scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730"}, + {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825"}, + {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7"}, + {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11"}, + {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126"}, + {file = "scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb"}, + {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723"}, + {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb"}, + {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4"}, + {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5"}, + {file = "scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca"}, + {file = "scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf"}, ] [package.dependencies] -numpy = ">=1.18.5,<1.26.0" +numpy = ">=1.23.5,<2.5" [package.extras] -dev = ["flake8", "mypy", "pycodestyle", "typing_extensions"] -doc = ["matplotlib (>2)", "numpydoc", "pydata-sphinx-theme (==0.9.0)", "sphinx (!=4.1.0)", "sphinx-panels (>=0.5.2)", "sphinx-tabs"] -test = ["asv", "gmpy2", "mpmath", "pytest", "pytest-cov", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] +doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "sentencepiece" @@ -1999,6 +2685,7 @@ version = "0.2.0" description = "SentencePiece python wrapper" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:188779e1298a1c8b8253c7d3ad729cb0a9891e5cef5e5d07ce4592c54869e227"}, {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bed9cf85b296fa2b76fc2547b9cbb691a523864cebaee86304c43a7b4cb1b452"}, @@ -2061,19 +2748,32 @@ version = "75.6.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "setuptools-75.6.0-py3-none-any.whl", hash = "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d"}, {file = "setuptools-75.6.0.tar.gz", hash = "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.7.0)"] -core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.7.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (>=1.12,<1.14)", "pytest-mypy"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] [[package]] name = "six" @@ -2081,6 +2781,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -2092,6 +2793,7 @@ version = "0.0.15" description = "A Sphinx extension for rendering tables written in markdown" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "sphinx-markdown-tables-0.0.15.tar.gz", hash = "sha256:722b70a3a14156b6777cf04920d015215eb65b9dfde5cc72f430763ff5e84173"}, {file = "sphinx_markdown_tables-0.0.15-py3-none-any.whl", hash = "sha256:24a37662d86ee8bceb7d4f7003df0b25bc52401369d1ddc40d13ae7b58697031"}, @@ -2102,27 +2804,14 @@ markdown = ">=3.0.1" [[package]] name = "sympy" -version = "1.12.1" -description = "Computer algebra system (CAS) in Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sympy-1.12.1-py3-none-any.whl", hash = "sha256:9b2cbc7f1a640289430e13d2a56f02f867a1da0190f2f99d8968c2f74da0e515"}, - {file = "sympy-1.12.1.tar.gz", hash = "sha256:2877b03f998cd8c08f07cd0de5b767119cd3ef40d09f41c30d722f6686b0fb88"}, -] - -[package.dependencies] -mpmath = ">=1.1.0,<1.4.0" - -[[package]] -name = "sympy" -version = "1.13.1" +version = "1.14.0" description = "Computer algebra system (CAS) in Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8"}, - {file = "sympy-1.13.1.tar.gz", hash = "sha256:9cebf7e04ff162015ce31c9c6c9144daa34a93bd082f54fd8f12deca4f47515f"}, + {file = "sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5"}, + {file = "sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517"}, ] [package.dependencies] @@ -2137,6 +2826,7 @@ version = "0.9.0" description = "Pretty-print tabular data" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, @@ -2151,6 +2841,7 @@ version = "3.5.0" description = "threadpoolctl" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"}, {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"}, @@ -2158,157 +2849,79 @@ files = [ [[package]] name = "tokenizers" -version = "0.20.3" +version = "0.22.2" description = "" optional = false -python-versions = ">=3.7" -files = [ - {file = "tokenizers-0.20.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:31ccab28dbb1a9fe539787210b0026e22debeab1662970f61c2d921f7557f7e4"}, - {file = "tokenizers-0.20.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6361191f762bda98c773da418cf511cbaa0cb8d0a1196f16f8c0119bde68ff8"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f128d5da1202b78fa0a10d8d938610472487da01b57098d48f7e944384362514"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79c4121a2e9433ad7ef0769b9ca1f7dd7fa4c0cd501763d0a030afcbc6384481"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b7850fde24197fe5cd6556e2fdba53a6d3bae67c531ea33a3d7c420b90904141"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b357970c095dc134978a68c67d845a1e3803ab7c4fbb39195bde914e7e13cf8b"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a333d878c4970b72d6c07848b90c05f6b045cf9273fc2bc04a27211721ad6118"}, - {file = "tokenizers-0.20.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd9fee817f655a8f50049f685e224828abfadd436b8ff67979fc1d054b435f1"}, - {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9e7816808b402129393a435ea2a509679b41246175d6e5e9f25b8692bfaa272b"}, - {file = "tokenizers-0.20.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba96367db9d8a730d3a1d5996b4b7babb846c3994b8ef14008cd8660f55db59d"}, - {file = "tokenizers-0.20.3-cp310-none-win32.whl", hash = "sha256:ee31ba9d7df6a98619426283e80c6359f167e2e9882d9ce1b0254937dbd32f3f"}, - {file = "tokenizers-0.20.3-cp310-none-win_amd64.whl", hash = "sha256:a845c08fdad554fe0871d1255df85772f91236e5fd6b9287ef8b64f5807dbd0c"}, - {file = "tokenizers-0.20.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:585b51e06ca1f4839ce7759941e66766d7b060dccfdc57c4ca1e5b9a33013a90"}, - {file = "tokenizers-0.20.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61cbf11954f3b481d08723ebd048ba4b11e582986f9be74d2c3bdd9293a4538d"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef820880d5e4e8484e2fa54ff8d297bb32519eaa7815694dc835ace9130a3eea"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:67ef4dcb8841a4988cd00dd288fb95dfc8e22ed021f01f37348fd51c2b055ba9"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff1ef8bd47a02b0dc191688ccb4da53600df5d4c9a05a4b68e1e3de4823e78eb"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:444d188186eab3148baf0615b522461b41b1f0cd58cd57b862ec94b6ac9780f1"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37c04c032c1442740b2c2d925f1857885c07619224a533123ac7ea71ca5713da"}, - {file = "tokenizers-0.20.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:453c7769d22231960ee0e883d1005c93c68015025a5e4ae56275406d94a3c907"}, - {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4bb31f7b2847e439766aaa9cc7bccf7ac7088052deccdb2275c952d96f691c6a"}, - {file = "tokenizers-0.20.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:843729bf0f991b29655a069a2ff58a4c24375a553c70955e15e37a90dd4e045c"}, - {file = "tokenizers-0.20.3-cp311-none-win32.whl", hash = "sha256:efcce3a927b1e20ca694ba13f7a68c59b0bd859ef71e441db68ee42cf20c2442"}, - {file = "tokenizers-0.20.3-cp311-none-win_amd64.whl", hash = "sha256:88301aa0801f225725b6df5dea3d77c80365ff2362ca7e252583f2b4809c4cc0"}, - {file = "tokenizers-0.20.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:49d12a32e190fad0e79e5bdb788d05da2f20d8e006b13a70859ac47fecf6ab2f"}, - {file = "tokenizers-0.20.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:282848cacfb9c06d5e51489f38ec5aa0b3cd1e247a023061945f71f41d949d73"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abe4e08c7d0cd6154c795deb5bf81d2122f36daf075e0c12a8b050d824ef0a64"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ca94fc1b73b3883c98f0c88c77700b13d55b49f1071dfd57df2b06f3ff7afd64"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef279c7e239f95c8bdd6ff319d9870f30f0d24915b04895f55b1adcf96d6c60d"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16384073973f6ccbde9852157a4fdfe632bb65208139c9d0c0bd0176a71fd67f"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:312d522caeb8a1a42ebdec87118d99b22667782b67898a76c963c058a7e41d4f"}, - {file = "tokenizers-0.20.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2b7cb962564785a83dafbba0144ecb7f579f1d57d8c406cdaa7f32fe32f18ad"}, - {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:124c5882ebb88dadae1fc788a582299fcd3a8bd84fc3e260b9918cf28b8751f5"}, - {file = "tokenizers-0.20.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2b6e54e71f84c4202111a489879005cb14b92616a87417f6c102c833af961ea2"}, - {file = "tokenizers-0.20.3-cp312-none-win32.whl", hash = "sha256:83d9bfbe9af86f2d9df4833c22e94d94750f1d0cd9bfb22a7bb90a86f61cdb1c"}, - {file = "tokenizers-0.20.3-cp312-none-win_amd64.whl", hash = "sha256:44def74cee574d609a36e17c8914311d1b5dbcfe37c55fd29369d42591b91cf2"}, - {file = "tokenizers-0.20.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e0b630e0b536ef0e3c8b42c685c1bc93bd19e98c0f1543db52911f8ede42cf84"}, - {file = "tokenizers-0.20.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a02d160d2b19bcbfdf28bd9a4bf11be4cb97d0499c000d95d4c4b1a4312740b6"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e3d80d89b068bc30034034b5319218c7c0a91b00af19679833f55f3becb6945"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:174a54910bed1b089226512b4458ea60d6d6fd93060254734d3bc3540953c51c"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:098b8a632b8656aa5802c46689462c5c48f02510f24029d71c208ec2c822e771"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78c8c143e3ae41e718588281eb3e212c2b31623c9d6d40410ec464d7d6221fb5"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b26b0aadb18cd8701077362ba359a06683662d5cafe3e8e8aba10eb05c037f1"}, - {file = "tokenizers-0.20.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07d7851a72717321022f3774e84aa9d595a041d643fafa2e87fbc9b18711dac0"}, - {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:bd44e48a430ada902c6266a8245f5036c4fe744fcb51f699999fbe82aa438797"}, - {file = "tokenizers-0.20.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a4c186bb006ccbe1f5cc4e0380d1ce7806f5955c244074fd96abc55e27b77f01"}, - {file = "tokenizers-0.20.3-cp313-none-win32.whl", hash = "sha256:6e19e0f1d854d6ab7ea0c743d06e764d1d9a546932be0a67f33087645f00fe13"}, - {file = "tokenizers-0.20.3-cp313-none-win_amd64.whl", hash = "sha256:d50ede425c7e60966a9680d41b58b3a0950afa1bb570488e2972fa61662c4273"}, - {file = "tokenizers-0.20.3-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:9adda1ff5fb9dcdf899ceca672a4e2ce9e797adb512a6467305ca3d8bfcfbdd0"}, - {file = "tokenizers-0.20.3-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:6dde2cae6004ba7a3badff4a11911cae03ebf23e97eebfc0e71fef2530e5074f"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4a7fd678b35614fca708579eb95b7587a5e8a6d328171bd2488fd9f27d82be4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1b80e3c7283a01a356bd2210f53d1a4a5d32b269c2024389ed0173137708d50e"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8cc0e8176b762973758a77f0d9c4467d310e33165fb74173418ca3734944da4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5634b2e2f5f3d2b4439d2d74066e22eb4b1f04f3fea05cb2a3c12d89b5a3bcd"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4ba635165bc1ea46f2da8e5d80b5f70f6ec42161e38d96dbef33bb39df73964"}, - {file = "tokenizers-0.20.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e4c7c64172e7789bd8b07aa3087ea87c4c4de7e90937a2aa036b5d92332536"}, - {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1f74909ef7675c26d4095a817ec3393d67f3158ca4836c233212e5613ef640c4"}, - {file = "tokenizers-0.20.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0e9b81321a1e05b16487d312b4264984513f8b4a7556229cafac6e88c2036b09"}, - {file = "tokenizers-0.20.3-cp37-none-win32.whl", hash = "sha256:ab48184cd58b4a03022a2ec75b54c9f600ffea9a733612c02325ed636f353729"}, - {file = "tokenizers-0.20.3-cp37-none-win_amd64.whl", hash = "sha256:60ac483cebee1c12c71878523e768df02fa17e4c54412966cb3ac862c91b36c1"}, - {file = "tokenizers-0.20.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3229ef103c89583d10b9378afa5d601b91e6337530a0988e17ca8d635329a996"}, - {file = "tokenizers-0.20.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6ac52cc24bad3de865c7e65b1c4e7b70d00938a8ae09a92a453b8f676e714ad5"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04627b7b502fa6a2a005e1bd446fa4247d89abcb1afaa1b81eb90e21aba9a60f"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c27ceb887f0e81a3c377eb4605dca7a95a81262761c0fba308d627b2abb98f2b"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65ab780194da4e1fcf5670523a2f377c4838ebf5249efe41fa1eddd2a84fb49d"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98d343134f47159e81f7f242264b0eb222e6b802f37173c8d7d7b64d5c9d1388"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2475bb004ab2009d29aff13b5047bfdb3d4b474f0aa9d4faa13a7f34dbbbb43"}, - {file = "tokenizers-0.20.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b6583a65c01db1197c1eb36857ceba8ec329d53afadd268b42a6b04f4965724"}, - {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:62d00ba208358c037eeab7bfc00a905adc67b2d31b68ab40ed09d75881e114ea"}, - {file = "tokenizers-0.20.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0fc7a39e5bedc817bda395a798dfe2d9c5f7c71153c90d381b5135a0328d9520"}, - {file = "tokenizers-0.20.3-cp38-none-win32.whl", hash = "sha256:84d40ee0f8550d64d3ea92dd7d24a8557a9172165bdb986c9fb2503b4fe4e3b6"}, - {file = "tokenizers-0.20.3-cp38-none-win_amd64.whl", hash = "sha256:205a45246ed7f1718cf3785cff88450ba603352412aaf220ace026384aa3f1c0"}, - {file = "tokenizers-0.20.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:93e37f0269a11dc3b1a953f1fca9707f0929ebf8b4063c591c71a0664219988e"}, - {file = "tokenizers-0.20.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f4cb0c614b0135e781de96c2af87e73da0389ac1458e2a97562ed26e29490d8d"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7eb2fb1c432f5746b22f8a7f09fc18c4156cb0031c77f53cb19379d82d43297a"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bfa8d029bb156181b006643309d6b673615a24e4ed24cf03aa191d599b996f51"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f90549622de3bf476ad9f1dd6f3f952ec3ed6ab8615ae88ef060d0c5bfad55d"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1d469c74eebf5c43fd61cd9b030e271d17198edd7bd45392e03a3c091d7d6d4"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bee8f53b2594749f4460d53253bae55d718f04e9b633efa0f5df8938bd98e4f0"}, - {file = "tokenizers-0.20.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:938441babf3e5720e4459e306ef2809fb267680df9d1ff2873458b22aef60248"}, - {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7310ab23d7b0caebecc0e8be11a1146f320f5f07284000f6ea54793e83de1b75"}, - {file = "tokenizers-0.20.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:16121eb030a2b13094cfec936b0c12e8b4063c5f839591ea7d0212336d8f9921"}, - {file = "tokenizers-0.20.3-cp39-none-win32.whl", hash = "sha256:401cc21ef642ee235985d747f65e18f639464d377c70836c9003df208d582064"}, - {file = "tokenizers-0.20.3-cp39-none-win_amd64.whl", hash = "sha256:7498f3ea7746133335a6adb67a77cf77227a8b82c8483f644a2e5f86fea42b8d"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e919f2e3e68bb51dc31de4fcbbeff3bdf9c1cad489044c75e2b982a91059bd3c"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b8e9608f2773996cc272156e305bd79066163a66b0390fe21750aff62df1ac07"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:39270a7050deaf50f7caff4c532c01b3c48f6608d42b3eacdebdc6795478c8df"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e005466632b1c5d2d2120f6de8aa768cc9d36cd1ab7d51d0c27a114c91a1e6ee"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a07962340b36189b6c8feda552ea1bfeee6cf067ff922a1d7760662c2ee229e5"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:55046ad3dd5f2b3c67501fcc8c9cbe3e901d8355f08a3b745e9b57894855f85b"}, - {file = "tokenizers-0.20.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:efcf0eb939988b627558aaf2b9dc3e56d759cad2e0cfa04fcab378e4b48fc4fd"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f3558a7ae6a6d38a77dfce12172a1e2e1bf3e8871e744a1861cd7591ea9ebe24"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d53029fe44bc70c3ff14ef512460a0cf583495a0f8e2f4b70e26eb9438e38a9"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57a2a56397b2bec5a629b516b23f0f8a3e4f978c7488d4a299980f8375954b85"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e5bfaae740ef9ece000f8a07e78ac0e2b085c5ce9648f8593ddf0243c9f76d"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fbaf3ea28fedfb2283da60e710aff25492e795a7397cad8a50f1e079b65a5a70"}, - {file = "tokenizers-0.20.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c47c037116310dc976eb96b008e41b9cfaba002ed8005848d4d632ee0b7ba9ae"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c31751f0721f58f5e19bb27c1acc259aeff860d8629c4e1a900b26a1979ada8e"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:c697cbd3be7a79ea250ea5f380d6f12e534c543cfb137d5c734966b3ee4f34cc"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b48971b88ef9130bf35b41b35fd857c3c4dae4a9cd7990ebc7fc03e59cc92438"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e615de179bbe060ab33773f0d98a8a8572b5883dd7dac66c1de8c056c7e748c"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da1ec842035ed9999c62e45fbe0ff14b7e8a7e02bb97688cc6313cf65e5cd755"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6ee4954c1dd23aadc27958dad759006e71659d497dcb0ef0c7c87ea992c16ebd"}, - {file = "tokenizers-0.20.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:3eda46ca402751ec82553a321bf35a617b76bbed7586e768c02ccacbdda94d6d"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:de082392a85eb0055cc055c535bff2f0cc15d7a000bdc36fbf601a0f3cf8507a"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c3db46cc0647bfd88263afdb739b92017a02a87ee30945cb3e86c7e25c7c9917"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a292392f24ab9abac5cfa8197e5a6208f2e43723420217e1ceba0b4ec77816ac"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8dcd91f4e60f62b20d83a87a84fe062035a1e3ff49a8c2bbdeb2d441c8e311f4"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900991a2b8ee35961b1095db7e265342e0e42a84c1a594823d5ee9f8fb791958"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5a8d8261ca2133d4f98aa9627c748189502b3787537ba3d7e2beb4f7cfc5d627"}, - {file = "tokenizers-0.20.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:c4fd4d71e6deb6ddf99d8d0eab87d1d16f635898906e631914a9bae8ae9f2cfb"}, - {file = "tokenizers-0.20.3.tar.gz", hash = "sha256:2278b34c5d0dd78e087e1ca7f9b1dcbf129d80211afa645f214bd6e051037539"}, +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c"}, + {file = "tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b"}, + {file = "tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a"}, + {file = "tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5"}, + {file = "tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92"}, + {file = "tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48"}, + {file = "tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:753d47ebd4542742ef9261d9da92cd545b2cacbb48349a1225466745bb866ec4"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e10bf9113d209be7cd046d40fbabbaf3278ff6d18eb4da4c500443185dc1896c"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64d94e84f6660764e64e7e0b22baa72f6cd942279fdbb21d46abd70d179f0195"}, + {file = "tokenizers-0.22.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f01a9c019878532f98927d2bacb79bbb404b43d3437455522a00a30718cdedb5"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:319f659ee992222f04e58f84cbf407cfa66a65fe3a8de44e8ad2bc53e7d99012"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1e50f8554d504f617d9e9d6e4c2c2884a12b388a97c5c77f0bc6cf4cd032feee"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a62ba2c5faa2dd175aaeed7b15abf18d20266189fb3406c5d0550dd34dd5f37"}, + {file = "tokenizers-0.22.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:143b999bdc46d10febb15cbffb4207ddd1f410e2c755857b5a0797961bbdc113"}, + {file = "tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917"}, ] [package.dependencies] -huggingface-hub = ">=0.16.4,<1.0" +huggingface-hub = ">=0.16.4,<2.0" [package.extras] dev = ["tokenizers[testing]"] docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] -testing = ["black (==22.3)", "datasets", "numpy", "pytest", "requests", "ruff"] +testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] [[package]] name = "torch" -version = "2.5.1" +version = "2.7.1" description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" optional = false -python-versions = ">=3.8.0" -files = [ - {file = "torch-2.5.1-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:71328e1bbe39d213b8721678f9dcac30dfc452a46d586f1d514a6aa0a99d4744"}, - {file = "torch-2.5.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:34bfa1a852e5714cbfa17f27c49d8ce35e1b7af5608c4bc6e81392c352dbc601"}, - {file = "torch-2.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:32a037bd98a241df6c93e4c789b683335da76a2ac142c0973675b715102dc5fa"}, - {file = "torch-2.5.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:23d062bf70776a3d04dbe74db950db2a5245e1ba4f27208a87f0d743b0d06e86"}, - {file = "torch-2.5.1-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:de5b7d6740c4b636ef4db92be922f0edc425b65ed78c5076c43c42d362a45457"}, - {file = "torch-2.5.1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:340ce0432cad0d37f5a31be666896e16788f1adf8ad7be481196b503dad675b9"}, - {file = "torch-2.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:603c52d2fe06433c18b747d25f5c333f9c1d58615620578c326d66f258686f9a"}, - {file = "torch-2.5.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:31f8c39660962f9ae4eeec995e3049b5492eb7360dd4f07377658ef4d728fa4c"}, - {file = "torch-2.5.1-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:ed231a4b3a5952177fafb661213d690a72caaad97d5824dd4fc17ab9e15cec03"}, - {file = "torch-2.5.1-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:3f4b7f10a247e0dcd7ea97dc2d3bfbfc90302ed36d7f3952b0008d0df264e697"}, - {file = "torch-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:73e58e78f7d220917c5dbfad1a40e09df9929d3b95d25e57d9f8558f84c9a11c"}, - {file = "torch-2.5.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:8c712df61101964eb11910a846514011f0b6f5920c55dbf567bff8a34163d5b1"}, - {file = "torch-2.5.1-cp313-cp313-manylinux1_x86_64.whl", hash = "sha256:9b61edf3b4f6e3b0e0adda8b3960266b9009d02b37555971f4d1c8f7a05afed7"}, - {file = "torch-2.5.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1f3b7fb3cf7ab97fae52161423f81be8c6b8afac8d9760823fd623994581e1a3"}, - {file = "torch-2.5.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:7974e3dce28b5a21fb554b73e1bc9072c25dde873fa00d54280861e7a009d7dc"}, - {file = "torch-2.5.1-cp39-cp39-win_amd64.whl", hash = "sha256:46c817d3ea33696ad3b9df5e774dba2257e9a4cd3c4a3afbf92f6bb13ac5ce2d"}, - {file = "torch-2.5.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:8046768b7f6d35b85d101b4b38cba8aa2f3cd51952bc4c06a49580f2ce682291"}, +python-versions = ">=3.9.0" +groups = ["main"] +markers = "python_version >= \"3.14\"" +files = [ + {file = "torch-2.7.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a103b5d782af5bd119b81dbcc7ffc6fa09904c423ff8db397a1e6ea8fd71508f"}, + {file = "torch-2.7.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:fe955951bdf32d182ee8ead6c3186ad54781492bf03d547d31771a01b3d6fb7d"}, + {file = "torch-2.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:885453d6fba67d9991132143bf7fa06b79b24352f4506fd4d10b309f53454162"}, + {file = "torch-2.7.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:d72acfdb86cee2a32c0ce0101606f3758f0d8bb5f8f31e7920dc2809e963aa7c"}, + {file = "torch-2.7.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:236f501f2e383f1cb861337bdf057712182f910f10aeaf509065d54d339e49b2"}, + {file = "torch-2.7.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:06eea61f859436622e78dd0cdd51dbc8f8c6d76917a9cf0555a333f9eac31ec1"}, + {file = "torch-2.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:8273145a2e0a3c6f9fd2ac36762d6ee89c26d430e612b95a99885df083b04e52"}, + {file = "torch-2.7.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:aea4fc1bf433d12843eb2c6b2204861f43d8364597697074c8d38ae2507f8730"}, + {file = "torch-2.7.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:27ea1e518df4c9de73af7e8a720770f3628e7f667280bce2be7a16292697e3fa"}, + {file = "torch-2.7.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c33360cfc2edd976c2633b3b66c769bdcbbf0e0b6550606d188431c81e7dd1fc"}, + {file = "torch-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:d8bf6e1856ddd1807e79dc57e54d3335f2b62e6f316ed13ed3ecfe1fc1df3d8b"}, + {file = "torch-2.7.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:787687087412c4bd68d315e39bc1223f08aae1d16a9e9771d95eabbb04ae98fb"}, + {file = "torch-2.7.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:03563603d931e70722dce0e11999d53aa80a375a3d78e6b39b9f6805ea0a8d28"}, + {file = "torch-2.7.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d632f5417b6980f61404a125b999ca6ebd0b8b4bbdbb5fbbba44374ab619a412"}, + {file = "torch-2.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:23660443e13995ee93e3d844786701ea4ca69f337027b05182f5ba053ce43b38"}, + {file = "torch-2.7.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:0da4f4dba9f65d0d203794e619fe7ca3247a55ffdcbd17ae8fb83c8b2dc9b585"}, + {file = "torch-2.7.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e08d7e6f21a617fe38eeb46dd2213ded43f27c072e9165dc27300c9ef9570934"}, + {file = "torch-2.7.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:30207f672328a42df4f2174b8f426f354b2baa0b7cca3a0adb3d6ab5daf00dc8"}, + {file = "torch-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:79042feca1c634aaf6603fe6feea8c6b30dfa140a6bbc0b973e2260c7e79a22e"}, + {file = "torch-2.7.1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:988b0cbc4333618a1056d2ebad9eb10089637b659eb645434d0809d8d937b946"}, + {file = "torch-2.7.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:e0d81e9a12764b6f3879a866607c8ae93113cbcad57ce01ebde63eb48a576369"}, + {file = "torch-2.7.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:8394833c44484547ed4a47162318337b88c97acdb3273d85ea06e03ffff44998"}, + {file = "torch-2.7.1-cp39-cp39-win_amd64.whl", hash = "sha256:df41989d9300e6e3c19ec9f56f856187a6ef060c3662fe54f4b6baf1fc90bd19"}, + {file = "torch-2.7.1-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:a737b5edd1c44a5c1ece2e9f3d00df9d1b3fb9541138bee56d83d38293fb6c9d"}, ] [package.dependencies] @@ -2316,29 +2929,89 @@ filelock = "*" fsspec = "*" jinja2 = "*" networkx = "*" -nvidia-cublas-cu12 = {version = "12.4.5.8", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-cupti-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-nvrtc-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cuda-runtime-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cudnn-cu12 = {version = "9.1.0.70", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cufft-cu12 = {version = "11.2.1.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-curand-cu12 = {version = "10.3.5.147", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusolver-cu12 = {version = "11.6.1.9", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-cusparse-cu12 = {version = "12.3.1.170", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nccl-cu12 = {version = "2.21.5", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nvjitlink-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} -nvidia-nvtx-cu12 = {version = "12.4.127", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cublas-cu12 = {version = "12.6.4.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-cupti-cu12 = {version = "12.6.80", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-nvrtc-cu12 = {version = "12.6.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cuda-runtime-cu12 = {version = "12.6.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cudnn-cu12 = {version = "9.5.1.17", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufft-cu12 = {version = "11.3.0.4", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cufile-cu12 = {version = "1.11.1.6", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-curand-cu12 = {version = "10.3.7.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusolver-cu12 = {version = "11.7.1.2", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparse-cu12 = {version = "12.5.4.2", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-cusparselt-cu12 = {version = "0.6.3", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nccl-cu12 = {version = "2.26.2", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvjitlink-cu12 = {version = "12.6.85", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +nvidia-nvtx-cu12 = {version = "12.6.77", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} setuptools = {version = "*", markers = "python_version >= \"3.12\""} -sympy = [ - {version = "1.12.1", markers = "python_version == \"3.8\""}, - {version = "1.13.1", markers = "python_version >= \"3.9\""}, +sympy = ">=1.13.3" +triton = {version = "3.3.1", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\""} +typing-extensions = ">=4.10.0" + +[package.extras] +opt-einsum = ["opt-einsum (>=3.3)"] +optree = ["optree (>=0.13.0)"] + +[[package]] +name = "torch" +version = "2.12.0" +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version < \"3.14\"" +files = [ + {file = "torch-2.12.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:1834bd984f8a2f4f16bdfbeecca9146184b220aa46276bf5756735b5dae12812"}, + {file = "torch-2.12.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:d4d029801cb7b6df858804a2a21b00cc2aa0bf0ee5d2ab18d343c9e9e5681f35"}, + {file = "torch-2.12.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:d47e7dee68ac4cd7a068b26bcd6b989935427709fae1c8f7bd0019978f829e15"}, + {file = "torch-2.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:cf9839790285dd472e7a16aafcb4a4e6bf58ec1b494045044b0eefb0eb4bd1f2"}, + {file = "torch-2.12.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:10802fd383bbfed646212e765a72c37d2185205d4f26eb197a254e8ac7ddcb25"}, + {file = "torch-2.12.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:c12592630aef72feaf18bd3f197ef587bbfa21131b31c38b23ab2e55fce92e36"}, + {file = "torch-2.12.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:415c1b8d0412f67551c8e89a2daca0fb3e56694af0281ba155eaa9da481f58b4"}, + {file = "torch-2.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:dd37188ea325042cb1f6cafa56822b11ada2520c04791a52629b0af25bdfbfd9"}, + {file = "torch-2.12.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b41339df93d491435e790ff8bcbae1c0ce777175889bfd1281d119862793e6a2"}, + {file = "torch-2.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8fbef9f108a863e7722a73740998967e3b074742a834fc5be3a535a2befa7057"}, + {file = "torch-2.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4b4f64c2c2b11f7510d93dd6412b87025ff6eddd6bb61c3b5a3d892ea20c4756"}, + {file = "torch-2.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:8b958caff4a14d3a3b0b2dfc6a378f64dda9728a9dad28c08a0db9ce4dafb549"}, + {file = "torch-2.12.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:90dd587a5f61bfe1307148b581e2084fc5bc4a06e2b90a20e9a36b81087ff16b"}, + {file = "torch-2.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:864392c73b7654f4d2b3ae712f607937d0dbb1101c4555fbb41848106b297f39"}, + {file = "torch-2.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5d6b560dfa7d56291c07d615c3bb73e8d9943d9b6d87f76cd0d9d570c4797fa6"}, + {file = "torch-2.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:3fee918902090ade827643e758e98363278815de583c75d111fdd665ebffde9f"}, + {file = "torch-2.12.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:10ee1448a9f304d3b987eb4656f664ba6e4d7b410ca7a5a7c642199777a2cf88"}, + {file = "torch-2.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:af68dbf403439cae9ceaeaaf92f8352b460787dcd27b92aa05c40dd4a19c0f1e"}, + {file = "torch-2.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:a6a2eebb237d3b1d9ad3b378e86d9b9e0782afdea8b1e0eba6a13646b9b49c07"}, + {file = "torch-2.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:2140e373e9a51a3e22ef62e8d14366d0b470d18f0adf19fdc757368077133a34"}, + {file = "torch-2.12.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:f7dfae4a519197dfa050e98d8e36378a0fb5899625a875c2b54445005a2e404e"}, + {file = "torch-2.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:891c769072637c74e9a5a77a3bc782894696d8ffec83b938df8536dee7f0ba78"}, + {file = "torch-2.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e2ad3eb85d39c3cab62dfa93ed5a73516e6a53c6713cb97d004004fe089f0f1f"}, + {file = "torch-2.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:c66696857e987efb8bc1777a37357ec4f60ab5e8af6250b83d6034437fa2d8f3"}, + {file = "torch-2.12.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:b4556715c8572758625d62b6e0ae3b1f76c440221913a6fb5e100f321fb4fb02"}, + {file = "torch-2.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a43ac605a5e13116c72b64c359644cce0229f213dde48d2ae0ae5eb5becf7feb"}, + {file = "torch-2.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a7512adfdd7f6732e40de1c620831e3c75b39b98cef60b11d0c5f0a76473ec5"}, + {file = "torch-2.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f96b63f8287f66a005dd1b5a6abba2920f11156c5e5c4d815f3e2050fd1aa16"}, ] -triton = {version = "3.1.0", markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version < \"3.13\""} -typing-extensions = ">=4.8.0" + +[package.dependencies] +cuda-bindings = {version = ">=13.0.3,<14", markers = "platform_system == \"Linux\""} +cuda-toolkit = {version = "13.0.2", extras = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], markers = "platform_system == \"Linux\""} +filelock = "*" +fsspec = ">=0.8.5" +jinja2 = "*" +networkx = ">=2.5.1" +nvidia-cublas = {version = ">=13.1.0.3,<=13.1.1.3", markers = "platform_system == \"Linux\""} +nvidia-cudnn-cu13 = {version = "9.20.0.48", markers = "platform_system == \"Linux\""} +nvidia-cusparselt-cu13 = {version = "0.8.1", markers = "platform_system == \"Linux\""} +nvidia-nccl-cu13 = {version = "2.29.7", markers = "platform_system == \"Linux\""} +nvidia-nvshmem-cu13 = {version = "3.4.5", markers = "platform_system == \"Linux\""} +setuptools = "<82" +sympy = ">=1.13.3" +triton = {version = "3.7.0", markers = "platform_system == \"Linux\""} +typing-extensions = ">=4.10.0" [package.extras] opt-einsum = ["opt-einsum (>=3.3)"] -optree = ["optree (>=0.12.0)"] +optree = ["optree (>=0.13.0)"] +pyyaml = ["pyyaml"] [[package]] name = "torchmetrics" @@ -2346,6 +3019,7 @@ version = "0.10.3" description = "PyTorch native Metrics" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "torchmetrics-0.10.3-py3-none-any.whl", hash = "sha256:b12cf92897545e24a825b0d168888c0f3052700c2901e2d4f7d90b252bc4a343"}, {file = "torchmetrics-0.10.3.tar.gz", hash = "sha256:9e6ab66175f2dc13e246c37485b2c27c77931dfe47fc2b81c76217b8efdc1e57"}, @@ -2355,7 +3029,6 @@ files = [ numpy = ">=1.17.2" packaging = "*" torch = ">=1.3.1" -typing-extensions = {version = "*", markers = "python_version < \"3.9\""} [package.extras] all = ["lpips", "nltk (>=3.6)", "pycocotools", "pystoi", "pytorch-lightning (>=1.5)", "regex (>=2021.9.24)", "scipy", "torch-fidelity", "torchvision", "torchvision (>=0.8)", "tqdm (>=4.41.0)"] @@ -2373,6 +3046,7 @@ version = "4.67.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2"}, {file = "tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2"}, @@ -2390,101 +3064,138 @@ telegram = ["requests"] [[package]] name = "transformers" -version = "4.46.3" -description = "State-of-the-art Machine Learning for JAX, PyTorch and TensorFlow" +version = "5.3.0" +description = "Transformers: the model-definition framework for state-of-the-art machine learning models in text, vision, audio, and multimodal models, for both inference and training." optional = false -python-versions = ">=3.8.0" +python-versions = ">=3.10.0" +groups = ["main"] files = [ - {file = "transformers-4.46.3-py3-none-any.whl", hash = "sha256:a12ef6f52841fd190a3e5602145b542d03507222f2c64ebb7ee92e8788093aef"}, - {file = "transformers-4.46.3.tar.gz", hash = "sha256:8ee4b3ae943fe33e82afff8e837f4b052058b07ca9be3cb5b729ed31295f72cc"}, + {file = "transformers-5.3.0-py3-none-any.whl", hash = "sha256:50ac8c89c3c7033444fb3f9f53138096b997ebb70d4b5e50a2e810bf12d3d29a"}, + {file = "transformers-5.3.0.tar.gz", hash = "sha256:009555b364029da9e2946d41f1c5de9f15e6b1df46b189b7293f33a161b9c557"}, ] [package.dependencies] -filelock = "*" -huggingface-hub = ">=0.23.2,<1.0" +huggingface-hub = ">=1.3.0,<2.0" numpy = ">=1.17" packaging = ">=20.0" pyyaml = ">=5.1" regex = "!=2019.12.17" -requests = "*" -safetensors = ">=0.4.1" -tokenizers = ">=0.20,<0.21" +safetensors = ">=0.4.3" +tokenizers = ">=0.22.0,<=0.23.0" tqdm = ">=4.27" +typer = "*" [package.extras] -accelerate = ["accelerate (>=0.26.0)"] -agents = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "datasets (!=2.5.0)", "diffusers", "opencv-python", "sentencepiece (>=0.1.91,!=0.1.92)", "torch"] -all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av (==9.2.0)", "codecarbon (==1.2.0)", "flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "librosa", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "ray[tune] (>=2.7.0)", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision"] -audio = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +accelerate = ["accelerate (>=1.1.0)"] +all = ["Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "av", "blobfile", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.10.2,<0.11)", "librosa", "mistral-common[image] (>=1.8.8)", "num2words", "phonemizer", "protobuf", "pyctcdecode (>=0.4.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "tiktoken", "timm (>=1.0.23)", "torch (>=2.4)", "torchaudio", "torchvision"] +audio = ["librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] benchmark = ["optimum-benchmark (>=0.3.0)"] -codecarbon = ["codecarbon (==1.2.0)"] -deepspeed = ["accelerate (>=0.26.0)", "deepspeed (>=0.9.3)"] -deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=0.26.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk (<=3.8.1)", "optuna", "parameterized", "protobuf", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "av (==9.2.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "flax (>=0.4.1,<=0.7.0)", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "librosa", "nltk (<=3.8.1)", "onnxconverter-common", "optax (>=0.0.8,<=0.1.4)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "scipy (<1.13.0)", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -dev-tensorflow = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "isort (>=5.5.4)", "kenlm", "keras-nlp (>=0.3.1,<0.14.0)", "libcst", "librosa", "nltk (<=3.8.1)", "onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx", "timeout-decorator", "tokenizers (>=0.20,<0.21)", "urllib3 (<2.0.0)"] -dev-torch = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=0.26.0)", "beautifulsoup4", "codecarbon (==1.2.0)", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "isort (>=5.5.4)", "kenlm", "libcst", "librosa", "nltk (<=3.8.1)", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "optuna", "parameterized", "phonemizer", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "ray[tune] (>=2.7.0)", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sigopt", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "timeout-decorator", "timm (<=0.9.16)", "tokenizers (>=0.20,<0.21)", "torch", "torchaudio", "torchvision", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)", "urllib3 (<2.0.0)"] -flax = ["flax (>=0.4.1,<=0.7.0)", "jax (>=0.4.1,<=0.4.13)", "jaxlib (>=0.4.1,<=0.4.13)", "optax (>=0.0.8,<=0.1.4)", "scipy (<1.13.0)"] -flax-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] -ftfy = ["ftfy"] -integrations = ["optuna", "ray[tune] (>=2.7.0)", "sigopt"] -ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict-core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic-lite (>=1.0.7)"] -modelcreation = ["cookiecutter (==1.7.3)"] -natten = ["natten (>=0.14.6,<0.15.0)"] -onnx = ["onnxconverter-common", "onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)", "tf2onnx"] -onnxruntime = ["onnxruntime (>=1.4.0)", "onnxruntime-tools (>=1.4.2)"] +chat-template = ["jinja2 (>=3.1.0)", "jmespath (>=1.0.1)"] +codecarbon = ["codecarbon (>=2.8.1)"] +deepspeed = ["accelerate (>=1.1.0)", "deepspeed (>=0.9.3)"] +deepspeed-testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "deepspeed (>=0.9.3)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "libcst", "mistral-common[image] (>=1.8.8)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "optuna", "parameterized (>=0.9)", "protobuf", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "torch (>=2.4)", "ty (==0.0.12)", "urllib3 (<2.0.0)", "uvicorn"] +dev = ["GitPython (<3.1.19)", "Pillow (>=10.0.1,<=15.0)", "accelerate (>=1.1.0)", "accelerate (>=1.1.0)", "av", "beautifulsoup4", "blobfile", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "jinja2 (>=3.1.0)", "jmespath (>=1.0.1)", "kernels (>=0.10.2,<0.11)", "libcst", "librosa", "mistral-common[image] (>=1.8.8)", "mistral-common[image] (>=1.8.8)", "nltk (<=3.8.1)", "num2words", "openai (>=1.98.0)", "parameterized (>=0.9)", "phonemizer", "protobuf", "protobuf", "psutil", "pyctcdecode (>=0.4.0)", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rhoknp (>=1.1.0,<1.3.1)", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "scikit-learn", "sentencepiece (>=0.1.91,!=0.1.92)", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "tensorboard", "tiktoken", "timeout-decorator", "timm (>=1.0.23)", "torch (>=2.4)", "torch (>=2.4)", "torchaudio", "torchvision", "ty (==0.0.12)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)", "urllib3 (<2.0.0)", "uvicorn"] +integrations = ["codecarbon (>=2.8.1)", "kernels (>=0.10.2,<0.11)", "optuna", "ray[tune] (>=2.7.0)"] +ja = ["fugashi (>=1.0)", "ipadic (>=1.0.0,<2.0)", "rhoknp (>=1.1.0,<1.3.1)", "sudachidict_core (>=20220729)", "sudachipy (>=0.6.6)", "unidic (>=1.0.2)", "unidic_lite (>=1.0.7)"] +kernels = ["kernels (>=0.10.2,<0.11)"] +mistral-common = ["mistral-common[image] (>=1.8.8)"] +num2words = ["num2words"] +open-telemetry = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-sdk"] optuna = ["optuna"] -quality = ["GitPython (<3.1.19)", "datasets (!=2.5.0)", "isort (>=5.5.4)", "libcst", "rich", "ruff (==0.5.1)", "urllib3 (<2.0.0)"] +quality = ["GitPython (<3.1.19)", "datasets (>=2.15.0)", "libcst", "rich", "ruff (==0.14.10)", "ty (==0.0.12)", "urllib3 (<2.0.0)"] ray = ["ray[tune] (>=2.7.0)"] -retrieval = ["datasets (!=2.5.0)", "faiss-cpu"] -ruff = ["ruff (==0.5.1)"] +retrieval = ["datasets (>=2.15.0)", "faiss-cpu"] sagemaker = ["sagemaker (>=2.31.0)"] sentencepiece = ["protobuf", "sentencepiece (>=0.1.91,!=0.1.92)"] -serving = ["fastapi", "pydantic", "starlette", "uvicorn"] -sigopt = ["sigopt"] +serving = ["accelerate (>=1.1.0)", "fastapi", "openai (>=1.98.0)", "pydantic (>=2)", "rich", "starlette", "torch (>=2.4)", "uvicorn"] sklearn = ["scikit-learn"] -speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -testing = ["GitPython (<3.1.19)", "beautifulsoup4", "cookiecutter (==1.7.3)", "datasets (!=2.5.0)", "dill (<0.3.5)", "evaluate (>=0.2.0)", "faiss-cpu", "nltk (<=3.8.1)", "parameterized", "psutil", "pydantic", "pytest (>=7.2.0,<8.0.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.5.1)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "timeout-decorator"] -tf = ["keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow (>2.9,<2.16)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-cpu = ["keras (>2.9,<2.16)", "keras-nlp (>=0.3.1,<0.14.0)", "onnxconverter-common", "tensorflow-cpu (>2.9,<2.16)", "tensorflow-probability (<0.24)", "tensorflow-text (<2.16)", "tf2onnx"] -tf-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)"] +testing = ["GitPython (<3.1.19)", "accelerate (>=1.1.0)", "beautifulsoup4", "datasets (>=2.15.0)", "datasets (>=2.15.0)", "dill (<0.3.5)", "evaluate (>=0.4.6)", "faiss-cpu", "fastapi", "filelock", "libcst", "mistral-common[image] (>=1.8.8)", "nltk (<=3.8.1)", "openai (>=1.98.0)", "parameterized (>=0.9)", "protobuf", "psutil", "pydantic (>=2)", "pytest (>=7.2.0,<9.0.0)", "pytest-asyncio (>=1.2.0)", "pytest-env", "pytest-order", "pytest-random-order", "pytest-rerunfailures (<16.0)", "pytest-rich", "pytest-timeout", "pytest-xdist", "rich", "rich", "rjieba", "rouge-score (!=0.0.7,!=0.0.8,!=0.1,!=0.1.1)", "ruff (==0.14.10)", "sacrebleu (>=1.4.12,<2.0.0)", "sacremoses", "sentencepiece (>=0.1.91,!=0.1.92)", "starlette", "tensorboard", "timeout-decorator", "torch (>=2.4)", "ty (==0.0.12)", "urllib3 (<2.0.0)", "uvicorn"] tiktoken = ["blobfile", "tiktoken"] -timm = ["timm (<=0.9.16)"] -tokenizers = ["tokenizers (>=0.20,<0.21)"] -torch = ["accelerate (>=0.26.0)", "torch"] -torch-speech = ["kenlm", "librosa", "phonemizer", "pyctcdecode (>=0.4.0)", "torchaudio"] -torch-vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] -torchhub = ["filelock", "huggingface-hub (>=0.23.2,<1.0)", "importlib-metadata", "numpy (>=1.17)", "packaging (>=20.0)", "protobuf", "regex (!=2019.12.17)", "requests", "sentencepiece (>=0.1.91,!=0.1.92)", "tokenizers (>=0.20,<0.21)", "torch", "tqdm (>=4.27)"] -video = ["av (==9.2.0)"] -vision = ["Pillow (>=10.0.1,<=15.0)"] +timm = ["timm (>=1.0.23)"] +torch = ["accelerate (>=1.1.0)", "torch (>=2.4)"] +video = ["av"] +vision = ["Pillow (>=10.0.1,<=15.0)", "torchvision"] [[package]] name = "triton" -version = "3.1.0" +version = "3.3.1" description = "A language and compiler for custom Deep Learning operations" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and platform_machine == \"x86_64\" and python_version >= \"3.14\"" files = [ - {file = "triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0dd10a925263abbe9fa37dcde67a5e9b2383fc269fdf59f5657cac38c5d1d8"}, - {file = "triton-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f34f6e7885d1bf0eaaf7ba875a5f0ce6f3c13ba98f9503651c1e6dc6757ed5c"}, - {file = "triton-3.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8182f42fd8080a7d39d666814fa36c5e30cc00ea7eeeb1a2983dbb4c99a0fdc"}, - {file = "triton-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6dadaca7fc24de34e180271b5cf864c16755702e9f63a16f62df714a8099126a"}, - {file = "triton-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aafa9a20cd0d9fee523cd4504aa7131807a864cd77dcf6efe7e981f18b8c6c11"}, + {file = "triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e"}, + {file = "triton-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b"}, + {file = "triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43"}, + {file = "triton-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b89d846b5a4198317fec27a5d3a609ea96b6d557ff44b56c23176546023c4240"}, + {file = "triton-3.3.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3198adb9d78b77818a5388bff89fa72ff36f9da0bc689db2f0a651a67ce6a42"}, + {file = "triton-3.3.1-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6139aeb04a146b0b8e0fbbd89ad1e65861c57cfed881f21d62d3cb94a36bab7"}, ] [package.dependencies] -filelock = "*" +setuptools = ">=40.8.0" [package.extras] build = ["cmake (>=3.20)", "lit"] -tests = ["autopep8", "flake8", "isort", "llnl-hatchet", "numpy", "pytest", "scipy (>=1.7.1)"] +tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] tutorials = ["matplotlib", "pandas", "tabulate"] +[[package]] +name = "triton" +version = "3.7.0" +description = "A language and compiler for custom Deep Learning operations" +optional = false +python-versions = "<3.15,>=3.10" +groups = ["main"] +markers = "python_version < \"3.14\" and platform_system == \"Linux\"" +files = [ + {file = "triton-3.7.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223ac302091491436c248a34ee1e6c47a1026486579103c906ffd805be50cb89"}, + {file = "triton-3.7.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c631b65668d4951213b948a413c0564184305b77bb45cc9d686d3e1ecc4701a3"}, + {file = "triton-3.7.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9e71fc392675fac364e0ecf4ef3f76f85b7f5433a16f4c3c5fe5f05a52c85fe"}, + {file = "triton-3.7.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22bacffce443f54593dd20f05294d5a40622e0ea9ab632816f87154504356221"}, + {file = "triton-3.7.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a4bf49b00a7a377a68a6da603a876e797614e6455a80e9021669c476a953ad9a"}, + {file = "triton-3.7.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f111161d49bf903c0eaedde3962353a3d841c08a836839b7cc1025b8426efcf"}, + {file = "triton-3.7.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abdf6beaa89b1bcfb9a43cd990536ce66091a997841a4814b260b7bee4c88c3c"}, + {file = "triton-3.7.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a35d7afe3f3f058e7ec49fcce09794049e0ffc5c59019ac25ec3413741b8c4e7"}, + {file = "triton-3.7.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc1d61c172d257db80ddf42595131fb196ad2e9bdd751e90fe2ef13531734e8b"}, + {file = "triton-3.7.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:70fb9bbdc9f400afc54bbf6eb2670af28829a6ae3996863317964783141daf56"}, + {file = "triton-3.7.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a44a8476d0d3571eac4e4d1048e1ff75aad81a09ff4602ccfc56c6dea1672e"}, + {file = "triton-3.7.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b85e72968a9d8bba5ddb24e9b64aaabaf48affb042f2755cb7cfa92b7531ce"}, + {file = "triton-3.7.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18a160de426fd99f92b0baf509045360afbd3bfaa0b4a5171dde800ec9f09684"}, + {file = "triton-3.7.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce061073102714b725f3660ec6939d94a1da7984b3aa99c921417cae273672f5"}, +] + +[package.extras] +build = ["cmake (>=3.20,<4.0)", "lit"] +tests = ["autopep8", "isort", "llnl-hatchet", "numpy", "pytest", "pytest-forked", "pytest-xdist", "scipy (>=1.7.1)"] +tutorials = ["matplotlib", "pandas", "tabulate"] + +[[package]] +name = "typer" +version = "0.25.1" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89"}, + {file = "typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc"}, +] + +[package.dependencies] +annotated-doc = ">=0.0.2" +click = ">=8.2.1" +rich = ">=13.8.0" +shellingham = ">=1.3.0" + [[package]] name = "typing-extensions" version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -2496,34 +3207,20 @@ version = "2024.2" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["main"] +markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version < \"3.11\" or python_version >= \"3.14\"" files = [ {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] -[[package]] -name = "urllib3" -version = "2.2.3" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.8" -files = [ - {file = "urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac"}, - {file = "urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - [[package]] name = "yarl" version = "1.15.2" description = "Yet another URL library" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "yarl-1.15.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e4ee8b8639070ff246ad3649294336b06db37a94bdea0d09ea491603e0be73b8"}, {file = "yarl-1.15.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a7cf963a357c5f00cb55b1955df8bbe68d2f2f65de065160a1c26b85a1e44172"}, @@ -2632,24 +3329,25 @@ propcache = ">=0.2.0" [[package]] name = "zipp" -version = "3.20.2" +version = "4.1.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false -python-versions = ">=3.8" +python-versions = ">=3.10" +groups = ["main"] files = [ - {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, - {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, + {file = "zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f"}, + {file = "zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] -type = ["pytest-mypy"] +enabler = ["pytest-enabler (>=3.4)"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""] [metadata] -lock-version = "2.0" -python-versions = "^3.8.0" -content-hash = "2aff3e907a2e72ce04fdcd5c61192068ee9131d861ca0079cacf2bf7e02b4921" +lock-version = "2.1" +python-versions = "^3.10.0" +content-hash = "7a53c261eb178883327062bc9b093a6acf1c40329f6290498f33ce9414ba36b2" diff --git a/pyproject.toml b/pyproject.toml index 214a48c6..d43723e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,24 +37,25 @@ comet-mbr = 'comet.cli.mbr:mbr_command' [tool.poetry.dependencies] python = "^3.8.0" -sentencepiece = "^0.2.0" +sentencepiece = ">=0.2.0" pandas = ">=1.4.1" -transformers = "^4.17" -pytorch-lightning = "^2.0.0" -jsonargparse = "3.13.1" +transformers = ">=4.17" +pytorch-lightning = ">=2.0.0" +jsonargparse = ">= 3.13" torch = ">=1.6.0" -numpy = "^1.20.0" -torchmetrics = "^0.10.2" -sacrebleu = "^2.0.0" -scipy = "^1.5.4" -entmax = "^1.1" -huggingface-hub = ">=0.19.3,<1.0" -protobuf = "^4.24.4" +numpy = ">=1.20.0" +torchmetrics = ">=0.10.2" +sacrebleu = ">=2.0.0" +scipy = ">=1.5.4" +entmax = ">=1.1" +huggingface-hub = ">=0.19.3" +importlib-metadata = "^9.0.0" [tool.poetry.dev-dependencies] -sphinx-markdown-tables = "0.0.15" -coverage = "^5.5" -scikit-learn = "^1.0" +sphinx-markdown-tables = ">=0.0.15" +coverage = ">=5.5" +scikit-learn = ">=1.0" +protobuf = ">=4.24.4" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/integration/models/test_ranking_metric.py b/tests/integration/models/test_ranking_metric.py index 903e61d9..16ba9b39 100644 --- a/tests/integration/models/test_ranking_metric.py +++ b/tests/integration/models/test_ranking_metric.py @@ -5,27 +5,28 @@ import warnings import torch -from comet.models import RankingMetric from pytorch_lightning import seed_everything from pytorch_lightning.trainer.trainer import Trainer from scipy.stats import pearsonr -from tests.data import DATA_PATH from torch.utils.data import DataLoader -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +from comet.models import RankingMetric +from tests.data import DATA_PATH + +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestRankingMetric(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_training(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=22, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -35,68 +36,68 @@ def test_training(self): model = RankingMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-04, learning_rate=1e-04, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - pool="cls", - layer="mix", - layer_transformation="softmax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + pool='cls', + layer='mix', + layer_transformation='softmax', layer_norm=True, dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "ranking_data.csv")], - validation_data=[os.path.join(DATA_PATH, "ranking_data.csv")], + train_data=[os.path.join(DATA_PATH, 'ranking_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'ranking_data.csv')], ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=21-step=154.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=21-step=154.ckpt') ) ) saved_model = RankingMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=21-step=154.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=21-step=154.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "ranking_data.csv") + os.path.join(DATA_PATH, 'ranking_data.csv') ) # Scores for "superior" translations pos_translations = [ - {"src": s["src"], "mt": s["pos"], "ref": s["ref"]} for s in dataset + {'src': s['src'], 'mt': s['pos'], 'ref': s['ref']} for s in dataset ] dataloader = DataLoader( dataset=pos_translations, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_pos = torch.cat([p["scores"] for p in predictions], dim=0) + y_pos = torch.cat([p['scores'] for p in predictions], dim=0) # Scores for "worse" translations neg_translations = [ - {"src": s["src"], "mt": s["neg"], "ref": s["ref"]} for s in dataset + {'src': s['src'], 'mt': s['neg'], 'ref': s['ref']} for s in dataset ] dataloader = DataLoader( dataset=neg_translations, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_neg = torch.cat([p["scores"] for p in predictions], dim=0) + y_neg = torch.cat([p['scores'] for p in predictions], dim=0) ## This shouldn't break! pearsonr(y_pos, y_neg)[0] diff --git a/tests/integration/models/test_referenceless_regression.py b/tests/integration/models/test_referenceless_regression.py index 44f2c8cf..2b023f1d 100644 --- a/tests/integration/models/test_referenceless_regression.py +++ b/tests/integration/models/test_referenceless_regression.py @@ -5,15 +5,16 @@ import warnings import torch -from comet.models import ReferencelessRegression from pytorch_lightning import seed_everything from pytorch_lightning.trainer.trainer import Trainer from scipy.stats import pearsonr -from tests.data import DATA_PATH from torch.utils.data import DataLoader -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +from comet.models import ReferencelessRegression +from tests.data import DATA_PATH + +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestReferencelessRegression(unittest.TestCase): @@ -21,13 +22,13 @@ class TestReferencelessRegression(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_training(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=10, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -37,52 +38,52 @@ def test_training(self): model = ReferencelessRegression( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-04, learning_rate=1e-04, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - pool="avg", - layer="mix", - layer_transformation="sparsemax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + pool='avg', + layer='mix', + layer_transformation='sparsemax', layer_norm=True, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) ) saved_model = ReferencelessRegression.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.85 diff --git a/tests/integration/models/test_regression_metric.py b/tests/integration/models/test_regression_metric.py index 7a7058ec..d32ce499 100644 --- a/tests/integration/models/test_regression_metric.py +++ b/tests/integration/models/test_regression_metric.py @@ -5,15 +5,16 @@ import warnings import torch -from comet.models import RegressionMetric from pytorch_lightning import seed_everything from pytorch_lightning.trainer.trainer import Trainer from scipy.stats import pearsonr -from tests.data import DATA_PATH from torch.utils.data import DataLoader -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +from comet.models import RegressionMetric +from tests.data import DATA_PATH + +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestRegressionMetric(unittest.TestCase): @@ -21,13 +22,13 @@ class TestRegressionMetric(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_training(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=10, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -37,52 +38,52 @@ def test_training(self): model = RegressionMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-04, learning_rate=1e-04, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - pool="avg", - layer="mix", - layer_transformation="sparsemax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + pool='avg', + layer='mix', + layer_transformation='sparsemax', layer_norm=True, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) ) saved_model = RegressionMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.85 diff --git a/tests/integration/models/test_unified_metric.py b/tests/integration/models/test_unified_metric.py index 13bd9c2a..61daf39b 100644 --- a/tests/integration/models/test_unified_metric.py +++ b/tests/integration/models/test_unified_metric.py @@ -14,8 +14,8 @@ from comet.models import UnifiedMetric from tests.data import DATA_PATH -os.environ["TOKENIZERS_PARALLELISM"] = "false" -os.environ["OMP_NUM_THREADS"] = "1" +os.environ['TOKENIZERS_PARALLELISM'] = 'false' +os.environ['OMP_NUM_THREADS'] = '1' class TestUnifiedMetric(unittest.TestCase): @@ -23,13 +23,13 @@ class TestUnifiedMetric(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "checkpoints")) + shutil.rmtree(os.path.join(DATA_PATH, 'checkpoints')) def test_regression_with_references(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=8, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -39,62 +39,62 @@ def test_regression_with_references(self): model = UnifiedMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-03, learning_rate=1e-03, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - sent_layer="mix", - layer_transformation="softmax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + sent_layer='mix', + layer_transformation='softmax', layer_norm=False, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, - input_segments=["mt", "src", "ref"], + input_segments=['mt', 'src', 'ref'], word_level_training=False, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=7-step=104.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=7-step=104.ckpt') ) ) saved_model = UnifiedMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=7-step=104.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=7-step=104.ckpt') ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.9 def test_regression_without_references(self): seed_everything(12) trainer = Trainer( devices=1 if torch.cuda.device_count() > 0 else 0, - accelerator="auto", + accelerator='auto', max_epochs=10, enable_checkpointing=True, default_root_dir=DATA_PATH, @@ -104,55 +104,54 @@ def test_regression_without_references(self): model = UnifiedMetric( nr_frozen_epochs=1, keep_embeddings_frozen=False, - optimizer="AdamW", + optimizer='AdamW', encoder_learning_rate=1e-03, learning_rate=1e-03, layerwise_decay=0.95, - encoder_model="BERT", - pretrained_model="google/bert_uncased_L-2_H-128_A-2", - sent_layer="mix", - layer_transformation="softmax", + encoder_model='BERT', + pretrained_model='google/bert_uncased_L-2_H-128_A-2', + sent_layer='mix', + layer_transformation='softmax', layer_norm=False, - loss="mse", + loss='mse', dropout=0.1, batch_size=32, - train_data=[os.path.join(DATA_PATH, "regression_data.csv")], - validation_data=[os.path.join(DATA_PATH, "regression_data.csv")], + train_data=[os.path.join(DATA_PATH, 'regression_data.csv')], + validation_data=[os.path.join(DATA_PATH, 'regression_data.csv')], hidden_sizes=[384], - activations="Tanh", + activations='Tanh', final_activation=None, - input_segments=["mt", "src"], + input_segments=['mt', 'src'], word_level_training=False, ) warnings.filterwarnings( - "ignore", + 'ignore', category=UserWarning, - message=".*Consider increasing the value of the `num_workers` argument` .*", + message='.*Consider increasing the value of the `num_workers` argument` .*', ) trainer.fit(model) self.assertTrue( os.path.exists( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt") + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt') ) ) saved_model = UnifiedMetric.load_from_checkpoint( - os.path.join(DATA_PATH, "checkpoints", "epoch=9-step=130.ckpt"), - input_segments=["mt", "src"], + os.path.join(DATA_PATH, 'checkpoints', 'epoch=9-step=130.ckpt'), + input_segments=['mt', 'src'], ) dataset = saved_model.read_validation_data( - os.path.join(DATA_PATH, "regression_data.csv") + os.path.join(DATA_PATH, 'regression_data.csv') ) - y = [s["score"] for s in dataset] + y = [s['score'] for s in dataset] dataloader = DataLoader( dataset=dataset, batch_size=256, - collate_fn=lambda x: saved_model.prepare_sample(x, stage="predict"), + collate_fn=lambda x: saved_model.prepare_sample(x, stage='predict'), num_workers=2, ) predictions = trainer.predict( - ckpt_path="best", dataloaders=dataloader, return_predictions=True + ckpt_path='best', dataloaders=dataloader, return_predictions=True ) - y_hat = torch.cat([p["scores"] for p in predictions], dim=0).tolist() + y_hat = torch.cat([p['scores'] for p in predictions], dim=0).tolist() assert pearsonr(y_hat, y)[0] > 0.9 - diff --git a/tests/integration/modules/test_feedforward.py b/tests/integration/modules/test_feedforward.py index 3b1aaf9f..da80f899 100644 --- a/tests/integration/modules/test_feedforward.py +++ b/tests/integration/modules/test_feedforward.py @@ -2,12 +2,12 @@ import unittest import torch +from pytorch_lightning import seed_everything from sklearn.datasets import load_digits from sklearn.model_selection import train_test_split from torch import nn from comet.modules.feedforward import FeedForward -from pytorch_lightning import seed_everything class TestFeedForward(unittest.TestCase): @@ -22,9 +22,12 @@ def test_MNIST(self): images = [torch.Tensor(images[i, :]) for i in range(images.shape[0])] labels = torch.tensor(labels, dtype=torch.long) - train_images, test_images, train_labels, test_labels = train_test_split( - images, labels, test_size=0.2, random_state=42 - ) + ( + train_images, + test_images, + train_labels, + test_labels, + ) = train_test_split(images, labels, test_size=0.2, random_state=42) train_dataset = list(zip(train_images, train_labels)) test_dataset = list(zip(test_images, test_labels)) @@ -52,7 +55,7 @@ def test_MNIST(self): in_dim=8 * 8, out_dim=10, hidden_sizes=[100], - activations="Tanh", + activations='Tanh', ) """ diff --git a/tests/unit/encoders/test_bert.py b/tests/unit/encoders/test_bert.py index 51d158d7..1dda6542 100644 --- a/tests/unit/encoders/test_bert.py +++ b/tests/unit/encoders/test_bert.py @@ -5,7 +5,7 @@ class TestBERTEncoder(unittest.TestCase): - bert = BERTEncoder.from_pretrained("google/bert_uncased_L-2_H-128_A-2") + bert = BERTEncoder.from_pretrained('google/bert_uncased_L-2_H-128_A-2') def test_num_layers(self): self.assertEqual(self.bert.num_layers, 3) @@ -17,26 +17,26 @@ def test_max_positions(self): self.assertEqual(self.bert.max_positions, 510) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) model_output = self.bert(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] translations_input = self.bert.prepare_sample( @@ -95,9 +95,14 @@ def test_concat_sequences(self): continuous_input = self.bert.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) self.assertListEqual( - continuous_input[0]["token_type_ids"].tolist(), expected_token_type_ids + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) + self.assertListEqual( + continuous_input[0]['token_type_ids'].tolist(), + expected_token_type_ids, ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/encoders/test_minilm.py b/tests/unit/encoders/test_minilm.py index b98dd31f..0d5b99dc 100644 --- a/tests/unit/encoders/test_minilm.py +++ b/tests/unit/encoders/test_minilm.py @@ -7,7 +7,9 @@ class TestMiniLMEncoder(unittest.TestCase): """MiniLMV2 uses XLM-R tokenizer thus, most tests are copy of minilmEncoder""" - minilm = MiniLMEncoder.from_pretrained("microsoft/Multilingual-MiniLM-L12-H384") + minilm = MiniLMEncoder.from_pretrained( + 'microsoft/Multilingual-MiniLM-L12-H384' + ) def test_num_layers(self): self.assertEqual(self.minilm.num_layers, 13) @@ -19,26 +21,26 @@ def test_max_positions(self): self.assertEqual(self.minilm.max_positions, 510) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.minilm.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.minilm.prepare_sample(sample) model_output = self.minilm(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] source_input = self.minilm.prepare_sample(source) @@ -63,7 +65,23 @@ def test_concat_sequences(self): 38, 2, ], - [0, 40088, 393, 286, 15946, 38, 2, 2, 3293, 83, 142, 27781, 38, 2, 1], + [ + 0, + 40088, + 393, + 286, + 15946, + 38, + 2, + 2, + 3293, + 83, + 142, + 27781, + 38, + 2, + 1, + ], ] expected_labels = [ [0, 0, 0, 0, 0, 2, 2, 0, -1, -1, -1, -1, -1, -1, -1], @@ -73,6 +91,10 @@ def test_concat_sequences(self): continuous_input = self.minilm.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) + self.assertListEqual( + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/encoders/test_rembert.py b/tests/unit/encoders/test_rembert.py index dfe47885..e53cd31e 100644 --- a/tests/unit/encoders/test_rembert.py +++ b/tests/unit/encoders/test_rembert.py @@ -5,7 +5,7 @@ class TestRemBERTEncoder(unittest.TestCase): - bert = RemBERTEncoder.from_pretrained("google/rembert") + bert = RemBERTEncoder.from_pretrained('google/rembert') def test_num_layers(self): self.assertEqual(self.bert.num_layers, 33) @@ -17,26 +17,26 @@ def test_max_positions(self): self.assertEqual(self.bert.max_positions, 510) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.bert.prepare_sample(sample) model_output = self.bert(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] translations_input = self.bert.prepare_sample( @@ -60,7 +60,22 @@ def test_concat_sequences(self): 646, 313, ], - [312, 58378, 921, 835, 17293, 646, 313, 1357, 619, 666, 7469, 646, 313, 0], + [ + 312, + 58378, + 921, + 835, + 17293, + 646, + 313, + 1357, + 619, + 666, + 7469, + 646, + 313, + 0, + ], ] expected_labels = [ [0, 0, 0, 0, 0, 2, 2, 0, -1, -1, -1, -1, -1, -1], @@ -74,9 +89,14 @@ def test_concat_sequences(self): continuous_input = self.bert.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) self.assertListEqual( - continuous_input[0]["token_type_ids"].tolist(), expected_token_type_ids + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) + self.assertListEqual( + continuous_input[0]['token_type_ids'].tolist(), + expected_token_type_ids, ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/encoders/test_xlmr.py b/tests/unit/encoders/test_xlmr.py index a0b9497d..43d1282b 100644 --- a/tests/unit/encoders/test_xlmr.py +++ b/tests/unit/encoders/test_xlmr.py @@ -5,7 +5,7 @@ class TestXLMREncoder(unittest.TestCase): - xlmr = XLMREncoder.from_pretrained("xlm-roberta-base") + xlmr = XLMREncoder.from_pretrained('xlm-roberta-base') def test_num_layers(self): self.assertEqual(self.xlmr.num_layers, 13) @@ -17,26 +17,26 @@ def test_max_positions(self): self.assertEqual(self.xlmr.max_positions, 512) def test_prepare_sample(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.xlmr.prepare_sample(sample) - self.assertIn("input_ids", model_input) - self.assertIn("attention_mask", model_input) + self.assertIn('input_ids', model_input) + self.assertIn('attention_mask', model_input) def test_forward(self): - sample = ["hello world, welcome to COMET!", "This is a batch"] + sample = ['hello world, welcome to COMET!', 'This is a batch'] model_input = self.xlmr.prepare_sample(sample) model_output = self.xlmr(**model_input) - self.assertIn("wordemb", model_output) - self.assertIn("sentemb", model_output) - self.assertIn("all_layers", model_output) - self.assertIn("attention_mask", model_output) + self.assertIn('wordemb', model_output) + self.assertIn('sentemb', model_output) + self.assertIn('all_layers', model_output) + self.assertIn('attention_mask', model_output) def test_concat_sequences(self): """Basic testcase to check if we can joint two sequences into a contiguous input""" - translations = ["Bem vindos ao COMET", "Isto é um exemplo!"] - source = ["Welcome to COMET!", "This is an example!"] + translations = ['Bem vindos ao COMET', 'Isto é um exemplo!'] + source = ['Welcome to COMET!', 'This is an example!'] annotations = [ - [{"start": 14, "end": 19, "text": "COMET", "severity": "major"}], + [{'start': 14, 'end': 19, 'text': 'COMET', 'severity': 'major'}], [], ] translations_input = self.xlmr.prepare_sample( @@ -61,7 +61,23 @@ def test_concat_sequences(self): 38, 2, ], - [0, 40088, 393, 286, 15946, 38, 2, 2, 3293, 83, 142, 27781, 38, 2, 1], + [ + 0, + 40088, + 393, + 286, + 15946, + 38, + 2, + 2, + 3293, + 83, + 142, + 27781, + 38, + 2, + 1, + ], ] expected_labels = [ [0, 0, 0, 0, 0, 2, 2, 0, -1, -1, -1, -1, -1, -1, -1], @@ -71,6 +87,10 @@ def test_concat_sequences(self): continuous_input = self.xlmr.concat_sequences( [translations_input, source_input], return_label_ids=True ) - self.assertListEqual(continuous_input[0]["input_ids"].tolist(), expected_tokens) + self.assertListEqual( + continuous_input[0]['input_ids'].tolist(), expected_tokens + ) self.assertListEqual(continuous_input[1].tolist(), seq_size) - self.assertListEqual(continuous_input[0]["label_ids"].tolist(), expected_labels) + self.assertListEqual( + continuous_input[0]['label_ids'].tolist(), expected_labels + ) diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 1301bb14..5e36cfbe 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -38,5 +38,6 @@ def test_cache(self): tmp = self.add(torch.tensor(0), torch.tensor(1)) self.assertTrue( - "Tensor needs to be at least 1-Dimensional." in str(context.exception) + 'Tensor needs to be at least 1-Dimensional.' + in str(context.exception) ) diff --git a/tests/unit/test_download_load.py b/tests/unit/test_download_load.py index be334398..afba7c4e 100644 --- a/tests/unit/test_download_load.py +++ b/tests/unit/test_download_load.py @@ -1,25 +1,32 @@ # -*- coding: utf-8 -*- -import unittest import os import shutil -from tests.data import DATA_PATH +import unittest + from comet import download_model from comet.models import load_from_checkpoint +from tests.data import DATA_PATH class TestDownloadModel(unittest.TestCase): @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "models--Unbabel--wmt22-comet-da")) - shutil.rmtree(os.path.join(DATA_PATH, "eamt22-cometinho-da")) + shutil.rmtree( + os.path.join(DATA_PATH, 'models--Unbabel--wmt22-comet-da') + ) + shutil.rmtree(os.path.join(DATA_PATH, 'eamt22-cometinho-da')) def test_download_from_aws(self): - data_path = download_model("eamt22-cometinho-da", saving_directory=DATA_PATH) + data_path = download_model( + 'eamt22-cometinho-da', saving_directory=DATA_PATH + ) load_from_checkpoint(data_path) def test_download_fail(self): - self.assertRaises(KeyError, download_model, "this_model_does_not_exist") + self.assertRaises(KeyError, download_model, 'this_model_does_not_exist') def test_download_from_hf(self): - data_path = download_model("Unbabel/wmt22-comet-da", saving_directory=DATA_PATH) + data_path = download_model( + 'Unbabel/wmt22-comet-da', saving_directory=DATA_PATH + ) load_from_checkpoint(data_path) diff --git a/tests/unit/test_models_predict.py b/tests/unit/test_models_predict.py index a1d0d1b5..7ee6dd46 100644 --- a/tests/unit/test_models_predict.py +++ b/tests/unit/test_models_predict.py @@ -11,92 +11,306 @@ from tests.data import DATA_PATH TEST_SAMPLES = [ - {"lp": "it-en", "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", "mt": "In 1884, Tesla accepted a job at the Pacific League of New York City and moved to the United States of America.", "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", "annotations": [{"start": 37, "end": 51, "text": "Pacific League", "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.5833333333333333}, - {"lp": "it-en", "src": "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", "mt": "In 1884, Tesla accepted a job at the Edison Company of New York City and moved to the United States of America.", "ref": "In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", "mt": "The American Civil War was also an inspiration for many other oppressed workers from various countries to start their own revolution.", "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", "annotations": [{"text": "American Civil War", "start": 4, "end": 22, "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.5454545454545454}, - {"lp": "it-en", "src": "La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.", "mt": "The French Revolution was also an inspiration for many other oppressed workers from various countries to start their own revolution.", "ref": "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", "mt": "Lokomotiv Moscow activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", "annotations": [{"start": 0, "end": 16, "text": "Lokomotiv Moscow", "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.6}, - {"lp": "it-en", "src": "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", "mt": "Kundalini Yoga activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.", "ref": "With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", "mt": "The East African islands are located in the Longxi River, off the east coast of Africa.", "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", "annotations": [{"start": 44, "end": 56, "text": "Longxi River", "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.4444444444444444}, - {"lp": "it-en", "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of American Dollar (GBP).", "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", "annotations": [{"text": "of American Dollar", "start": 112, "end": 130, "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.6551724137931034}, - {"lp": "it-en", "src": "Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).", "mt": "The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of the British Pound (GBP).", "ref": "The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).", "annotations": [], "score": 1.0}, - {"lp": "it-en", "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful Indonesian wildlife.", "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", "annotations": [{"text": "Indonesian", "start": 115, "end": 125, "category": "critical_id9_ne_replaced", "severity": "major"}], "score": 0.6153846153846154}, - {"lp": "it-en", "src": "Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.", "mt": "In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful African wildlife.", "ref": "The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.", "annotations": [], "score": 1.0} + { + 'lp': 'it-en', + 'src': "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", + 'mt': 'In 1884, Tesla accepted a job at the Pacific League of New York City and moved to the United States of America.', + 'ref': 'In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.', + 'annotations': [ + { + 'start': 37, + 'end': 51, + 'text': 'Pacific League', + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', + } + ], + 'score': 0.5833333333333333, + }, + { + 'lp': 'it-en', + 'src': "Nel 1884, Tesla accettò un'offerta di lavoro presso la Edison Company di New York City e questo lo portò a trasferirsi negli Stati Uniti d’America.", + 'mt': 'In 1884, Tesla accepted a job at the Edison Company of New York City and moved to the United States of America.', + 'ref': 'In 1884, Tesla accepted a job with the Edison Company in New York City and moved to the United States of America.', + 'annotations': [], + 'score': 1.0, + }, + { + 'lp': 'it-en', + 'src': 'La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.', + 'mt': 'The American Civil War was also an inspiration for many other oppressed workers from various countries to start their own revolution.', + 'ref': "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", + 'annotations': [ + { + 'text': 'American Civil War', + 'start': 4, + 'end': 22, + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', + } + ], + 'score': 0.5454545454545454, + }, + { + 'lp': 'it-en', + 'src': 'La Rivoluzione francese è stata una fonte di ispirazione anche per molti altri lavoratori oppressi di vari Paesi per iniziare la propria rivoluzione.', + 'mt': 'The French Revolution was also an inspiration for many other oppressed workers from various countries to start their own revolution.', + 'ref': "The French Revolution also inspired many other repressed working class people of other country's to began their own revolutions.", + 'annotations': [], + 'score': 1.0, + }, + { + 'lp': 'it-en', + 'src': "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", + 'mt': 'Lokomotiv Moscow activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.', + 'ref': 'With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.', + 'annotations': [ + { + 'start': 0, + 'end': 16, + 'text': 'Lokomotiv Moscow', + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', + } + ], + 'score': 0.6, + }, + { + 'lp': 'it-en', + 'src': "Il Kundalini Yoga attiva l'energia Kundalini (energia dell'illuminazione) mediante l'impiego di posizioni yoga, esercizi di respirazione, mantra e visualizzazioni.", + 'mt': 'Kundalini Yoga activates the Kundalini energy (energy of enlightenment) through the use of yoga postures, breathing exercises, mantras and visualizations.', + 'ref': 'With Kundalini Yoga the Kundalini energy (enlightenment energy) is awakened through yoga postures, breathing exercises, mantras and visualizations.', + 'annotations': [], + 'score': 1.0, + }, + { + 'lp': 'it-en', + 'src': "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + 'mt': 'The East African islands are located in the Longxi River, off the east coast of Africa.', + 'ref': 'The East African Islands are in the Indian Ocean off the eastern coast of Africa.', + 'annotations': [ + { + 'start': 44, + 'end': 56, + 'text': 'Longxi River', + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', + } + ], + 'score': 0.4444444444444444, + }, + { + 'lp': 'it-en', + 'src': "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + 'mt': 'The East African islands are located in the Indian Ocean, off the east coast of Africa.', + 'ref': 'The East African Islands are in the Indian Ocean off the eastern coast of Africa.', + 'annotations': [], + 'score': 1.0, + }, + { + 'lp': 'it-en', + 'src': 'Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).', + 'mt': 'The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of American Dollar (GBP).', + 'ref': 'The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).', + 'annotations': [ + { + 'text': 'of American Dollar', + 'start': 112, + 'end': 130, + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', + } + ], + 'score': 0.6551724137931034, + }, + { + 'lp': 'it-en', + 'src': 'Le Falkland hanno la loro moneta ufficiale, la sterlina delle Falkland (FKP), il cui valore equivale a quello della sterlina britannica (GBP).', + 'mt': 'The Falklands have their official currency, the Falklands Pound (FKP), the value of which is equivalent to that of the British Pound (GBP).', + 'ref': 'The official Falklands currency is the Falkland pound (FKP) whose value is set equivalent to that of one British pound (GBP).', + 'annotations': [], + 'score': 1.0, + }, + { + 'lp': 'it-en', + 'src': 'Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.', + 'mt': 'In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful Indonesian wildlife.', + 'ref': 'The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.', + 'annotations': [ + { + 'text': 'Indonesian', + 'start': 115, + 'end': 125, + 'category': 'critical_id9_ne_replaced', + 'severity': 'major', + } + ], + 'score': 0.6153846153846154, + }, + { + 'lp': 'it-en', + 'src': 'Nel suo uso popolare, il termine safari fa riferimento a viaggi svolti via terra, in particolare nella savana, per vedere la bellissima fauna selvatica africana.', + 'mt': 'In its popular usage, the term safari refers to trips made by land, particularly in the savannah, to see beautiful African wildlife.', + 'ref': 'The term safari in popular use refers to overland travel to view the stunning African wildlife, particularly on savanna.', + 'annotations': [], + 'score': 1.0, + }, ] CONTEXT_TEST_SAMPLES = [ - {"lp": "it-en", "context_src": None, "src": "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", "context_mt": None, "mt": "The East African islands are located in the Indian Ocean, off the east coast of Africa.", "context_ref": None, "ref": "The East African Islands are in the Indian Ocean off the eastern coast of Africa.", "annotations": [], "score": 1.0}, + { + 'lp': 'it-en', + 'context_src': None, + 'src': "Le isole dell'Africa orientale sono situate nell'Oceano Indiano, al largo della costa est dell'Africa.", + 'context_mt': None, + 'mt': 'The East African islands are located in the Indian Ocean, off the east coast of Africa.', + 'context_ref': None, + 'ref': 'The East African Islands are in the Indian Ocean off the eastern coast of Africa.', + 'annotations': [], + 'score': 1.0, + }, ] + class TestUnifiedMetricPredict(unittest.TestCase): - - model = load_from_checkpoint(download_model("Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH)) + + model = load_from_checkpoint( + download_model( + 'Unbabel/test-model-whimsical-whisper', saving_directory=DATA_PATH + ) + ) gpus = 1 if torch.cuda.device_count() > 0 else 0 - + @classmethod def tearDownClass(cls): - shutil.rmtree(os.path.join(DATA_PATH, "models--Unbabel--test-model-whimsical-whisper")) + shutil.rmtree( + os.path.join( + DATA_PATH, 'models--Unbabel--test-model-whimsical-whisper' + ) + ) def test_predict(self): - model_output = self.model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) - assert "error_spans" in model_output.metadata - assert "src_scores" in model_output.metadata - assert "ref_scores" in model_output.metadata - assert "unified_scores" in model_output.metadata - + model_output = self.model.predict( + TEST_SAMPLES, batch_size=12, gpus=self.gpus + ) + assert 'error_spans' in model_output.metadata + assert 'src_scores' in model_output.metadata + assert 'ref_scores' in model_output.metadata + assert 'unified_scores' in model_output.metadata + expected_scores = np.array( - [model_output.metadata.src_scores, model_output.metadata.ref_scores, model_output.metadata.unified_scores] + [ + model_output.metadata.src_scores, + model_output.metadata.ref_scores, + model_output.metadata.unified_scores, + ] ).mean(axis=0) - + # Assert for almost equal Arrays or Numbers - np.testing.assert_almost_equal(expected_scores, np.array(model_output.scores), decimal=5) - np.testing.assert_almost_equal(model_output.system_score, expected_scores.mean(), 5) + np.testing.assert_almost_equal( + expected_scores, np.array(model_output.scores), decimal=5 + ) + np.testing.assert_almost_equal( + model_output.system_score, expected_scores.mean(), 5 + ) def test_context_predict(self): self.model.enable_context() assert self.model.use_context == False - + def test_length_batching(self): - output_without_length_batching = self.model.predict(TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=False) - output_with_length_batching = self.model.predict(TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=True) - self.assertListEqual(output_without_length_batching.scores, output_with_length_batching.scores) - + output_without_length_batching = self.model.predict( + TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=False + ) + output_with_length_batching = self.model.predict( + TEST_SAMPLES, batch_size=1, gpus=self.gpus, length_batching=True + ) + self.assertListEqual( + output_without_length_batching.scores, + output_with_length_batching.scores, + ) + def test_xcomet_predict(self): model = XCOMETMetric.load_from_checkpoint( - checkpoint_path=download_model("Unbabel/test-model-whimsical-whisper", saving_directory=DATA_PATH), - map_location=torch.device("cpu"), + checkpoint_path=download_model( + 'Unbabel/test-model-whimsical-whisper', + saving_directory=DATA_PATH, + ), + map_location=torch.device('cpu'), strict=False, **dict(self.model.hparams), ) model.score_weights = [0.25, 0.25, 0.25, 0.25] - model_output = model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) - assert "mqm_scores" in model_output.metadata - + model_output = model.predict( + TEST_SAMPLES, batch_size=12, gpus=self.gpus + ) + assert 'mqm_scores' in model_output.metadata + # on XCOMET we cap all scores at 1. and final score is a weighted average of 4 features. - expected_scores = np.array([ - np.array(list(map(lambda x: 1.0 if x > 1.0 else x, model_output.metadata.src_scores))), - np.array(list(map(lambda x: 1.0 if x > 1.0 else x, model_output.metadata.ref_scores))), - np.array(list(map(lambda x: 1.0 if x > 1.0 else x, model_output.metadata.unified_scores))), - model_output.metadata.mqm_scores - ]).mean(axis=0) - np.testing.assert_almost_equal(expected_scores, np.array(model_output.scores), decimal=5) + expected_scores = np.array( + [ + np.array( + list( + map( + lambda x: 1.0 if x > 1.0 else x, + model_output.metadata.src_scores, + ) + ) + ), + np.array( + list( + map( + lambda x: 1.0 if x > 1.0 else x, + model_output.metadata.ref_scores, + ) + ) + ), + np.array( + list( + map( + lambda x: 1.0 if x > 1.0 else x, + model_output.metadata.unified_scores, + ) + ) + ), + model_output.metadata.mqm_scores, + ] + ).mean(axis=0) + np.testing.assert_almost_equal( + expected_scores, np.array(model_output.scores), decimal=5 + ) # Put all the weight on MQM score. model.score_weights = [0, 0, 0, 1] - model_output = model.predict(TEST_SAMPLES, batch_size=12, gpus=self.gpus) - self.assertListEqual(model_output.scores, model_output.metadata.mqm_scores) + model_output = model.predict( + TEST_SAMPLES, batch_size=12, gpus=self.gpus + ) + self.assertListEqual( + model_output.scores, model_output.metadata.mqm_scores + ) class TestRegressionMetricPredict(unittest.TestCase): - - model = load_from_checkpoint(download_model("Unbabel/eamt22-cometinho-da", saving_directory=DATA_PATH)) + + model = load_from_checkpoint( + download_model( + 'Unbabel/eamt22-cometinho-da', saving_directory=DATA_PATH + ) + ) gpus = 1 if torch.cuda.device_count() > 0 else 0 - + def test_context_predict(self): # Enabling context should not change scores" - model_output_context_disabled = self.model.predict(CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus) + model_output_context_disabled = self.model.predict( + CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus + ) self.model.enable_context() assert self.model.use_context == True - model_output_context_enabled = self.model.predict(CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus) - np.testing.assert_almost_equal(np.array(model_output_context_disabled.scores), np.array(model_output_context_enabled.scores), decimal=5) + model_output_context_enabled = self.model.predict( + CONTEXT_TEST_SAMPLES, batch_size=2, gpus=self.gpus + ) + np.testing.assert_almost_equal( + np.array(model_output_context_disabled.scores), + np.array(model_output_context_enabled.scores), + decimal=5, + )