From 32f1ed497d0c103faa3861996d631971d868c347 Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Sun, 26 Jul 2026 18:41:52 -0400 Subject: [PATCH 01/14] chore: bump whisper.cpp to v1.9.0 --- whisper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper.cpp b/whisper.cpp index 9386f23..86c40c3 160000 --- a/whisper.cpp +++ b/whisper.cpp @@ -1 +1 @@ -Subproject commit 9386f239401074690479731c1e41683fbbeac557 +Subproject commit 86c40c3bd6fc86f1187fb751d111b49e0fc18e84 From cf316fce6d483a6550adb2c2c703f591f9bcfe4b Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Sun, 26 Jul 2026 18:49:52 -0400 Subject: [PATCH 02/14] chore: ignore versioned .so files --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d28e8f1..c61ff42 100644 --- a/.gitignore +++ b/.gitignore @@ -29,7 +29,7 @@ __pycache__/ *$py.class # C extensions -*.so +*.so* # Distribution / packaging .Python From a841f694ed9293ef5f45f2a6390d975bba2a185b Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Sun, 26 Jul 2026 19:58:07 -0400 Subject: [PATCH 03/14] refactor: extract whisper bindings into separate file --- .gitignore | 2 + src/main.cpp | 1371 +------------------------------------- src/whisper_bindings.cpp | 1371 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 1375 insertions(+), 1369 deletions(-) create mode 100644 src/whisper_bindings.cpp diff --git a/.gitignore b/.gitignore index c61ff42..67ab37e 100644 --- a/.gitignore +++ b/.gitignore @@ -153,4 +153,6 @@ dmypy.json # Pyre type checker .pyre/ +.vscode + diff --git a/src/main.cpp b/src/main.cpp index 6bc3c00..805ffce 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,845 +10,7 @@ ******************************************************************************** */ -#include -#include -#include -#include - -#include "whisper.h" - - -#define STRINGIFY(x) #x -#define MACRO_STRINGIFY(x) STRINGIFY(x) - -#define DEF_RELEASE_GIL(name, fn, doc) \ - m.def(name, fn, doc, py::call_guard()) - - -namespace py = pybind11; -using namespace pybind11::literals; // to bring in the `_a` literal - -inline bool has_python_user_data(const py::object & obj) { - return obj.ptr() != nullptr && obj.ptr() != Py_None; -} - - -py::object py_log_callback; - - -// whisper context wrapper, to solve the incomplete type issue -// Thanks to https://github.com/pybind/pybind11/issues/2770 -struct whisper_context_wrapper { - whisper_context* ptr; -}; - -// struct inside params -struct greedy{ - int best_of; -}; - -struct beam_search{ - int beam_size; - float patience; -}; - - -struct whisper_model_loader_wrapper { - whisper_model_loader* ptr; - -}; - -struct whisper_context_wrapper whisper_init_from_file_with_params_wrapper( - const char * path_model, - struct whisper_context_params cparams){ - struct whisper_context * ctx = whisper_init_from_file_with_params(path_model, cparams); - struct whisper_context_wrapper ctw_w; - ctw_w.ptr = ctx; - return ctw_w; -} - -struct whisper_context_wrapper whisper_init_from_buffer_with_params_wrapper( - void * buffer, - size_t buffer_size, - struct whisper_context_params cparams){ - struct whisper_context * ctx = whisper_init_from_buffer_with_params(buffer, buffer_size, cparams); - struct whisper_context_wrapper ctw_w; - ctw_w.ptr = ctx; - return ctw_w; -} - -struct whisper_context_wrapper whisper_init_with_params_wrapper( - struct whisper_model_loader_wrapper * loader, - struct whisper_context_params cparams){ - struct whisper_context * ctx = whisper_init_with_params(loader->ptr, cparams); - struct whisper_context_wrapper ctw_w; - ctw_w.ptr = ctx; - return ctw_w; -}; - -void whisper_free_wrapper(struct whisper_context_wrapper * ctx_w){ - whisper_free(ctx_w->ptr); -}; - -int whisper_pcm_to_mel_wrapper( - struct whisper_context_wrapper * ctx, - py::array_t samples, - int n_samples, - int n_threads){ - py::buffer_info buf = samples.request(); - float *samples_ptr = static_cast(buf.ptr); - return whisper_pcm_to_mel(ctx->ptr, samples_ptr, n_samples, n_threads); -}; - -int whisper_set_mel_wrapper( - struct whisper_context_wrapper * ctx, - py::array_t data, - int n_len, - int n_mel){ - py::buffer_info buf = data.request(); - float *data_ptr = static_cast(buf.ptr); - return whisper_set_mel(ctx->ptr, data_ptr, n_len, n_mel); - -}; - -int whisper_n_len_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_n_len(ctx_w->ptr); -}; - -int whisper_n_vocab_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_n_vocab(ctx_w->ptr); -}; - -int whisper_n_text_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_n_text_ctx(ctx_w->ptr); -}; - -int whisper_n_audio_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_n_audio_ctx(ctx_w->ptr); -} - -int whisper_is_multilingual_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_is_multilingual(ctx_w->ptr); -} - - -float * whisper_get_logits_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_get_logits(ctx_w->ptr); -}; - -const char * whisper_token_to_str_wrapper(struct whisper_context_wrapper * ctx_w, whisper_token token){ - return whisper_token_to_str(ctx_w->ptr, token); -}; - -py::bytes whisper_token_to_bytes_wrapper(struct whisper_context_wrapper * ctx_w, whisper_token token){ - const char* str = whisper_token_to_str(ctx_w->ptr, token); - size_t l = strlen(str); - return py::bytes(str, l); -} - -whisper_token whisper_token_eot_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_eot(ctx_w->ptr); -} - -whisper_token whisper_token_sot_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_sot(ctx_w->ptr); -} - -whisper_token whisper_token_prev_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_prev(ctx_w->ptr); -} - -whisper_token whisper_token_solm_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_solm(ctx_w->ptr); -} - -whisper_token whisper_token_not_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_not(ctx_w->ptr); -} - -whisper_token whisper_token_beg_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_beg(ctx_w->ptr); -} - -whisper_token whisper_token_lang_wrapper(struct whisper_context_wrapper * ctx_w, int lang_id){ - return whisper_token_lang(ctx_w->ptr, lang_id); -} - -whisper_token whisper_token_translate_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_translate(ctx_w->ptr); -} - -whisper_token whisper_token_transcribe_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_token_transcribe(ctx_w->ptr); -} - -void whisper_print_timings_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_print_timings(ctx_w->ptr); -} - -void whisper_reset_timings_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_reset_timings(ctx_w->ptr); -} - -int whisper_encode_wrapper( - struct whisper_context_wrapper * ctx, - int offset, - int n_threads){ - return whisper_encode(ctx->ptr, offset, n_threads); -} - - -int whisper_decode_wrapper( - struct whisper_context_wrapper * ctx, - const whisper_token * tokens, - int n_tokens, - int n_past, - int n_threads){ - return whisper_decode(ctx->ptr, tokens, n_tokens, n_past, n_threads); -}; - -int whisper_tokenize_wrapper( - struct whisper_context_wrapper * ctx, - const char * text, - whisper_token * tokens, - int n_max_tokens){ - return whisper_tokenize(ctx->ptr, text, tokens, n_max_tokens); -}; - -int whisper_lang_auto_detect_wrapper( - struct whisper_context_wrapper * ctx, - int offset_ms, - int n_threads, - py::array_t lang_probs){ - - py::buffer_info buf = lang_probs.request(); - float *lang_probs_ptr = static_cast(buf.ptr); - return whisper_lang_auto_detect(ctx->ptr, offset_ms, n_threads, lang_probs_ptr); - -} - -int whisper_full_wrapper( - struct whisper_context_wrapper * ctx_w, - struct whisper_full_params params, - py::array_t samples, - int n_samples){ - py::buffer_info buf = samples.request(); - float *samples_ptr = static_cast(buf.ptr); - - py::gil_scoped_release release; - return whisper_full(ctx_w->ptr, params, samples_ptr, n_samples); -} - -int whisper_full_parallel_wrapper( - struct whisper_context_wrapper * ctx_w, - struct whisper_full_params params, - py::array_t samples, - int n_samples, - int n_processors){ - py::buffer_info buf = samples.request(); - float *samples_ptr = static_cast(buf.ptr); - - py::gil_scoped_release release; - return whisper_full_parallel(ctx_w->ptr, params, samples_ptr, n_samples, n_processors); -} - - -int whisper_full_n_segments_wrapper(struct whisper_context_wrapper * ctx){ - py::gil_scoped_release release; - return whisper_full_n_segments(ctx->ptr); -} - -int whisper_full_lang_id_wrapper(struct whisper_context_wrapper * ctx){ - return whisper_full_lang_id(ctx->ptr); -} - -int64_t whisper_full_get_segment_t0_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ - return whisper_full_get_segment_t0(ctx->ptr, i_segment); -} - -int64_t whisper_full_get_segment_t1_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ - return whisper_full_get_segment_t1(ctx->ptr, i_segment); -} - -// https://pybind11.readthedocs.io/en/stable/advanced/cast/strings.html -const py::bytes whisper_full_get_segment_text_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ - const char * c_array = whisper_full_get_segment_text(ctx->ptr, i_segment); - size_t length = strlen(c_array); // Determine the length of the array - return py::bytes(c_array, length); // Return the data without transcoding -}; - -int whisper_full_n_tokens_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ - return whisper_full_n_tokens(ctx->ptr, i_segment); -} - -const char * whisper_full_get_token_text_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ - return whisper_full_get_token_text(ctx->ptr, i_segment, i_token); -} - -whisper_token whisper_full_get_token_id_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ - return whisper_full_get_token_id(ctx->ptr, i_segment, i_token); -} - -whisper_token_data whisper_full_get_token_data_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ - return whisper_full_get_token_data(ctx->ptr, i_segment, i_token); -} - -float whisper_full_get_token_p_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ - return whisper_full_get_token_p(ctx->ptr, i_segment, i_token); -} - -bool whisper_full_get_segment_speaker_turn_next_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ - return whisper_full_get_segment_speaker_turn_next(ctx->ptr, i_segment); -} - -const char * whisper_model_type_readable_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_type_readable(ctx_w->ptr); -} - -int whisper_model_n_vocab_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_vocab(ctx_w->ptr); -} - -int whisper_model_n_audio_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_audio_ctx(ctx_w->ptr); -} - -int whisper_model_n_audio_state_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_audio_state(ctx_w->ptr); -} - -int whisper_model_n_audio_head_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_audio_head(ctx_w->ptr); -} - -int whisper_model_n_audio_layer_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_audio_layer(ctx_w->ptr); -} - -int whisper_model_n_text_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_text_ctx(ctx_w->ptr); -} - -int whisper_model_n_text_state_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_text_state(ctx_w->ptr); -} - -int whisper_model_n_text_head_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_text_head(ctx_w->ptr); -} - -int whisper_model_n_text_layer_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_text_layer(ctx_w->ptr); -} - -int whisper_model_n_mels_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_n_mels(ctx_w->ptr); -} - -int whisper_model_ftype_wrapper(struct whisper_context_wrapper * ctx_w){ - return whisper_model_ftype(ctx_w->ptr); -} - -bool _abort_callback(void * user_data); -void _new_segment_callback(struct whisper_context * ctx, struct whisper_state * state, int n_new, void * user_data); -bool _encoder_begin_callback(struct whisper_context * ctx, struct whisper_state * state, void * user_data); -void _logits_filter_callback( - struct whisper_context * ctx, - struct whisper_state * state, - const whisper_token_data * tokens, - int n_tokens, - float * logits, - void * user_data); - -int whisper_ctx_init_openvino_encoder_wrapper(struct whisper_context_wrapper * ctx, const char * model_path, - const char * device, - const char * cache_dir){ - return whisper_ctx_init_openvino_encoder(ctx->ptr, model_path, device, cache_dir); -} - -struct WhisperFullParamsWrapper : public whisper_full_params { - std::string initial_prompt_str; - std::string suppress_regex_str; - std::string vad_model_path_str; - std::vector prompt_token_storage; - - void reset_progress_callback() { - progress_callback_user_data = this; - progress_callback = [](struct whisper_context* ctx, struct whisper_state* state, int progress, void* user_data) { - (void) ctx; - (void) state; - auto* self = static_cast(user_data); - if (self && self->print_progress) { - if (self->py_progress_callback) { - py::gil_scoped_acquire gil; - if (!has_python_user_data(self->py_progress_callback_user_data)) { - self->py_progress_callback(progress); - } else { - self->py_progress_callback(progress, self->py_progress_callback_user_data); - } - } else { - fprintf(stderr, "Progress: %3d%%\n", progress); - } - } - }; - } - - void sync_prompt_tokens() { - prompt_tokens = prompt_token_storage.empty() ? nullptr : prompt_token_storage.data(); - prompt_n_tokens = prompt_token_storage.size(); - } -public: - py::function py_new_segment_callback; - py::object py_new_segment_callback_user_data; - py::function py_encoder_begin_callback; - py::object py_encoder_begin_callback_user_data; - py::function py_progress_callback; - py::object py_progress_callback_user_data; - py::function py_logits_filter_callback; - py::object py_logits_filter_callback_user_data; - py::function py_abort_callback; - py::object py_abort_callback_user_data; - WhisperFullParamsWrapper(const whisper_full_params& params = whisper_full_params()) - : whisper_full_params(params), - initial_prompt_str(params.initial_prompt ? params.initial_prompt : ""), - suppress_regex_str(params.suppress_regex ? params.suppress_regex : ""), - vad_model_path_str(params.vad_model_path ? params.vad_model_path : ""), - prompt_token_storage(), - py_new_segment_callback_user_data(py::none()), - py_encoder_begin_callback_user_data(py::none()), - py_progress_callback_user_data(py::none()), - py_logits_filter_callback_user_data(py::none()), - py_abort_callback(), - py_abort_callback_user_data(py::none()) - { - initial_prompt = initial_prompt_str.empty() ? nullptr : initial_prompt_str.c_str(); - suppress_regex = suppress_regex_str.empty() ? nullptr : suppress_regex_str.c_str(); - vad_model_path = vad_model_path_str.empty() ? nullptr : vad_model_path_str.c_str(); - new_segment_callback_user_data = this; - encoder_begin_callback_user_data = this; - abort_callback_user_data = this; - logits_filter_callback_user_data = this; - if (params.prompt_tokens && params.prompt_n_tokens > 0) { - prompt_token_storage.assign(params.prompt_tokens, params.prompt_tokens + params.prompt_n_tokens); - } - sync_prompt_tokens(); - reset_progress_callback(); - } - WhisperFullParamsWrapper(const WhisperFullParamsWrapper& other) - : whisper_full_params(static_cast(other)), // Copy base struct - initial_prompt_str(other.initial_prompt_str), - suppress_regex_str(other.suppress_regex_str), - vad_model_path_str(other.vad_model_path_str), - prompt_token_storage(other.prompt_token_storage), - py_new_segment_callback(other.py_new_segment_callback), - py_new_segment_callback_user_data(other.py_new_segment_callback_user_data), - py_encoder_begin_callback(other.py_encoder_begin_callback), - py_encoder_begin_callback_user_data(other.py_encoder_begin_callback_user_data), - py_progress_callback(other.py_progress_callback), - py_progress_callback_user_data(other.py_progress_callback_user_data), - py_logits_filter_callback(other.py_logits_filter_callback), - py_logits_filter_callback_user_data(other.py_logits_filter_callback_user_data), - py_abort_callback(other.py_abort_callback), - py_abort_callback_user_data(other.py_abort_callback_user_data) { - // Reset pointers to new string copies - initial_prompt = initial_prompt_str.empty() ? nullptr : initial_prompt_str.c_str(); - suppress_regex = suppress_regex_str.empty() ? nullptr : suppress_regex_str.c_str(); - vad_model_path = vad_model_path_str.empty() ? nullptr : vad_model_path_str.c_str(); - new_segment_callback_user_data = this; - encoder_begin_callback_user_data = this; - abort_callback_user_data = this; - logits_filter_callback_user_data = this; - sync_prompt_tokens(); - reset_progress_callback(); - } - void set_initial_prompt(const std::string& prompt) { - initial_prompt_str = prompt; - initial_prompt = initial_prompt_str.c_str(); - } - void set_suppress_regex(const std::string& regex) { - suppress_regex_str = regex; - suppress_regex = suppress_regex_str.c_str(); - } - void set_vad_model_path(const std::string& model_path) { - vad_model_path_str = model_path; - vad_model_path = vad_model_path_str.c_str(); - } - py::tuple get_prompt_tokens() const { - py::tuple tokens(prompt_token_storage.size()); - for (size_t index = 0; index < prompt_token_storage.size(); ++index) { - tokens[index] = prompt_token_storage[index]; - } - return tokens; - } - void set_prompt_tokens(const std::vector& tokens) { - prompt_token_storage = tokens; - sync_prompt_tokens(); - } - void clear_prompt_tokens() { - prompt_token_storage.clear(); - sync_prompt_tokens(); - } - py::object get_new_segment_callback_user_data() const { - return py_new_segment_callback_user_data; - } - void set_new_segment_callback_user_data(py::object user_data) { - py_new_segment_callback_user_data = std::move(user_data); - new_segment_callback_user_data = this; - } - void set_new_segment_callback(py::function callback) { - py_new_segment_callback = std::move(callback); - new_segment_callback_user_data = this; - new_segment_callback = _new_segment_callback; - } - void clear_new_segment_callback() { - py_new_segment_callback = py::function(); - new_segment_callback = nullptr; - new_segment_callback_user_data = this; - } - py::object get_encoder_begin_callback_user_data() const { - return py_encoder_begin_callback_user_data; - } - void set_encoder_begin_callback_user_data(py::object user_data) { - py_encoder_begin_callback_user_data = std::move(user_data); - encoder_begin_callback_user_data = this; - } - void set_encoder_begin_callback(py::function callback) { - py_encoder_begin_callback = std::move(callback); - encoder_begin_callback_user_data = this; - encoder_begin_callback = _encoder_begin_callback; - } - void clear_encoder_begin_callback() { - py_encoder_begin_callback = py::function(); - encoder_begin_callback = nullptr; - encoder_begin_callback_user_data = this; - } - py::object get_progress_callback_user_data() const { - return py_progress_callback_user_data; - } - void set_progress_callback_user_data(py::object user_data) { - py_progress_callback_user_data = std::move(user_data); - progress_callback_user_data = this; - } - void set_progress_callback(py::function callback) { - py_progress_callback = std::move(callback); - reset_progress_callback(); - } - void clear_progress_callback() { - py_progress_callback = py::function(); - reset_progress_callback(); - } - py::object get_logits_filter_callback_user_data() const { - return py_logits_filter_callback_user_data; - } - void set_logits_filter_callback_user_data(py::object user_data) { - py_logits_filter_callback_user_data = std::move(user_data); - logits_filter_callback_user_data = this; - } - void set_logits_filter_callback(py::function callback) { - py_logits_filter_callback = std::move(callback); - logits_filter_callback_user_data = this; - logits_filter_callback = _logits_filter_callback; - } - void clear_logits_filter_callback() { - py_logits_filter_callback = py::function(); - logits_filter_callback = nullptr; - logits_filter_callback_user_data = this; - } - py::object get_abort_callback_user_data() const { - return py_abort_callback_user_data; - } - void set_abort_callback_user_data(py::object user_data) { - py_abort_callback_user_data = std::move(user_data); - abort_callback_user_data = this; - } - void set_abort_callback(py::function callback) { - py_abort_callback = std::move(callback); - abort_callback_user_data = this; - abort_callback = _abort_callback; - } - void clear_abort_callback() { - py_abort_callback = py::function(); - abort_callback = nullptr; - abort_callback_user_data = this; - } -}; -WhisperFullParamsWrapper whisper_full_default_params_wrapper(enum whisper_sampling_strategy strategy) { - return WhisperFullParamsWrapper(whisper_full_default_params(strategy)); -} - -// callbacks mechanism - -void _new_segment_callback(struct whisper_context * ctx, struct whisper_state * state, int n_new, void * user_data){ - (void) state; - struct whisper_context_wrapper ctx_w; - ctx_w.ptr = ctx; - auto * params = static_cast(user_data); - if (!params || !params->py_new_segment_callback) { - return; - } - - py::gil_scoped_acquire gil; - py::function callback = params->py_new_segment_callback; - if (!has_python_user_data(params->py_new_segment_callback_user_data)) { - callback(ctx_w, n_new); - } else { - callback(ctx_w, n_new, params->py_new_segment_callback_user_data); - } -}; - -void assign_new_segment_callback(struct whisper_full_params *params_base, py::object callback){ - auto * params = static_cast(params_base); - if (callback.is_none()) { - params->clear_new_segment_callback(); - return; - } - - params->set_new_segment_callback(callback.cast()); -} - -void clear_new_segment_callback(struct whisper_full_params *params_base) { - auto * params = static_cast(params_base); - params->clear_new_segment_callback(); -}; - -bool _encoder_begin_callback(struct whisper_context * ctx, struct whisper_state * state, void * user_data){ - (void) state; - struct whisper_context_wrapper ctx_w; - ctx_w.ptr = ctx; - auto * params = static_cast(user_data); - if (!params || !params->py_encoder_begin_callback) { - return false; - } - - py::gil_scoped_acquire gil; - py::function callback = params->py_encoder_begin_callback; - py::object result_py; - if (!has_python_user_data(params->py_encoder_begin_callback_user_data)) { - result_py = callback(ctx_w); - } else { - result_py = callback(ctx_w, params->py_encoder_begin_callback_user_data); - } - bool res = result_py.cast(); - return res; -} - -void assign_encoder_begin_callback(struct whisper_full_params *params_base, py::object callback){ - auto * params = static_cast(params_base); - if (callback.is_none()) { - params->clear_encoder_begin_callback(); - return; - } - - params->set_encoder_begin_callback(callback.cast()); -} - -void clear_encoder_begin_callback(struct whisper_full_params *params_base) { - auto * params = static_cast(params_base); - params->clear_encoder_begin_callback(); -} - -void _logits_filter_callback( - struct whisper_context * ctx, - struct whisper_state * state, - const whisper_token_data * tokens, - int n_tokens, - float * logits, - void * user_data){ - (void) state; - (void) tokens; - struct whisper_context_wrapper ctx_w; - ctx_w.ptr = ctx; - auto * params = static_cast(user_data); - if (!params || !params->py_logits_filter_callback) { - return; - } - - py::gil_scoped_acquire gil; - py::function callback = params->py_logits_filter_callback; - if (!has_python_user_data(params->py_logits_filter_callback_user_data)) { - callback(ctx_w, n_tokens, logits); - } else { - callback(ctx_w, n_tokens, logits, params->py_logits_filter_callback_user_data); - } -} - -void assign_logits_filter_callback(struct whisper_full_params *params_base, py::object callback){ - auto * params = static_cast(params_base); - if (callback.is_none()) { - params->clear_logits_filter_callback(); - return; - } - - params->set_logits_filter_callback(callback.cast()); -} - -void clear_logits_filter_callback(struct whisper_full_params *params_base) { - auto * params = static_cast(params_base); - params->clear_logits_filter_callback(); -} - -void assign_progress_callback(whisper_full_params *params_base, py::object callback) { - auto * params = static_cast(params_base); - if (callback.is_none()) { - params->clear_progress_callback(); - return; - } - - params->set_progress_callback(callback.cast()); -} - -void clear_progress_callback(whisper_full_params *params_base) { - auto * params = static_cast(params_base); - params->clear_progress_callback(); -} - -bool _abort_callback(void * user_data) { - auto * params = static_cast(user_data); - if (!params || !params->py_abort_callback) { - return false; - } - - py::gil_scoped_acquire gil; - py::function callback = params->py_abort_callback; - py::object result_py; - if (!has_python_user_data(params->py_abort_callback_user_data)) { - result_py = callback(); - } else { - result_py = callback(params->py_abort_callback_user_data); - } - return result_py.cast(); -} - -void assign_abort_callback(whisper_full_params *params_base, py::object callback){ - auto * params = static_cast(params_base); - if (callback.is_none()) { - params->clear_abort_callback(); - return; - } - - params->set_abort_callback(callback.cast()); -} - -void clear_abort_callback(whisper_full_params *params_base) { - auto * params = static_cast(params_base); - params->clear_abort_callback(); -} - -void whisper_log_set_wrapper(py::object callback) { - if (callback.is_none()) { - py_log_callback = py::none(); - whisper_log_set(nullptr, nullptr); - return; - } - - py_log_callback = callback.cast(); - whisper_log_set( - [](enum ggml_log_level level, const char * text, void * user_data) { - (void) user_data; - py::gil_scoped_acquire gil; - py::function log_callback = py_log_callback.cast(); - log_callback(py::int_(static_cast(level)), py::str(text ? text : "")); - }, - nullptr); -} - -py::dict get_greedy(whisper_full_params * params){ - py::dict d("best_of"_a=params->greedy.best_of); - return d; -} - - -// Voice Activity Detection (VAD) -struct whisper_vad_context_wrapper { - whisper_vad_context* ptr; -}; - -struct whisper_vad_context_wrapper whisper_vad_init_from_file_with_params_wrapper(const char * path_model, struct whisper_vad_context_params params){ - struct whisper_vad_context * ctx = whisper_vad_init_from_file_with_params(path_model, params); - struct whisper_vad_context_wrapper ctw_w; - ctw_w.ptr = ctx; - return ctw_w; -} - -bool whisper_vad_detect_speech_wrapper( - struct whisper_vad_context_wrapper * ctx, - py::array_t samples, - int n_samples){ - py::buffer_info buf = samples.request(); - float *samples_ptr = static_cast(buf.ptr); - - py::gil_scoped_release release; - return whisper_vad_detect_speech(ctx->ptr, samples_ptr, n_samples); -} - -int whisper_vad_n_probs_wrapper(struct whisper_vad_context_wrapper * ctx){ - return whisper_vad_n_probs(ctx->ptr); -} - -py::array_t whisper_vad_probs_wrapper(struct whisper_vad_context_wrapper * ctx) { - float * probs_ptr = whisper_vad_probs(ctx->ptr); - int n_probs = whisper_vad_n_probs(ctx->ptr); - - if (probs_ptr == nullptr || n_probs <= 0) { - return py::array_t(0); - } - return py::array_t( - {n_probs}, - {sizeof(float)}, - probs_ptr - ); -} - -struct whisper_vad_segments_wrapper { - struct whisper_vad_segments * ptr; -}; - -struct whisper_vad_segments_wrapper whisper_vad_segments_from_probs_wrapper( - struct whisper_vad_context_wrapper * vctx_w, - struct whisper_vad_params params - ){ - struct whisper_vad_segments * wvs = whisper_vad_segments_from_probs(vctx_w->ptr, params); - struct whisper_vad_segments_wrapper wvs_w; - wvs_w.ptr = wvs; - return wvs_w; -} - -struct whisper_vad_segments_wrapper whisper_vad_segments_from_samples_wrapper( - struct whisper_vad_context_wrapper * vctx_w, - struct whisper_vad_params params, - py::array_t samples, - int n_samples){ - - py::buffer_info buf = samples.request(); - float *samples_ptr = static_cast(buf.ptr); - - struct whisper_vad_segments * wvs = whisper_vad_segments_from_samples(vctx_w->ptr, params, samples_ptr, n_samples); - struct whisper_vad_segments_wrapper wvs_w; - wvs_w.ptr = wvs; - return wvs_w; -} - -int whisper_vad_segments_n_segments_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper){ - return whisper_vad_segments_n_segments(segments_wrapper->ptr); -} - -float whisper_vad_segments_get_segment_t0_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper, int i_segment) { - return whisper_vad_segments_get_segment_t0(segments_wrapper->ptr, i_segment); -} - -float whisper_vad_segments_get_segment_t1_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper, int i_segment) { - return whisper_vad_segments_get_segment_t1(segments_wrapper->ptr, i_segment); -} - -void whisper_vad_free_segments_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper){ - return whisper_vad_free_segments(segments_wrapper->ptr); -} - -void whisper_vad_free_wrapper(struct whisper_vad_context_wrapper * ctx_w){ - return whisper_vad_free(ctx_w->ptr); -} - -//////////// +#include "whisper_bindings.cpp" PYBIND11_MODULE(_pywhispercpp, m) { m.doc() = R"pbdoc( @@ -862,534 +24,5 @@ PYBIND11_MODULE(_pywhispercpp, m) { )pbdoc"; - m.attr("WHISPER_SAMPLE_RATE") = WHISPER_SAMPLE_RATE; - m.attr("WHISPER_N_FFT") = WHISPER_N_FFT; - m.attr("WHISPER_HOP_LENGTH") = WHISPER_HOP_LENGTH; - m.attr("WHISPER_CHUNK_SIZE") = WHISPER_CHUNK_SIZE; - - py::enum_(m, "whisper_alignment_heads_preset") - .value("WHISPER_AHEADS_NONE", whisper_alignment_heads_preset::WHISPER_AHEADS_NONE) - .value("WHISPER_AHEADS_N_TOP_MOST", whisper_alignment_heads_preset::WHISPER_AHEADS_N_TOP_MOST) - .value("WHISPER_AHEADS_CUSTOM", whisper_alignment_heads_preset::WHISPER_AHEADS_CUSTOM) - .value("WHISPER_AHEADS_TINY_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_TINY_EN) - .value("WHISPER_AHEADS_TINY", whisper_alignment_heads_preset::WHISPER_AHEADS_TINY) - .value("WHISPER_AHEADS_BASE_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_BASE_EN) - .value("WHISPER_AHEADS_BASE", whisper_alignment_heads_preset::WHISPER_AHEADS_BASE) - .value("WHISPER_AHEADS_SMALL_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_SMALL_EN) - .value("WHISPER_AHEADS_SMALL", whisper_alignment_heads_preset::WHISPER_AHEADS_SMALL) - .value("WHISPER_AHEADS_MEDIUM_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_MEDIUM_EN) - .value("WHISPER_AHEADS_MEDIUM", whisper_alignment_heads_preset::WHISPER_AHEADS_MEDIUM) - .value("WHISPER_AHEADS_LARGE_V1", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V1) - .value("WHISPER_AHEADS_LARGE_V2", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V2) - .value("WHISPER_AHEADS_LARGE_V3", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V3) - .value("WHISPER_AHEADS_LARGE_V3_TURBO", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V3_TURBO) - .export_values(); - - py::class_(m, "whisper_context"); - py::class_(m, "whisper_context_params") - .def(py::init<>()) - .def_readwrite("use_gpu", &whisper_context_params::use_gpu) - .def_readwrite("flash_attn", &whisper_context_params::flash_attn) - .def_readwrite("gpu_device", &whisper_context_params::gpu_device) - .def_readwrite("dtw_token_timestamps", &whisper_context_params::dtw_token_timestamps) - .def_readwrite("dtw_aheads_preset", &whisper_context_params::dtw_aheads_preset) - .def_readwrite("dtw_n_top", &whisper_context_params::dtw_n_top) - .def_readwrite("dtw_mem_size", &whisper_context_params::dtw_mem_size); - py::class_(m, "whisper_token") - .def(py::init<>()); - py::class_(m,"whisper_token_data") - .def(py::init<>()) - .def_readwrite("id", &whisper_token_data::id) - .def_readwrite("tid", &whisper_token_data::tid) - .def_readwrite("p", &whisper_token_data::p) - .def_readwrite("plog", &whisper_token_data::plog) - .def_readwrite("pt", &whisper_token_data::pt) - .def_readwrite("ptsum", &whisper_token_data::ptsum) - .def_readwrite("t0", &whisper_token_data::t0) - .def_readwrite("t1", &whisper_token_data::t1) - .def_readwrite("t_dtw", &whisper_token_data::t_dtw) - .def_readwrite("vlen", &whisper_token_data::vlen); - - py::class_(m,"whisper_model_loader") - .def(py::init<>()); - - m.def("whisper_context_default_params", &whisper_context_default_params, - "Return the default context parameters used during model initialization."); - DEF_RELEASE_GIL("whisper_init_from_file_with_params", &whisper_init_from_file_with_params_wrapper, "Various functions for loading a ggml whisper model.\n" - "Allocate (almost) all memory needed for the model.\n" - "Return NULL on failure"); - DEF_RELEASE_GIL("whisper_init_from_buffer_with_params", &whisper_init_from_buffer_with_params_wrapper, "Various functions for loading a ggml whisper model.\n" - "Allocate (almost) all memory needed for the model.\n" - "Return NULL on failure"); - DEF_RELEASE_GIL("whisper_init_with_params", &whisper_init_with_params_wrapper, "Various functions for loading a ggml whisper model.\n" - "Allocate (almost) all memory needed for the model.\n" - "Return NULL on failure"); - - - m.def("whisper_free", &whisper_free_wrapper, "Frees all memory allocated by the model."); - - m.def("whisper_pcm_to_mel", &whisper_pcm_to_mel_wrapper, "Convert RAW PCM audio to log mel spectrogram.\n" - "The resulting spectrogram is stored inside the provided whisper context.\n" - "Returns 0 on success"); - - m.def("whisper_set_mel", &whisper_set_mel_wrapper, " This can be used to set a custom log mel spectrogram inside the provided whisper context.\n" - "Use this instead of whisper_pcm_to_mel() if you want to provide your own log mel spectrogram.\n" - "n_mel must be 80\n" - "Returns 0 on success"); - - m.def("whisper_encode", &whisper_encode_wrapper, "Run the Whisper encoder on the log mel spectrogram stored inside the provided whisper context.\n" - "Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.\n" - "offset can be used to specify the offset of the first frame in the spectrogram.\n" - "Returns 0 on success"); - - m.def("whisper_decode", &whisper_decode_wrapper, "Run the Whisper decoder to obtain the logits and probabilities for the next token.\n" - "Make sure to call whisper_encode() first.\n" - "tokens + n_tokens is the provided context for the decoder.\n" - "n_past is the number of tokens to use from previous decoder calls.\n" - "Returns 0 on success\n" - "TODO: add support for multiple decoders"); - - m.def("whisper_tokenize", &whisper_tokenize_wrapper, "Convert the provided text into tokens.\n" - "The tokens pointer must be large enough to hold the resulting tokens.\n" - "Returns the number of tokens on success, no more than n_max_tokens\n" - "Returns -1 on failure\n" - "TODO: not sure if correct"); - - m.def("whisper_lang_max_id", &whisper_lang_max_id, "Largest language id (i.e. number of available languages - 1)"); - m.def("whisper_lang_id", &whisper_lang_id, "Return the id of the specified language, returns -1 if not found\n" - "Examples:\n" - "\"de\" -> 2\n" - "\"german\" -> 2"); - m.def("whisper_lang_str", &whisper_lang_str, "Return the short string of the specified language id (e.g. 2 -> \"de\"), returns nullptr if not found"); - - - - - - - - m.def("whisper_lang_auto_detect", &whisper_lang_auto_detect_wrapper, "Use mel data at offset_ms to try and auto-detect the spoken language\n" - "Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first\n" - "Returns the top language id or negative on failure\n" - "If not null, fills the lang_probs array with the probabilities of all languages\n" - "The array must be whispe_lang_max_id() + 1 in size\n" - "ref: https://github.com/openai/whisper/blob/main/whisper/decoding.py#L18-L69\n"); - m.def("whisper_n_len", &whisper_n_len_wrapper, "whisper_n_len"); - m.def("whisper_n_vocab", &whisper_n_vocab_wrapper, "wrapper_whisper_n_vocab"); - m.def("whisper_n_text_ctx", &whisper_n_text_ctx_wrapper, "whisper_n_text_ctx"); - m.def("whisper_n_audio_ctx", &whisper_n_audio_ctx_wrapper, "whisper_n_audio_ctx"); - m.def("whisper_is_multilingual", &whisper_is_multilingual_wrapper, "whisper_is_multilingual"); - m.def("whisper_get_logits", &whisper_get_logits_wrapper, "Token logits obtained from the last call to whisper_decode()\n" - "The logits for the last token are stored in the last row\n" - "Rows: n_tokens\n" - "Cols: n_vocab"); - - - m.def("whisper_token_to_str", &whisper_token_to_str_wrapper, "whisper_token_to_str"); - m.def("whisper_token_to_bytes", &whisper_token_to_bytes_wrapper, "whisper_token_to_bytes"); - m.def("whisper_token_eot", &whisper_token_eot_wrapper, "whisper_token_eot"); - m.def("whisper_token_sot", &whisper_token_sot_wrapper, "whisper_token_sot"); - m.def("whisper_token_prev", &whisper_token_prev_wrapper); - m.def("whisper_token_solm", &whisper_token_solm_wrapper); - m.def("whisper_token_not", &whisper_token_not_wrapper); - m.def("whisper_token_beg", &whisper_token_beg_wrapper); - m.def("whisper_token_lang", &whisper_token_lang_wrapper); - - m.def("whisper_token_translate", &whisper_token_translate_wrapper); - m.def("whisper_token_transcribe", &whisper_token_transcribe_wrapper); - - m.def("whisper_print_timings", &whisper_print_timings_wrapper); - m.def("whisper_reset_timings", &whisper_reset_timings_wrapper); - - m.def("whisper_print_system_info", &whisper_print_system_info); - - - - ////////////////////// - - py::enum_(m, "whisper_sampling_strategy") - .value("WHISPER_SAMPLING_GREEDY", whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY) - .value("WHISPER_SAMPLING_BEAM_SEARCH", whisper_sampling_strategy::WHISPER_SAMPLING_BEAM_SEARCH) - .export_values(); - - py::class_(m, "__whisper_full_params__internal") - .def(py::init<>()) - .def("__repr__", [](const whisper_full_params& self) { - std::ostringstream oss; - oss << "whisper_full_params(" - << "strategy=" << self.strategy << ", " - << "n_threads=" << self.n_threads << ", " - << "n_max_text_ctx=" << self.n_max_text_ctx << ", " - << "offset_ms=" << self.offset_ms << ", " - << "duration_ms=" << self.duration_ms << ", " - << "translate=" << (self.translate ? "True" : "False") << ", " - << "no_context=" << (self.no_context ? "True" : "False") << ", " - << "no_timestamps=" << (self.no_timestamps ? "True" : "False") << ", " - << "single_segment=" << (self.single_segment ? "True" : "False") << ", " - << "print_special=" << (self.print_special ? "True" : "False") << ", " - << "print_progress=" << (self.print_progress ? "True" : "False") << ", " - << "print_realtime=" << (self.print_realtime ? "True" : "False") << ", " - << "print_timestamps=" << (self.print_timestamps ? "True" : "False") << ", " - << "token_timestamps=" << (self.token_timestamps ? "True" : "False") << ", " - << "thold_pt=" << self.thold_pt << ", " - << "thold_ptsum=" << self.thold_ptsum << ", " - << "max_len=" << self.max_len << ", " - << "split_on_word=" << (self.split_on_word ? "True" : "False") << ", " - << "max_tokens=" << self.max_tokens << ", " - << "debug_mode=" << (self.debug_mode ? "True" : "False") << ", " - << "audio_ctx=" << self.audio_ctx << ", " - << "tdrz_enable=" << (self.tdrz_enable ? "True" : "False") << ", " - << "suppress_regex=" << (self.suppress_regex ? self.suppress_regex : "None") << ", " - << "initial_prompt=" << (self.initial_prompt ? self.initial_prompt : "None") << ", " - << "prompt_tokens=" << (self.prompt_tokens ? "(whisper_token *)" : "None") << ", " - << "prompt_n_tokens=" << self.prompt_n_tokens << ", " - << "language=" << (self.language ? self.language : "None") << ", " - << "detect_language=" << (self.detect_language ? "True" : "False") << ", " - << "suppress_blank=" << (self.suppress_blank ? "True" : "False") << ", " - << "temperature=" << self.temperature << ", " - << "max_initial_ts=" << self.max_initial_ts << ", " - << "length_penalty=" << self.length_penalty << ", " - << "temperature_inc=" << self.temperature_inc << ", " - << "entropy_thold=" << self.entropy_thold << ", " - << "logprob_thold=" << self.logprob_thold << ", " - << "no_speech_thold=" << self.no_speech_thold << ", " - << "greedy={best_of=" << self.greedy.best_of << "}, " - << "beam_search={beam_size=" << self.beam_search.beam_size << ", patience=" << self.beam_search.patience << "}, " - << "new_segment_callback=" << (self.new_segment_callback ? "(function pointer)" : "None") << ", " - << "progress_callback=" << (self.progress_callback ? "(function pointer)" : "None") << ", " - << "encoder_begin_callback=" << (self.encoder_begin_callback ? "(function pointer)" : "None") << ", " - << "abort_callback=" << (self.abort_callback ? "(function pointer)" : "None") << ", " - << "logits_filter_callback=" << (self.logits_filter_callback ? "(function pointer)" : "None") - << ")"; - return oss.str(); - }); - - py::class_(m, "whisper_full_params") - .def(py::init<>()) - .def_readwrite("strategy", &WhisperFullParamsWrapper::strategy) - .def_readwrite("n_threads", &WhisperFullParamsWrapper::n_threads) - .def_readwrite("n_max_text_ctx", &WhisperFullParamsWrapper::n_max_text_ctx) - .def_readwrite("offset_ms", &WhisperFullParamsWrapper::offset_ms) - .def_readwrite("duration_ms", &WhisperFullParamsWrapper::duration_ms) - .def_readwrite("translate", &WhisperFullParamsWrapper::translate) - .def_readwrite("no_context", &WhisperFullParamsWrapper::no_context) - .def_readwrite("no_timestamps", &WhisperFullParamsWrapper::no_timestamps) - .def_readwrite("single_segment", &WhisperFullParamsWrapper::single_segment) - .def_readwrite("print_special", &WhisperFullParamsWrapper::print_special) - .def_readwrite("print_progress", &WhisperFullParamsWrapper::print_progress) - .def_readwrite("progress_callback", &WhisperFullParamsWrapper::py_progress_callback) - .def("set_progress_callback", - [](WhisperFullParamsWrapper &self, py::object callback) { - if (callback.is_none()) { - self.clear_progress_callback(); - } else { - self.set_progress_callback(callback.cast()); - } - }, - py::arg("callback") = py::none(), - "Assign a progress callback that receives progress updates.") - .def("clear_progress_callback", &WhisperFullParamsWrapper::clear_progress_callback, - "Clear any previously assigned progress callback while preserving default progress behavior.") - .def_readwrite("print_realtime", &WhisperFullParamsWrapper::print_realtime) - .def_readwrite("print_timestamps", &WhisperFullParamsWrapper::print_timestamps) - .def_readwrite("token_timestamps", &WhisperFullParamsWrapper::token_timestamps) - .def_readwrite("thold_pt", &WhisperFullParamsWrapper::thold_pt) - .def_readwrite("thold_ptsum", &WhisperFullParamsWrapper::thold_ptsum) - .def_readwrite("max_len", &WhisperFullParamsWrapper::max_len) - .def_readwrite("split_on_word", &WhisperFullParamsWrapper::split_on_word) - .def_readwrite("max_tokens", &WhisperFullParamsWrapper::max_tokens) - .def_readwrite("debug_mode", &WhisperFullParamsWrapper::debug_mode) - .def_readwrite("audio_ctx", &WhisperFullParamsWrapper::audio_ctx) - .def_readwrite("tdrz_enable", &WhisperFullParamsWrapper::tdrz_enable) - .def_property("suppress_regex", - [](WhisperFullParamsWrapper &self) { - return py::str(self.suppress_regex ? self.suppress_regex : ""); - }, - [](WhisperFullParamsWrapper &self, const std::string &new_c) { - self.set_suppress_regex(new_c); - }) - .def_property("initial_prompt", - [](WhisperFullParamsWrapper &self) { - return py::str(self.initial_prompt ? self.initial_prompt : ""); - }, - [](WhisperFullParamsWrapper &self, const std::string &initial_prompt) { - self.set_initial_prompt(initial_prompt); - } - ) - .def_property("prompt_tokens", - [](WhisperFullParamsWrapper &self) { - return self.get_prompt_tokens(); - }, - [](WhisperFullParamsWrapper &self, py::object tokens) { - if (tokens.is_none()) { - self.clear_prompt_tokens(); - } else { - self.set_prompt_tokens(tokens.cast>()); - } - }) - .def("set_prompt_tokens", &WhisperFullParamsWrapper::set_prompt_tokens, - py::arg("tokens"), - "Assign prompt tokens from a Python sequence.") - .def("clear_prompt_tokens", &WhisperFullParamsWrapper::clear_prompt_tokens, - "Clear any previously assigned prompt tokens.") - .def("set_new_segment_callback", - [](WhisperFullParamsWrapper &self, py::object callback) { - if (callback.is_none()) { - self.clear_new_segment_callback(); - } else { - self.set_new_segment_callback(callback.cast()); - } - }, - py::arg("callback") = py::none(), - "Assign a new-segment callback.") - .def("clear_new_segment_callback", &WhisperFullParamsWrapper::clear_new_segment_callback, - "Clear any previously assigned new-segment callback.") - .def("set_encoder_begin_callback", - [](WhisperFullParamsWrapper &self, py::object callback) { - if (callback.is_none()) { - self.clear_encoder_begin_callback(); - } else { - self.set_encoder_begin_callback(callback.cast()); - } - }, - py::arg("callback") = py::none(), - "Assign an encoder-begin callback.") - .def("clear_encoder_begin_callback", &WhisperFullParamsWrapper::clear_encoder_begin_callback, - "Clear any previously assigned encoder-begin callback.") - .def("set_abort_callback", - [](WhisperFullParamsWrapper &self, py::object callback) { - if (callback.is_none()) { - self.clear_abort_callback(); - } else { - self.set_abort_callback(callback.cast()); - } - }, - py::arg("callback") = py::none(), - "Assign an abort callback that returns True to stop processing.") - .def("clear_abort_callback", &WhisperFullParamsWrapper::clear_abort_callback, - "Clear any previously assigned abort callback.") - .def_readwrite("prompt_n_tokens", &WhisperFullParamsWrapper::prompt_n_tokens) - .def_readwrite("carry_initial_prompt", &WhisperFullParamsWrapper::carry_initial_prompt) - .def_property("language", - [](WhisperFullParamsWrapper &self) { - return py::str(self.language); - }, - [](WhisperFullParamsWrapper &self, const char *new_c) {// using lang_id let us avoid issues with memory management - const int lang_id = (new_c && strlen(new_c) > 0) ? whisper_lang_id(new_c) : -1; - if (lang_id != -1) { - self.language = whisper_lang_str(lang_id); - } else { - self.language = ""; //defaults to auto-detect - } - }) - .def_readwrite("detect_language", &WhisperFullParamsWrapper::detect_language) - .def_readwrite("suppress_blank", &WhisperFullParamsWrapper::suppress_blank) - .def_readwrite("suppress_nst", &WhisperFullParamsWrapper::suppress_nst) - .def_readwrite("temperature", &WhisperFullParamsWrapper::temperature) - .def_readwrite("max_initial_ts", &WhisperFullParamsWrapper::max_initial_ts) - .def_readwrite("length_penalty", &WhisperFullParamsWrapper::length_penalty) - .def_readwrite("temperature_inc", &WhisperFullParamsWrapper::temperature_inc) - .def_readwrite("entropy_thold", &WhisperFullParamsWrapper::entropy_thold) - .def_readwrite("logprob_thold", &WhisperFullParamsWrapper::logprob_thold) - .def_readwrite("no_speech_thold", &WhisperFullParamsWrapper::no_speech_thold) - // little hack for the internal stuct - .def_property("greedy", [](WhisperFullParamsWrapper &self) {return py::dict("best_of"_a=self.greedy.best_of);}, - [](WhisperFullParamsWrapper &self, py::dict dict) {self.greedy.best_of = dict["best_of"].cast();}) - .def_property("beam_search", [](WhisperFullParamsWrapper &self) {return py::dict("beam_size"_a=self.beam_search.beam_size, "patience"_a=self.beam_search.patience);}, - [](WhisperFullParamsWrapper &self, py::dict dict) {self.beam_search.beam_size = dict["beam_size"].cast(); self.beam_search.patience = dict["patience"].cast();}) - .def_property("new_segment_callback_user_data", - &WhisperFullParamsWrapper::get_new_segment_callback_user_data, - &WhisperFullParamsWrapper::set_new_segment_callback_user_data) - .def_property("progress_callback_user_data", - &WhisperFullParamsWrapper::get_progress_callback_user_data, - &WhisperFullParamsWrapper::set_progress_callback_user_data) - .def_property("encoder_begin_callback_user_data", - &WhisperFullParamsWrapper::get_encoder_begin_callback_user_data, - &WhisperFullParamsWrapper::set_encoder_begin_callback_user_data) - .def_property("abort_callback_user_data", - &WhisperFullParamsWrapper::get_abort_callback_user_data, - &WhisperFullParamsWrapper::set_abort_callback_user_data) - .def_property("logits_filter_callback_user_data", - &WhisperFullParamsWrapper::get_logits_filter_callback_user_data, - &WhisperFullParamsWrapper::set_logits_filter_callback_user_data) - .def("set_logits_filter_callback", - [](WhisperFullParamsWrapper &self, py::object callback) { - if (callback.is_none()) { - self.clear_logits_filter_callback(); - } else { - self.set_logits_filter_callback(callback.cast()); - } - }, - py::arg("callback") = py::none(), - "Assign a logits-filter callback.") - .def("clear_logits_filter_callback", &WhisperFullParamsWrapper::clear_logits_filter_callback, - "Clear any previously assigned logits-filter callback.") - .def_readwrite("vad", &WhisperFullParamsWrapper::vad) - .def_property("vad_model_path", - [](WhisperFullParamsWrapper &self) { - return py::str(self.vad_model_path ? self.vad_model_path : ""); - }, - [](WhisperFullParamsWrapper &self, const std::string &vad_model_path) { - self.set_vad_model_path(vad_model_path); - } - ) - .def_readwrite("vad_params", &WhisperFullParamsWrapper::vad_params); - - - py::implicitly_convertible(); - - m.def("whisper_full_default_params", &whisper_full_default_params_wrapper); - - m.def("whisper_full", &whisper_full_wrapper, "Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text\n" - "Uses the specified decoding strategy to obtain the text.\n"); - - m.def("whisper_full_parallel", &whisper_full_parallel_wrapper, "Split the input audio in chunks and process each chunk separately using whisper_full()\n" - "It seems this approach can offer some speedup in some cases.\n" - "However, the transcription accuracy can be worse at the beginning and end of each chunk."); - - m.def("whisper_full_n_segments", &whisper_full_n_segments_wrapper, "Number of generated text segments.\n" - "A segment can be a few words, a sentence, or even a paragraph.\n"); - - m.def("whisper_full_lang_id", &whisper_full_lang_id_wrapper, "Language id associated with the current context"); - m.def("whisper_full_get_segment_t0", &whisper_full_get_segment_t0_wrapper, "Get the start time of the specified segment"); - m.def("whisper_full_get_segment_t1", &whisper_full_get_segment_t1_wrapper, "Get the end time of the specified segment"); - m.def("whisper_full_get_segment_speaker_turn_next", &whisper_full_get_segment_speaker_turn_next_wrapper, - "Get whether the next segment is predicted as a speaker turn."); - - m.def("whisper_full_get_segment_text", &whisper_full_get_segment_text_wrapper, "Get the text of the specified segment"); - m.def("whisper_full_n_tokens", &whisper_full_n_tokens_wrapper, "Get number of tokens in the specified segment."); - - m.def("whisper_full_get_token_text", &whisper_full_get_token_text_wrapper, "Get the token text of the specified token in the specified segment."); - m.def("whisper_full_get_token_id", &whisper_full_get_token_id_wrapper, "Get the token text of the specified token in the specified segment."); - - m.def("whisper_full_get_token_data", &whisper_full_get_token_data_wrapper, "Get token data for the specified token in the specified segment.\n" - "This contains probabilities, timestamps, etc."); - - m.def("whisper_full_get_token_p", &whisper_full_get_token_p_wrapper, "Get the probability of the specified token in the specified segment."); - - m.def("whisper_ctx_init_openvino_encoder", &whisper_ctx_init_openvino_encoder_wrapper, "Given a context, enable use of OpenVINO for encode inference."); - m.def("whisper_model_type_readable", &whisper_model_type_readable_wrapper, "Return the readable model type string."); - m.def("whisper_model_n_vocab", &whisper_model_n_vocab_wrapper, "Return the model vocabulary size."); - m.def("whisper_model_n_audio_ctx", &whisper_model_n_audio_ctx_wrapper, "Return the audio context size baked into the model."); - m.def("whisper_model_n_audio_state", &whisper_model_n_audio_state_wrapper, "Return the number of audio state units in the model."); - m.def("whisper_model_n_audio_head", &whisper_model_n_audio_head_wrapper, "Return the number of audio attention heads in the model."); - m.def("whisper_model_n_audio_layer", &whisper_model_n_audio_layer_wrapper, "Return the number of audio layers in the model."); - m.def("whisper_model_n_text_ctx", &whisper_model_n_text_ctx_wrapper, "Return the text context size baked into the model."); - m.def("whisper_model_n_text_state", &whisper_model_n_text_state_wrapper, "Return the number of text state units in the model."); - m.def("whisper_model_n_text_head", &whisper_model_n_text_head_wrapper, "Return the number of text attention heads in the model."); - m.def("whisper_model_n_text_layer", &whisper_model_n_text_layer_wrapper, "Return the number of text layers in the model."); - m.def("whisper_model_n_mels", &whisper_model_n_mels_wrapper, "Return the number of mel bins used by the model."); - m.def("whisper_model_ftype", &whisper_model_ftype_wrapper, "Return the model file type identifier."); - - - //////////////////////////////////////////////////////////////////////////// - - m.def("whisper_bench_memcpy", &whisper_bench_memcpy, "Temporary helpers needed for exposing ggml interface"); - m.def("whisper_bench_ggml_mul_mat", &whisper_bench_ggml_mul_mat, "Temporary helpers needed for exposing ggml interface"); - - //////////////////////////////////////////////////////////////////////////// - // Helper mechanism to set callbacks from python - // The only difference from the C-Style API - - m.def("assign_new_segment_callback", - [](whisper_full_params * params, py::object callback) { - assign_new_segment_callback(params, callback); - }, - "Assign a new-segment callback.", - py::arg("params"), py::arg("callback") = py::none()); - - m.def("clear_new_segment_callback", &clear_new_segment_callback, - "Clear any previously assigned new-segment callback.", - py::arg("params")); - - m.def("assign_encoder_begin_callback", - [](whisper_full_params * params, py::object callback) { - assign_encoder_begin_callback(params, callback); - }, - "Assign an encoder-begin callback.", - py::arg("params"), py::arg("callback") = py::none()); - - m.def("clear_encoder_begin_callback", &clear_encoder_begin_callback, - "Clear any previously assigned encoder-begin callback.", - py::arg("params")); - - m.def("assign_logits_filter_callback", - [](whisper_full_params * params, py::object callback) { - assign_logits_filter_callback(params, callback); - }, - "Assign a logits-filter callback.", - py::arg("params"), py::arg("callback") = py::none()); - - m.def("clear_logits_filter_callback", &clear_logits_filter_callback, - "Clear any previously assigned logits-filter callback.", - py::arg("params")); - - m.def("assign_progress_callback", - [](whisper_full_params * params, py::object callback) { - assign_progress_callback(params, callback); - }, - "Assign a progress callback that receives progress updates.", - py::arg("params"), py::arg("callback") = py::none()); - - m.def("clear_progress_callback", &clear_progress_callback, - "Clear any previously assigned progress callback while preserving default progress behavior.", - py::arg("params")); - - m.def("assign_abort_callback", - [](whisper_full_params * params, py::object callback) { - assign_abort_callback(params, callback); - }, - "Assign an abort callback that returns True to stop processing.", - py::arg("params"), py::arg("callback") = py::none()); - - m.def("clear_abort_callback", &clear_abort_callback, "Clear any previously assigned abort callback.", - py::arg("params")); - - m.def("whisper_log_set", - [](py::object callback) { - whisper_log_set_wrapper(callback); - }, - "Assign a Python log callback or None to restore the default logger.", - py::arg("callback") = py::none()); - - // VAD - py::class_(m,"whisper_vad_params") - .def(py::init<>()) - .def_readwrite("threshold", &whisper_vad_params::threshold) - .def_readwrite("min_speech_duration_ms", &whisper_vad_params::min_speech_duration_ms) - .def_readwrite("min_silence_duration_ms", &whisper_vad_params::min_silence_duration_ms) - .def_readwrite("max_speech_duration_s", &whisper_vad_params::max_speech_duration_s) - .def_readwrite("speech_pad_ms", &whisper_vad_params::speech_pad_ms) - .def_readwrite("samples_overlap", &whisper_vad_params::samples_overlap); - - m.def("whisper_vad_default_params", &whisper_vad_default_params); - - py::class_(m,"whisper_vad_context_params") - .def(py::init<>()) - .def_readwrite("n_threads", &whisper_vad_context_params::n_threads) - .def_readwrite("use_gpu", &whisper_vad_context_params::use_gpu) - .def_readwrite("gpu_device", &whisper_vad_context_params::gpu_device); - - m.def("whisper_vad_default_context_params", &whisper_vad_default_context_params); - m.def("whisper_vad_init_from_file_with_params", &whisper_vad_init_from_file_with_params_wrapper); - m.def("whisper_vad_detect_speech", &whisper_vad_detect_speech_wrapper); - m.def("whisper_vad_n_probs", &whisper_vad_n_probs_wrapper); - m.def("whisper_vad_probs", &whisper_vad_probs_wrapper); - py::class_(m, "whisper_vad_segments"); - m.def("whisper_vad_segments_from_probs", &whisper_vad_segments_from_probs_wrapper); - m.def("whisper_vad_segments_from_samples", &whisper_vad_segments_from_samples_wrapper); - m.def("whisper_vad_segments_n_segments", &whisper_vad_segments_n_segments_wrapper); - m.def("whisper_vad_segments_get_segment_t0", &whisper_vad_segments_get_segment_t0_wrapper); - m.def("whisper_vad_segments_get_segment_t1", &whisper_vad_segments_get_segment_t1_wrapper); - m.def("whisper_vad_free_segments", &whisper_vad_free_segments_wrapper); - m.def("whisper_vad_free", &whisper_vad_free_wrapper); - - - - -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif + whisper_bindings(m); } diff --git a/src/whisper_bindings.cpp b/src/whisper_bindings.cpp new file mode 100644 index 0000000..f32a1dd --- /dev/null +++ b/src/whisper_bindings.cpp @@ -0,0 +1,1371 @@ +#include +#include +#include +#include + +#include "whisper.h" + + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +#define DEF_RELEASE_GIL(name, fn, doc) \ + m.def(name, fn, doc, py::call_guard()) + + +namespace py = pybind11; +using namespace pybind11::literals; // to bring in the `_a` literal + +inline bool has_python_user_data(const py::object & obj) { + return obj.ptr() != nullptr && obj.ptr() != Py_None; +} + + +py::object py_log_callback; + + +// whisper context wrapper, to solve the incomplete type issue +// Thanks to https://github.com/pybind/pybind11/issues/2770 +struct whisper_context_wrapper { + whisper_context* ptr; +}; + +// struct inside params +struct greedy{ + int best_of; +}; + +struct beam_search{ + int beam_size; + float patience; +}; + + +struct whisper_model_loader_wrapper { + whisper_model_loader* ptr; + +}; + +struct whisper_context_wrapper whisper_init_from_file_with_params_wrapper( + const char * path_model, + struct whisper_context_params cparams){ + struct whisper_context * ctx = whisper_init_from_file_with_params(path_model, cparams); + struct whisper_context_wrapper ctw_w; + ctw_w.ptr = ctx; + return ctw_w; +} + +struct whisper_context_wrapper whisper_init_from_buffer_with_params_wrapper( + void * buffer, + size_t buffer_size, + struct whisper_context_params cparams){ + struct whisper_context * ctx = whisper_init_from_buffer_with_params(buffer, buffer_size, cparams); + struct whisper_context_wrapper ctw_w; + ctw_w.ptr = ctx; + return ctw_w; +} + +struct whisper_context_wrapper whisper_init_with_params_wrapper( + struct whisper_model_loader_wrapper * loader, + struct whisper_context_params cparams){ + struct whisper_context * ctx = whisper_init_with_params(loader->ptr, cparams); + struct whisper_context_wrapper ctw_w; + ctw_w.ptr = ctx; + return ctw_w; +}; + +void whisper_free_wrapper(struct whisper_context_wrapper * ctx_w){ + whisper_free(ctx_w->ptr); +}; + +int whisper_pcm_to_mel_wrapper( + struct whisper_context_wrapper * ctx, + py::array_t samples, + int n_samples, + int n_threads){ + py::buffer_info buf = samples.request(); + float *samples_ptr = static_cast(buf.ptr); + return whisper_pcm_to_mel(ctx->ptr, samples_ptr, n_samples, n_threads); +}; + +int whisper_set_mel_wrapper( + struct whisper_context_wrapper * ctx, + py::array_t data, + int n_len, + int n_mel){ + py::buffer_info buf = data.request(); + float *data_ptr = static_cast(buf.ptr); + return whisper_set_mel(ctx->ptr, data_ptr, n_len, n_mel); + +}; + +int whisper_n_len_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_n_len(ctx_w->ptr); +}; + +int whisper_n_vocab_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_n_vocab(ctx_w->ptr); +}; + +int whisper_n_text_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_n_text_ctx(ctx_w->ptr); +}; + +int whisper_n_audio_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_n_audio_ctx(ctx_w->ptr); +} + +int whisper_is_multilingual_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_is_multilingual(ctx_w->ptr); +} + + +float * whisper_get_logits_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_get_logits(ctx_w->ptr); +}; + +const char * whisper_token_to_str_wrapper(struct whisper_context_wrapper * ctx_w, whisper_token token){ + return whisper_token_to_str(ctx_w->ptr, token); +}; + +py::bytes whisper_token_to_bytes_wrapper(struct whisper_context_wrapper * ctx_w, whisper_token token){ + const char* str = whisper_token_to_str(ctx_w->ptr, token); + size_t l = strlen(str); + return py::bytes(str, l); +} + +whisper_token whisper_token_eot_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_eot(ctx_w->ptr); +} + +whisper_token whisper_token_sot_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_sot(ctx_w->ptr); +} + +whisper_token whisper_token_prev_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_prev(ctx_w->ptr); +} + +whisper_token whisper_token_solm_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_solm(ctx_w->ptr); +} + +whisper_token whisper_token_not_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_not(ctx_w->ptr); +} + +whisper_token whisper_token_beg_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_beg(ctx_w->ptr); +} + +whisper_token whisper_token_lang_wrapper(struct whisper_context_wrapper * ctx_w, int lang_id){ + return whisper_token_lang(ctx_w->ptr, lang_id); +} + +whisper_token whisper_token_translate_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_translate(ctx_w->ptr); +} + +whisper_token whisper_token_transcribe_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_token_transcribe(ctx_w->ptr); +} + +void whisper_print_timings_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_print_timings(ctx_w->ptr); +} + +void whisper_reset_timings_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_reset_timings(ctx_w->ptr); +} + +int whisper_encode_wrapper( + struct whisper_context_wrapper * ctx, + int offset, + int n_threads){ + return whisper_encode(ctx->ptr, offset, n_threads); +} + + +int whisper_decode_wrapper( + struct whisper_context_wrapper * ctx, + const whisper_token * tokens, + int n_tokens, + int n_past, + int n_threads){ + return whisper_decode(ctx->ptr, tokens, n_tokens, n_past, n_threads); +}; + +int whisper_tokenize_wrapper( + struct whisper_context_wrapper * ctx, + const char * text, + whisper_token * tokens, + int n_max_tokens){ + return whisper_tokenize(ctx->ptr, text, tokens, n_max_tokens); +}; + +int whisper_lang_auto_detect_wrapper( + struct whisper_context_wrapper * ctx, + int offset_ms, + int n_threads, + py::array_t lang_probs){ + + py::buffer_info buf = lang_probs.request(); + float *lang_probs_ptr = static_cast(buf.ptr); + return whisper_lang_auto_detect(ctx->ptr, offset_ms, n_threads, lang_probs_ptr); + +} + +int whisper_full_wrapper( + struct whisper_context_wrapper * ctx_w, + struct whisper_full_params params, + py::array_t samples, + int n_samples){ + py::buffer_info buf = samples.request(); + float *samples_ptr = static_cast(buf.ptr); + + py::gil_scoped_release release; + return whisper_full(ctx_w->ptr, params, samples_ptr, n_samples); +} + +int whisper_full_parallel_wrapper( + struct whisper_context_wrapper * ctx_w, + struct whisper_full_params params, + py::array_t samples, + int n_samples, + int n_processors){ + py::buffer_info buf = samples.request(); + float *samples_ptr = static_cast(buf.ptr); + + py::gil_scoped_release release; + return whisper_full_parallel(ctx_w->ptr, params, samples_ptr, n_samples, n_processors); +} + + +int whisper_full_n_segments_wrapper(struct whisper_context_wrapper * ctx){ + py::gil_scoped_release release; + return whisper_full_n_segments(ctx->ptr); +} + +int whisper_full_lang_id_wrapper(struct whisper_context_wrapper * ctx){ + return whisper_full_lang_id(ctx->ptr); +} + +int64_t whisper_full_get_segment_t0_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ + return whisper_full_get_segment_t0(ctx->ptr, i_segment); +} + +int64_t whisper_full_get_segment_t1_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ + return whisper_full_get_segment_t1(ctx->ptr, i_segment); +} + +// https://pybind11.readthedocs.io/en/stable/advanced/cast/strings.html +const py::bytes whisper_full_get_segment_text_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ + const char * c_array = whisper_full_get_segment_text(ctx->ptr, i_segment); + size_t length = strlen(c_array); // Determine the length of the array + return py::bytes(c_array, length); // Return the data without transcoding +}; + +int whisper_full_n_tokens_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ + return whisper_full_n_tokens(ctx->ptr, i_segment); +} + +const char * whisper_full_get_token_text_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ + return whisper_full_get_token_text(ctx->ptr, i_segment, i_token); +} + +whisper_token whisper_full_get_token_id_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ + return whisper_full_get_token_id(ctx->ptr, i_segment, i_token); +} + +whisper_token_data whisper_full_get_token_data_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ + return whisper_full_get_token_data(ctx->ptr, i_segment, i_token); +} + +float whisper_full_get_token_p_wrapper(struct whisper_context_wrapper * ctx, int i_segment, int i_token){ + return whisper_full_get_token_p(ctx->ptr, i_segment, i_token); +} + +bool whisper_full_get_segment_speaker_turn_next_wrapper(struct whisper_context_wrapper * ctx, int i_segment){ + return whisper_full_get_segment_speaker_turn_next(ctx->ptr, i_segment); +} + +const char * whisper_model_type_readable_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_type_readable(ctx_w->ptr); +} + +int whisper_model_n_vocab_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_vocab(ctx_w->ptr); +} + +int whisper_model_n_audio_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_audio_ctx(ctx_w->ptr); +} + +int whisper_model_n_audio_state_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_audio_state(ctx_w->ptr); +} + +int whisper_model_n_audio_head_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_audio_head(ctx_w->ptr); +} + +int whisper_model_n_audio_layer_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_audio_layer(ctx_w->ptr); +} + +int whisper_model_n_text_ctx_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_text_ctx(ctx_w->ptr); +} + +int whisper_model_n_text_state_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_text_state(ctx_w->ptr); +} + +int whisper_model_n_text_head_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_text_head(ctx_w->ptr); +} + +int whisper_model_n_text_layer_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_text_layer(ctx_w->ptr); +} + +int whisper_model_n_mels_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_n_mels(ctx_w->ptr); +} + +int whisper_model_ftype_wrapper(struct whisper_context_wrapper * ctx_w){ + return whisper_model_ftype(ctx_w->ptr); +} + +bool _abort_callback(void * user_data); +void _new_segment_callback(struct whisper_context * ctx, struct whisper_state * state, int n_new, void * user_data); +bool _encoder_begin_callback(struct whisper_context * ctx, struct whisper_state * state, void * user_data); +void _logits_filter_callback( + struct whisper_context * ctx, + struct whisper_state * state, + const whisper_token_data * tokens, + int n_tokens, + float * logits, + void * user_data); + +int whisper_ctx_init_openvino_encoder_wrapper(struct whisper_context_wrapper * ctx, const char * model_path, + const char * device, + const char * cache_dir){ + return whisper_ctx_init_openvino_encoder(ctx->ptr, model_path, device, cache_dir); +} + +struct WhisperFullParamsWrapper : public whisper_full_params { + std::string initial_prompt_str; + std::string suppress_regex_str; + std::string vad_model_path_str; + std::vector prompt_token_storage; + + void reset_progress_callback() { + progress_callback_user_data = this; + progress_callback = [](struct whisper_context* ctx, struct whisper_state* state, int progress, void* user_data) { + (void) ctx; + (void) state; + auto* self = static_cast(user_data); + if (self && self->print_progress) { + if (self->py_progress_callback) { + py::gil_scoped_acquire gil; + if (!has_python_user_data(self->py_progress_callback_user_data)) { + self->py_progress_callback(progress); + } else { + self->py_progress_callback(progress, self->py_progress_callback_user_data); + } + } else { + fprintf(stderr, "Progress: %3d%%\n", progress); + } + } + }; + } + + void sync_prompt_tokens() { + prompt_tokens = prompt_token_storage.empty() ? nullptr : prompt_token_storage.data(); + prompt_n_tokens = prompt_token_storage.size(); + } +public: + py::function py_new_segment_callback; + py::object py_new_segment_callback_user_data; + py::function py_encoder_begin_callback; + py::object py_encoder_begin_callback_user_data; + py::function py_progress_callback; + py::object py_progress_callback_user_data; + py::function py_logits_filter_callback; + py::object py_logits_filter_callback_user_data; + py::function py_abort_callback; + py::object py_abort_callback_user_data; + WhisperFullParamsWrapper(const whisper_full_params& params = whisper_full_params()) + : whisper_full_params(params), + initial_prompt_str(params.initial_prompt ? params.initial_prompt : ""), + suppress_regex_str(params.suppress_regex ? params.suppress_regex : ""), + vad_model_path_str(params.vad_model_path ? params.vad_model_path : ""), + prompt_token_storage(), + py_new_segment_callback_user_data(py::none()), + py_encoder_begin_callback_user_data(py::none()), + py_progress_callback_user_data(py::none()), + py_logits_filter_callback_user_data(py::none()), + py_abort_callback(), + py_abort_callback_user_data(py::none()) + { + initial_prompt = initial_prompt_str.empty() ? nullptr : initial_prompt_str.c_str(); + suppress_regex = suppress_regex_str.empty() ? nullptr : suppress_regex_str.c_str(); + vad_model_path = vad_model_path_str.empty() ? nullptr : vad_model_path_str.c_str(); + new_segment_callback_user_data = this; + encoder_begin_callback_user_data = this; + abort_callback_user_data = this; + logits_filter_callback_user_data = this; + if (params.prompt_tokens && params.prompt_n_tokens > 0) { + prompt_token_storage.assign(params.prompt_tokens, params.prompt_tokens + params.prompt_n_tokens); + } + sync_prompt_tokens(); + reset_progress_callback(); + } + WhisperFullParamsWrapper(const WhisperFullParamsWrapper& other) + : whisper_full_params(static_cast(other)), // Copy base struct + initial_prompt_str(other.initial_prompt_str), + suppress_regex_str(other.suppress_regex_str), + vad_model_path_str(other.vad_model_path_str), + prompt_token_storage(other.prompt_token_storage), + py_new_segment_callback(other.py_new_segment_callback), + py_new_segment_callback_user_data(other.py_new_segment_callback_user_data), + py_encoder_begin_callback(other.py_encoder_begin_callback), + py_encoder_begin_callback_user_data(other.py_encoder_begin_callback_user_data), + py_progress_callback(other.py_progress_callback), + py_progress_callback_user_data(other.py_progress_callback_user_data), + py_logits_filter_callback(other.py_logits_filter_callback), + py_logits_filter_callback_user_data(other.py_logits_filter_callback_user_data), + py_abort_callback(other.py_abort_callback), + py_abort_callback_user_data(other.py_abort_callback_user_data) { + // Reset pointers to new string copies + initial_prompt = initial_prompt_str.empty() ? nullptr : initial_prompt_str.c_str(); + suppress_regex = suppress_regex_str.empty() ? nullptr : suppress_regex_str.c_str(); + vad_model_path = vad_model_path_str.empty() ? nullptr : vad_model_path_str.c_str(); + new_segment_callback_user_data = this; + encoder_begin_callback_user_data = this; + abort_callback_user_data = this; + logits_filter_callback_user_data = this; + sync_prompt_tokens(); + reset_progress_callback(); + } + void set_initial_prompt(const std::string& prompt) { + initial_prompt_str = prompt; + initial_prompt = initial_prompt_str.c_str(); + } + void set_suppress_regex(const std::string& regex) { + suppress_regex_str = regex; + suppress_regex = suppress_regex_str.c_str(); + } + void set_vad_model_path(const std::string& model_path) { + vad_model_path_str = model_path; + vad_model_path = vad_model_path_str.c_str(); + } + py::tuple get_prompt_tokens() const { + py::tuple tokens(prompt_token_storage.size()); + for (size_t index = 0; index < prompt_token_storage.size(); ++index) { + tokens[index] = prompt_token_storage[index]; + } + return tokens; + } + void set_prompt_tokens(const std::vector& tokens) { + prompt_token_storage = tokens; + sync_prompt_tokens(); + } + void clear_prompt_tokens() { + prompt_token_storage.clear(); + sync_prompt_tokens(); + } + py::object get_new_segment_callback_user_data() const { + return py_new_segment_callback_user_data; + } + void set_new_segment_callback_user_data(py::object user_data) { + py_new_segment_callback_user_data = std::move(user_data); + new_segment_callback_user_data = this; + } + void set_new_segment_callback(py::function callback) { + py_new_segment_callback = std::move(callback); + new_segment_callback_user_data = this; + new_segment_callback = _new_segment_callback; + } + void clear_new_segment_callback() { + py_new_segment_callback = py::function(); + new_segment_callback = nullptr; + new_segment_callback_user_data = this; + } + py::object get_encoder_begin_callback_user_data() const { + return py_encoder_begin_callback_user_data; + } + void set_encoder_begin_callback_user_data(py::object user_data) { + py_encoder_begin_callback_user_data = std::move(user_data); + encoder_begin_callback_user_data = this; + } + void set_encoder_begin_callback(py::function callback) { + py_encoder_begin_callback = std::move(callback); + encoder_begin_callback_user_data = this; + encoder_begin_callback = _encoder_begin_callback; + } + void clear_encoder_begin_callback() { + py_encoder_begin_callback = py::function(); + encoder_begin_callback = nullptr; + encoder_begin_callback_user_data = this; + } + py::object get_progress_callback_user_data() const { + return py_progress_callback_user_data; + } + void set_progress_callback_user_data(py::object user_data) { + py_progress_callback_user_data = std::move(user_data); + progress_callback_user_data = this; + } + void set_progress_callback(py::function callback) { + py_progress_callback = std::move(callback); + reset_progress_callback(); + } + void clear_progress_callback() { + py_progress_callback = py::function(); + reset_progress_callback(); + } + py::object get_logits_filter_callback_user_data() const { + return py_logits_filter_callback_user_data; + } + void set_logits_filter_callback_user_data(py::object user_data) { + py_logits_filter_callback_user_data = std::move(user_data); + logits_filter_callback_user_data = this; + } + void set_logits_filter_callback(py::function callback) { + py_logits_filter_callback = std::move(callback); + logits_filter_callback_user_data = this; + logits_filter_callback = _logits_filter_callback; + } + void clear_logits_filter_callback() { + py_logits_filter_callback = py::function(); + logits_filter_callback = nullptr; + logits_filter_callback_user_data = this; + } + py::object get_abort_callback_user_data() const { + return py_abort_callback_user_data; + } + void set_abort_callback_user_data(py::object user_data) { + py_abort_callback_user_data = std::move(user_data); + abort_callback_user_data = this; + } + void set_abort_callback(py::function callback) { + py_abort_callback = std::move(callback); + abort_callback_user_data = this; + abort_callback = _abort_callback; + } + void clear_abort_callback() { + py_abort_callback = py::function(); + abort_callback = nullptr; + abort_callback_user_data = this; + } +}; +WhisperFullParamsWrapper whisper_full_default_params_wrapper(enum whisper_sampling_strategy strategy) { + return WhisperFullParamsWrapper(whisper_full_default_params(strategy)); +} + +// callbacks mechanism + +void _new_segment_callback(struct whisper_context * ctx, struct whisper_state * state, int n_new, void * user_data){ + (void) state; + struct whisper_context_wrapper ctx_w; + ctx_w.ptr = ctx; + auto * params = static_cast(user_data); + if (!params || !params->py_new_segment_callback) { + return; + } + + py::gil_scoped_acquire gil; + py::function callback = params->py_new_segment_callback; + if (!has_python_user_data(params->py_new_segment_callback_user_data)) { + callback(ctx_w, n_new); + } else { + callback(ctx_w, n_new, params->py_new_segment_callback_user_data); + } +}; + +void assign_new_segment_callback(struct whisper_full_params *params_base, py::object callback){ + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_new_segment_callback(); + return; + } + + params->set_new_segment_callback(callback.cast()); +} + +void clear_new_segment_callback(struct whisper_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_new_segment_callback(); +}; + +bool _encoder_begin_callback(struct whisper_context * ctx, struct whisper_state * state, void * user_data){ + (void) state; + struct whisper_context_wrapper ctx_w; + ctx_w.ptr = ctx; + auto * params = static_cast(user_data); + if (!params || !params->py_encoder_begin_callback) { + return false; + } + + py::gil_scoped_acquire gil; + py::function callback = params->py_encoder_begin_callback; + py::object result_py; + if (!has_python_user_data(params->py_encoder_begin_callback_user_data)) { + result_py = callback(ctx_w); + } else { + result_py = callback(ctx_w, params->py_encoder_begin_callback_user_data); + } + bool res = result_py.cast(); + return res; +} + +void assign_encoder_begin_callback(struct whisper_full_params *params_base, py::object callback){ + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_encoder_begin_callback(); + return; + } + + params->set_encoder_begin_callback(callback.cast()); +} + +void clear_encoder_begin_callback(struct whisper_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_encoder_begin_callback(); +} + +void _logits_filter_callback( + struct whisper_context * ctx, + struct whisper_state * state, + const whisper_token_data * tokens, + int n_tokens, + float * logits, + void * user_data){ + (void) state; + (void) tokens; + struct whisper_context_wrapper ctx_w; + ctx_w.ptr = ctx; + auto * params = static_cast(user_data); + if (!params || !params->py_logits_filter_callback) { + return; + } + + py::gil_scoped_acquire gil; + py::function callback = params->py_logits_filter_callback; + if (!has_python_user_data(params->py_logits_filter_callback_user_data)) { + callback(ctx_w, n_tokens, logits); + } else { + callback(ctx_w, n_tokens, logits, params->py_logits_filter_callback_user_data); + } +} + +void assign_logits_filter_callback(struct whisper_full_params *params_base, py::object callback){ + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_logits_filter_callback(); + return; + } + + params->set_logits_filter_callback(callback.cast()); +} + +void clear_logits_filter_callback(struct whisper_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_logits_filter_callback(); +} + +void assign_progress_callback(whisper_full_params *params_base, py::object callback) { + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_progress_callback(); + return; + } + + params->set_progress_callback(callback.cast()); +} + +void clear_progress_callback(whisper_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_progress_callback(); +} + +bool _abort_callback(void * user_data) { + auto * params = static_cast(user_data); + if (!params || !params->py_abort_callback) { + return false; + } + + py::gil_scoped_acquire gil; + py::function callback = params->py_abort_callback; + py::object result_py; + if (!has_python_user_data(params->py_abort_callback_user_data)) { + result_py = callback(); + } else { + result_py = callback(params->py_abort_callback_user_data); + } + return result_py.cast(); +} + +void assign_abort_callback(whisper_full_params *params_base, py::object callback){ + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_abort_callback(); + return; + } + + params->set_abort_callback(callback.cast()); +} + +void clear_abort_callback(whisper_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_abort_callback(); +} + +void whisper_log_set_wrapper(py::object callback) { + if (callback.is_none()) { + py_log_callback = py::none(); + whisper_log_set(nullptr, nullptr); + return; + } + + py_log_callback = callback.cast(); + whisper_log_set( + [](enum ggml_log_level level, const char * text, void * user_data) { + (void) user_data; + py::gil_scoped_acquire gil; + py::function log_callback = py_log_callback.cast(); + log_callback(py::int_(static_cast(level)), py::str(text ? text : "")); + }, + nullptr); +} + +py::dict get_greedy(whisper_full_params * params){ + py::dict d("best_of"_a=params->greedy.best_of); + return d; +} + + +// Voice Activity Detection (VAD) +struct whisper_vad_context_wrapper { + whisper_vad_context* ptr; +}; + +struct whisper_vad_context_wrapper whisper_vad_init_from_file_with_params_wrapper(const char * path_model, struct whisper_vad_context_params params){ + struct whisper_vad_context * ctx = whisper_vad_init_from_file_with_params(path_model, params); + struct whisper_vad_context_wrapper ctw_w; + ctw_w.ptr = ctx; + return ctw_w; +} + +bool whisper_vad_detect_speech_wrapper( + struct whisper_vad_context_wrapper * ctx, + py::array_t samples, + int n_samples){ + py::buffer_info buf = samples.request(); + float *samples_ptr = static_cast(buf.ptr); + + py::gil_scoped_release release; + return whisper_vad_detect_speech(ctx->ptr, samples_ptr, n_samples); +} + +int whisper_vad_n_probs_wrapper(struct whisper_vad_context_wrapper * ctx){ + return whisper_vad_n_probs(ctx->ptr); +} + +py::array_t whisper_vad_probs_wrapper(struct whisper_vad_context_wrapper * ctx) { + float * probs_ptr = whisper_vad_probs(ctx->ptr); + int n_probs = whisper_vad_n_probs(ctx->ptr); + + if (probs_ptr == nullptr || n_probs <= 0) { + return py::array_t(0); + } + return py::array_t( + {n_probs}, + {sizeof(float)}, + probs_ptr + ); +} + +struct whisper_vad_segments_wrapper { + struct whisper_vad_segments * ptr; +}; + +struct whisper_vad_segments_wrapper whisper_vad_segments_from_probs_wrapper( + struct whisper_vad_context_wrapper * vctx_w, + struct whisper_vad_params params + ){ + struct whisper_vad_segments * wvs = whisper_vad_segments_from_probs(vctx_w->ptr, params); + struct whisper_vad_segments_wrapper wvs_w; + wvs_w.ptr = wvs; + return wvs_w; +} + +struct whisper_vad_segments_wrapper whisper_vad_segments_from_samples_wrapper( + struct whisper_vad_context_wrapper * vctx_w, + struct whisper_vad_params params, + py::array_t samples, + int n_samples){ + + py::buffer_info buf = samples.request(); + float *samples_ptr = static_cast(buf.ptr); + + struct whisper_vad_segments * wvs = whisper_vad_segments_from_samples(vctx_w->ptr, params, samples_ptr, n_samples); + struct whisper_vad_segments_wrapper wvs_w; + wvs_w.ptr = wvs; + return wvs_w; +} + +int whisper_vad_segments_n_segments_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper){ + return whisper_vad_segments_n_segments(segments_wrapper->ptr); +} + +float whisper_vad_segments_get_segment_t0_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper, int i_segment) { + return whisper_vad_segments_get_segment_t0(segments_wrapper->ptr, i_segment); +} + +float whisper_vad_segments_get_segment_t1_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper, int i_segment) { + return whisper_vad_segments_get_segment_t1(segments_wrapper->ptr, i_segment); +} + +void whisper_vad_free_segments_wrapper(struct whisper_vad_segments_wrapper * segments_wrapper){ + return whisper_vad_free_segments(segments_wrapper->ptr); +} + +void whisper_vad_free_wrapper(struct whisper_vad_context_wrapper * ctx_w){ + return whisper_vad_free(ctx_w->ptr); +} + +//////////// + +void whisper_bindings(py::module_ & m) { + + m.attr("WHISPER_SAMPLE_RATE") = WHISPER_SAMPLE_RATE; + m.attr("WHISPER_N_FFT") = WHISPER_N_FFT; + m.attr("WHISPER_HOP_LENGTH") = WHISPER_HOP_LENGTH; + m.attr("WHISPER_CHUNK_SIZE") = WHISPER_CHUNK_SIZE; + + py::enum_(m, "whisper_alignment_heads_preset") + .value("WHISPER_AHEADS_NONE", whisper_alignment_heads_preset::WHISPER_AHEADS_NONE) + .value("WHISPER_AHEADS_N_TOP_MOST", whisper_alignment_heads_preset::WHISPER_AHEADS_N_TOP_MOST) + .value("WHISPER_AHEADS_CUSTOM", whisper_alignment_heads_preset::WHISPER_AHEADS_CUSTOM) + .value("WHISPER_AHEADS_TINY_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_TINY_EN) + .value("WHISPER_AHEADS_TINY", whisper_alignment_heads_preset::WHISPER_AHEADS_TINY) + .value("WHISPER_AHEADS_BASE_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_BASE_EN) + .value("WHISPER_AHEADS_BASE", whisper_alignment_heads_preset::WHISPER_AHEADS_BASE) + .value("WHISPER_AHEADS_SMALL_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_SMALL_EN) + .value("WHISPER_AHEADS_SMALL", whisper_alignment_heads_preset::WHISPER_AHEADS_SMALL) + .value("WHISPER_AHEADS_MEDIUM_EN", whisper_alignment_heads_preset::WHISPER_AHEADS_MEDIUM_EN) + .value("WHISPER_AHEADS_MEDIUM", whisper_alignment_heads_preset::WHISPER_AHEADS_MEDIUM) + .value("WHISPER_AHEADS_LARGE_V1", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V1) + .value("WHISPER_AHEADS_LARGE_V2", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V2) + .value("WHISPER_AHEADS_LARGE_V3", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V3) + .value("WHISPER_AHEADS_LARGE_V3_TURBO", whisper_alignment_heads_preset::WHISPER_AHEADS_LARGE_V3_TURBO) + .export_values(); + + py::class_(m, "whisper_context"); + py::class_(m, "whisper_context_params") + .def(py::init<>()) + .def_readwrite("use_gpu", &whisper_context_params::use_gpu) + .def_readwrite("flash_attn", &whisper_context_params::flash_attn) + .def_readwrite("gpu_device", &whisper_context_params::gpu_device) + .def_readwrite("dtw_token_timestamps", &whisper_context_params::dtw_token_timestamps) + .def_readwrite("dtw_aheads_preset", &whisper_context_params::dtw_aheads_preset) + .def_readwrite("dtw_n_top", &whisper_context_params::dtw_n_top) + .def_readwrite("dtw_mem_size", &whisper_context_params::dtw_mem_size); + py::class_(m, "whisper_token") + .def(py::init<>()); + py::class_(m,"whisper_token_data") + .def(py::init<>()) + .def_readwrite("id", &whisper_token_data::id) + .def_readwrite("tid", &whisper_token_data::tid) + .def_readwrite("p", &whisper_token_data::p) + .def_readwrite("plog", &whisper_token_data::plog) + .def_readwrite("pt", &whisper_token_data::pt) + .def_readwrite("ptsum", &whisper_token_data::ptsum) + .def_readwrite("t0", &whisper_token_data::t0) + .def_readwrite("t1", &whisper_token_data::t1) + .def_readwrite("t_dtw", &whisper_token_data::t_dtw) + .def_readwrite("vlen", &whisper_token_data::vlen); + + py::class_(m,"whisper_model_loader") + .def(py::init<>()); + + m.def("whisper_context_default_params", &whisper_context_default_params, + "Return the default context parameters used during model initialization."); + DEF_RELEASE_GIL("whisper_init_from_file_with_params", &whisper_init_from_file_with_params_wrapper, "Various functions for loading a ggml whisper model.\n" + "Allocate (almost) all memory needed for the model.\n" + "Return NULL on failure"); + DEF_RELEASE_GIL("whisper_init_from_buffer_with_params", &whisper_init_from_buffer_with_params_wrapper, "Various functions for loading a ggml whisper model.\n" + "Allocate (almost) all memory needed for the model.\n" + "Return NULL on failure"); + DEF_RELEASE_GIL("whisper_init_with_params", &whisper_init_with_params_wrapper, "Various functions for loading a ggml whisper model.\n" + "Allocate (almost) all memory needed for the model.\n" + "Return NULL on failure"); + + + m.def("whisper_free", &whisper_free_wrapper, "Frees all memory allocated by the model."); + + m.def("whisper_pcm_to_mel", &whisper_pcm_to_mel_wrapper, "Convert RAW PCM audio to log mel spectrogram.\n" + "The resulting spectrogram is stored inside the provided whisper context.\n" + "Returns 0 on success"); + + m.def("whisper_set_mel", &whisper_set_mel_wrapper, " This can be used to set a custom log mel spectrogram inside the provided whisper context.\n" + "Use this instead of whisper_pcm_to_mel() if you want to provide your own log mel spectrogram.\n" + "n_mel must be 80\n" + "Returns 0 on success"); + + m.def("whisper_encode", &whisper_encode_wrapper, "Run the Whisper encoder on the log mel spectrogram stored inside the provided whisper context.\n" + "Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first.\n" + "offset can be used to specify the offset of the first frame in the spectrogram.\n" + "Returns 0 on success"); + + m.def("whisper_decode", &whisper_decode_wrapper, "Run the Whisper decoder to obtain the logits and probabilities for the next token.\n" + "Make sure to call whisper_encode() first.\n" + "tokens + n_tokens is the provided context for the decoder.\n" + "n_past is the number of tokens to use from previous decoder calls.\n" + "Returns 0 on success\n" + "TODO: add support for multiple decoders"); + + m.def("whisper_tokenize", &whisper_tokenize_wrapper, "Convert the provided text into tokens.\n" + "The tokens pointer must be large enough to hold the resulting tokens.\n" + "Returns the number of tokens on success, no more than n_max_tokens\n" + "Returns -1 on failure\n" + "TODO: not sure if correct"); + + m.def("whisper_lang_max_id", &whisper_lang_max_id, "Largest language id (i.e. number of available languages - 1)"); + m.def("whisper_lang_id", &whisper_lang_id, "Return the id of the specified language, returns -1 if not found\n" + "Examples:\n" + "\"de\" -> 2\n" + "\"german\" -> 2"); + m.def("whisper_lang_str", &whisper_lang_str, "Return the short string of the specified language id (e.g. 2 -> \"de\"), returns nullptr if not found"); + + + + + + + + m.def("whisper_lang_auto_detect", &whisper_lang_auto_detect_wrapper, "Use mel data at offset_ms to try and auto-detect the spoken language\n" + "Make sure to call whisper_pcm_to_mel() or whisper_set_mel() first\n" + "Returns the top language id or negative on failure\n" + "If not null, fills the lang_probs array with the probabilities of all languages\n" + "The array must be whispe_lang_max_id() + 1 in size\n" + "ref: https://github.com/openai/whisper/blob/main/whisper/decoding.py#L18-L69\n"); + m.def("whisper_n_len", &whisper_n_len_wrapper, "whisper_n_len"); + m.def("whisper_n_vocab", &whisper_n_vocab_wrapper, "wrapper_whisper_n_vocab"); + m.def("whisper_n_text_ctx", &whisper_n_text_ctx_wrapper, "whisper_n_text_ctx"); + m.def("whisper_n_audio_ctx", &whisper_n_audio_ctx_wrapper, "whisper_n_audio_ctx"); + m.def("whisper_is_multilingual", &whisper_is_multilingual_wrapper, "whisper_is_multilingual"); + m.def("whisper_get_logits", &whisper_get_logits_wrapper, "Token logits obtained from the last call to whisper_decode()\n" + "The logits for the last token are stored in the last row\n" + "Rows: n_tokens\n" + "Cols: n_vocab"); + + + m.def("whisper_token_to_str", &whisper_token_to_str_wrapper, "whisper_token_to_str"); + m.def("whisper_token_to_bytes", &whisper_token_to_bytes_wrapper, "whisper_token_to_bytes"); + m.def("whisper_token_eot", &whisper_token_eot_wrapper, "whisper_token_eot"); + m.def("whisper_token_sot", &whisper_token_sot_wrapper, "whisper_token_sot"); + m.def("whisper_token_prev", &whisper_token_prev_wrapper); + m.def("whisper_token_solm", &whisper_token_solm_wrapper); + m.def("whisper_token_not", &whisper_token_not_wrapper); + m.def("whisper_token_beg", &whisper_token_beg_wrapper); + m.def("whisper_token_lang", &whisper_token_lang_wrapper); + + m.def("whisper_token_translate", &whisper_token_translate_wrapper); + m.def("whisper_token_transcribe", &whisper_token_transcribe_wrapper); + + m.def("whisper_print_timings", &whisper_print_timings_wrapper); + m.def("whisper_reset_timings", &whisper_reset_timings_wrapper); + + m.def("whisper_print_system_info", &whisper_print_system_info); + + + + ////////////////////// + + py::enum_(m, "whisper_sampling_strategy") + .value("WHISPER_SAMPLING_GREEDY", whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY) + .value("WHISPER_SAMPLING_BEAM_SEARCH", whisper_sampling_strategy::WHISPER_SAMPLING_BEAM_SEARCH) + .export_values(); + + py::class_(m, "__whisper_full_params__internal") + .def(py::init<>()) + .def("__repr__", [](const whisper_full_params& self) { + std::ostringstream oss; + oss << "whisper_full_params(" + << "strategy=" << self.strategy << ", " + << "n_threads=" << self.n_threads << ", " + << "n_max_text_ctx=" << self.n_max_text_ctx << ", " + << "offset_ms=" << self.offset_ms << ", " + << "duration_ms=" << self.duration_ms << ", " + << "translate=" << (self.translate ? "True" : "False") << ", " + << "no_context=" << (self.no_context ? "True" : "False") << ", " + << "no_timestamps=" << (self.no_timestamps ? "True" : "False") << ", " + << "single_segment=" << (self.single_segment ? "True" : "False") << ", " + << "print_special=" << (self.print_special ? "True" : "False") << ", " + << "print_progress=" << (self.print_progress ? "True" : "False") << ", " + << "print_realtime=" << (self.print_realtime ? "True" : "False") << ", " + << "print_timestamps=" << (self.print_timestamps ? "True" : "False") << ", " + << "token_timestamps=" << (self.token_timestamps ? "True" : "False") << ", " + << "thold_pt=" << self.thold_pt << ", " + << "thold_ptsum=" << self.thold_ptsum << ", " + << "max_len=" << self.max_len << ", " + << "split_on_word=" << (self.split_on_word ? "True" : "False") << ", " + << "max_tokens=" << self.max_tokens << ", " + << "debug_mode=" << (self.debug_mode ? "True" : "False") << ", " + << "audio_ctx=" << self.audio_ctx << ", " + << "tdrz_enable=" << (self.tdrz_enable ? "True" : "False") << ", " + << "suppress_regex=" << (self.suppress_regex ? self.suppress_regex : "None") << ", " + << "initial_prompt=" << (self.initial_prompt ? self.initial_prompt : "None") << ", " + << "prompt_tokens=" << (self.prompt_tokens ? "(whisper_token *)" : "None") << ", " + << "prompt_n_tokens=" << self.prompt_n_tokens << ", " + << "language=" << (self.language ? self.language : "None") << ", " + << "detect_language=" << (self.detect_language ? "True" : "False") << ", " + << "suppress_blank=" << (self.suppress_blank ? "True" : "False") << ", " + << "temperature=" << self.temperature << ", " + << "max_initial_ts=" << self.max_initial_ts << ", " + << "length_penalty=" << self.length_penalty << ", " + << "temperature_inc=" << self.temperature_inc << ", " + << "entropy_thold=" << self.entropy_thold << ", " + << "logprob_thold=" << self.logprob_thold << ", " + << "no_speech_thold=" << self.no_speech_thold << ", " + << "greedy={best_of=" << self.greedy.best_of << "}, " + << "beam_search={beam_size=" << self.beam_search.beam_size << ", patience=" << self.beam_search.patience << "}, " + << "new_segment_callback=" << (self.new_segment_callback ? "(function pointer)" : "None") << ", " + << "progress_callback=" << (self.progress_callback ? "(function pointer)" : "None") << ", " + << "encoder_begin_callback=" << (self.encoder_begin_callback ? "(function pointer)" : "None") << ", " + << "abort_callback=" << (self.abort_callback ? "(function pointer)" : "None") << ", " + << "logits_filter_callback=" << (self.logits_filter_callback ? "(function pointer)" : "None") + << ")"; + return oss.str(); + }); + + py::class_(m, "whisper_full_params") + .def(py::init<>()) + .def_readwrite("strategy", &WhisperFullParamsWrapper::strategy) + .def_readwrite("n_threads", &WhisperFullParamsWrapper::n_threads) + .def_readwrite("n_max_text_ctx", &WhisperFullParamsWrapper::n_max_text_ctx) + .def_readwrite("offset_ms", &WhisperFullParamsWrapper::offset_ms) + .def_readwrite("duration_ms", &WhisperFullParamsWrapper::duration_ms) + .def_readwrite("translate", &WhisperFullParamsWrapper::translate) + .def_readwrite("no_context", &WhisperFullParamsWrapper::no_context) + .def_readwrite("no_timestamps", &WhisperFullParamsWrapper::no_timestamps) + .def_readwrite("single_segment", &WhisperFullParamsWrapper::single_segment) + .def_readwrite("print_special", &WhisperFullParamsWrapper::print_special) + .def_readwrite("print_progress", &WhisperFullParamsWrapper::print_progress) + .def_readwrite("progress_callback", &WhisperFullParamsWrapper::py_progress_callback) + .def("set_progress_callback", + [](WhisperFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_progress_callback(); + } else { + self.set_progress_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign a progress callback that receives progress updates.") + .def("clear_progress_callback", &WhisperFullParamsWrapper::clear_progress_callback, + "Clear any previously assigned progress callback while preserving default progress behavior.") + .def_readwrite("print_realtime", &WhisperFullParamsWrapper::print_realtime) + .def_readwrite("print_timestamps", &WhisperFullParamsWrapper::print_timestamps) + .def_readwrite("token_timestamps", &WhisperFullParamsWrapper::token_timestamps) + .def_readwrite("thold_pt", &WhisperFullParamsWrapper::thold_pt) + .def_readwrite("thold_ptsum", &WhisperFullParamsWrapper::thold_ptsum) + .def_readwrite("max_len", &WhisperFullParamsWrapper::max_len) + .def_readwrite("split_on_word", &WhisperFullParamsWrapper::split_on_word) + .def_readwrite("max_tokens", &WhisperFullParamsWrapper::max_tokens) + .def_readwrite("debug_mode", &WhisperFullParamsWrapper::debug_mode) + .def_readwrite("audio_ctx", &WhisperFullParamsWrapper::audio_ctx) + .def_readwrite("tdrz_enable", &WhisperFullParamsWrapper::tdrz_enable) + .def_property("suppress_regex", + [](WhisperFullParamsWrapper &self) { + return py::str(self.suppress_regex ? self.suppress_regex : ""); + }, + [](WhisperFullParamsWrapper &self, const std::string &new_c) { + self.set_suppress_regex(new_c); + }) + .def_property("initial_prompt", + [](WhisperFullParamsWrapper &self) { + return py::str(self.initial_prompt ? self.initial_prompt : ""); + }, + [](WhisperFullParamsWrapper &self, const std::string &initial_prompt) { + self.set_initial_prompt(initial_prompt); + } + ) + .def_property("prompt_tokens", + [](WhisperFullParamsWrapper &self) { + return self.get_prompt_tokens(); + }, + [](WhisperFullParamsWrapper &self, py::object tokens) { + if (tokens.is_none()) { + self.clear_prompt_tokens(); + } else { + self.set_prompt_tokens(tokens.cast>()); + } + }) + .def("set_prompt_tokens", &WhisperFullParamsWrapper::set_prompt_tokens, + py::arg("tokens"), + "Assign prompt tokens from a Python sequence.") + .def("clear_prompt_tokens", &WhisperFullParamsWrapper::clear_prompt_tokens, + "Clear any previously assigned prompt tokens.") + .def("set_new_segment_callback", + [](WhisperFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_new_segment_callback(); + } else { + self.set_new_segment_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign a new-segment callback.") + .def("clear_new_segment_callback", &WhisperFullParamsWrapper::clear_new_segment_callback, + "Clear any previously assigned new-segment callback.") + .def("set_encoder_begin_callback", + [](WhisperFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_encoder_begin_callback(); + } else { + self.set_encoder_begin_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign an encoder-begin callback.") + .def("clear_encoder_begin_callback", &WhisperFullParamsWrapper::clear_encoder_begin_callback, + "Clear any previously assigned encoder-begin callback.") + .def("set_abort_callback", + [](WhisperFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_abort_callback(); + } else { + self.set_abort_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign an abort callback that returns True to stop processing.") + .def("clear_abort_callback", &WhisperFullParamsWrapper::clear_abort_callback, + "Clear any previously assigned abort callback.") + .def_readwrite("prompt_n_tokens", &WhisperFullParamsWrapper::prompt_n_tokens) + .def_readwrite("carry_initial_prompt", &WhisperFullParamsWrapper::carry_initial_prompt) + .def_property("language", + [](WhisperFullParamsWrapper &self) { + return py::str(self.language); + }, + [](WhisperFullParamsWrapper &self, const char *new_c) {// using lang_id let us avoid issues with memory management + const int lang_id = (new_c && strlen(new_c) > 0) ? whisper_lang_id(new_c) : -1; + if (lang_id != -1) { + self.language = whisper_lang_str(lang_id); + } else { + self.language = ""; //defaults to auto-detect + } + }) + .def_readwrite("detect_language", &WhisperFullParamsWrapper::detect_language) + .def_readwrite("suppress_blank", &WhisperFullParamsWrapper::suppress_blank) + .def_readwrite("suppress_nst", &WhisperFullParamsWrapper::suppress_nst) + .def_readwrite("temperature", &WhisperFullParamsWrapper::temperature) + .def_readwrite("max_initial_ts", &WhisperFullParamsWrapper::max_initial_ts) + .def_readwrite("length_penalty", &WhisperFullParamsWrapper::length_penalty) + .def_readwrite("temperature_inc", &WhisperFullParamsWrapper::temperature_inc) + .def_readwrite("entropy_thold", &WhisperFullParamsWrapper::entropy_thold) + .def_readwrite("logprob_thold", &WhisperFullParamsWrapper::logprob_thold) + .def_readwrite("no_speech_thold", &WhisperFullParamsWrapper::no_speech_thold) + // little hack for the internal stuct + .def_property("greedy", [](WhisperFullParamsWrapper &self) {return py::dict("best_of"_a=self.greedy.best_of);}, + [](WhisperFullParamsWrapper &self, py::dict dict) {self.greedy.best_of = dict["best_of"].cast();}) + .def_property("beam_search", [](WhisperFullParamsWrapper &self) {return py::dict("beam_size"_a=self.beam_search.beam_size, "patience"_a=self.beam_search.patience);}, + [](WhisperFullParamsWrapper &self, py::dict dict) {self.beam_search.beam_size = dict["beam_size"].cast(); self.beam_search.patience = dict["patience"].cast();}) + .def_property("new_segment_callback_user_data", + &WhisperFullParamsWrapper::get_new_segment_callback_user_data, + &WhisperFullParamsWrapper::set_new_segment_callback_user_data) + .def_property("progress_callback_user_data", + &WhisperFullParamsWrapper::get_progress_callback_user_data, + &WhisperFullParamsWrapper::set_progress_callback_user_data) + .def_property("encoder_begin_callback_user_data", + &WhisperFullParamsWrapper::get_encoder_begin_callback_user_data, + &WhisperFullParamsWrapper::set_encoder_begin_callback_user_data) + .def_property("abort_callback_user_data", + &WhisperFullParamsWrapper::get_abort_callback_user_data, + &WhisperFullParamsWrapper::set_abort_callback_user_data) + .def_property("logits_filter_callback_user_data", + &WhisperFullParamsWrapper::get_logits_filter_callback_user_data, + &WhisperFullParamsWrapper::set_logits_filter_callback_user_data) + .def("set_logits_filter_callback", + [](WhisperFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_logits_filter_callback(); + } else { + self.set_logits_filter_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign a logits-filter callback.") + .def("clear_logits_filter_callback", &WhisperFullParamsWrapper::clear_logits_filter_callback, + "Clear any previously assigned logits-filter callback.") + .def_readwrite("vad", &WhisperFullParamsWrapper::vad) + .def_property("vad_model_path", + [](WhisperFullParamsWrapper &self) { + return py::str(self.vad_model_path ? self.vad_model_path : ""); + }, + [](WhisperFullParamsWrapper &self, const std::string &vad_model_path) { + self.set_vad_model_path(vad_model_path); + } + ) + .def_readwrite("vad_params", &WhisperFullParamsWrapper::vad_params); + + + py::implicitly_convertible(); + + m.def("whisper_full_default_params", &whisper_full_default_params_wrapper); + + m.def("whisper_full", &whisper_full_wrapper, "Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text\n" + "Uses the specified decoding strategy to obtain the text.\n"); + + m.def("whisper_full_parallel", &whisper_full_parallel_wrapper, "Split the input audio in chunks and process each chunk separately using whisper_full()\n" + "It seems this approach can offer some speedup in some cases.\n" + "However, the transcription accuracy can be worse at the beginning and end of each chunk."); + + m.def("whisper_full_n_segments", &whisper_full_n_segments_wrapper, "Number of generated text segments.\n" + "A segment can be a few words, a sentence, or even a paragraph.\n"); + + m.def("whisper_full_lang_id", &whisper_full_lang_id_wrapper, "Language id associated with the current context"); + m.def("whisper_full_get_segment_t0", &whisper_full_get_segment_t0_wrapper, "Get the start time of the specified segment"); + m.def("whisper_full_get_segment_t1", &whisper_full_get_segment_t1_wrapper, "Get the end time of the specified segment"); + m.def("whisper_full_get_segment_speaker_turn_next", &whisper_full_get_segment_speaker_turn_next_wrapper, + "Get whether the next segment is predicted as a speaker turn."); + + m.def("whisper_full_get_segment_text", &whisper_full_get_segment_text_wrapper, "Get the text of the specified segment"); + m.def("whisper_full_n_tokens", &whisper_full_n_tokens_wrapper, "Get number of tokens in the specified segment."); + + m.def("whisper_full_get_token_text", &whisper_full_get_token_text_wrapper, "Get the token text of the specified token in the specified segment."); + m.def("whisper_full_get_token_id", &whisper_full_get_token_id_wrapper, "Get the token text of the specified token in the specified segment."); + + m.def("whisper_full_get_token_data", &whisper_full_get_token_data_wrapper, "Get token data for the specified token in the specified segment.\n" + "This contains probabilities, timestamps, etc."); + + m.def("whisper_full_get_token_p", &whisper_full_get_token_p_wrapper, "Get the probability of the specified token in the specified segment."); + + m.def("whisper_ctx_init_openvino_encoder", &whisper_ctx_init_openvino_encoder_wrapper, "Given a context, enable use of OpenVINO for encode inference."); + m.def("whisper_model_type_readable", &whisper_model_type_readable_wrapper, "Return the readable model type string."); + m.def("whisper_model_n_vocab", &whisper_model_n_vocab_wrapper, "Return the model vocabulary size."); + m.def("whisper_model_n_audio_ctx", &whisper_model_n_audio_ctx_wrapper, "Return the audio context size baked into the model."); + m.def("whisper_model_n_audio_state", &whisper_model_n_audio_state_wrapper, "Return the number of audio state units in the model."); + m.def("whisper_model_n_audio_head", &whisper_model_n_audio_head_wrapper, "Return the number of audio attention heads in the model."); + m.def("whisper_model_n_audio_layer", &whisper_model_n_audio_layer_wrapper, "Return the number of audio layers in the model."); + m.def("whisper_model_n_text_ctx", &whisper_model_n_text_ctx_wrapper, "Return the text context size baked into the model."); + m.def("whisper_model_n_text_state", &whisper_model_n_text_state_wrapper, "Return the number of text state units in the model."); + m.def("whisper_model_n_text_head", &whisper_model_n_text_head_wrapper, "Return the number of text attention heads in the model."); + m.def("whisper_model_n_text_layer", &whisper_model_n_text_layer_wrapper, "Return the number of text layers in the model."); + m.def("whisper_model_n_mels", &whisper_model_n_mels_wrapper, "Return the number of mel bins used by the model."); + m.def("whisper_model_ftype", &whisper_model_ftype_wrapper, "Return the model file type identifier."); + + + //////////////////////////////////////////////////////////////////////////// + + m.def("whisper_bench_memcpy", &whisper_bench_memcpy, "Temporary helpers needed for exposing ggml interface"); + m.def("whisper_bench_ggml_mul_mat", &whisper_bench_ggml_mul_mat, "Temporary helpers needed for exposing ggml interface"); + + //////////////////////////////////////////////////////////////////////////// + // Helper mechanism to set callbacks from python + // The only difference from the C-Style API + + m.def("assign_new_segment_callback", + [](whisper_full_params * params, py::object callback) { + assign_new_segment_callback(params, callback); + }, + "Assign a new-segment callback.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("clear_new_segment_callback", &clear_new_segment_callback, + "Clear any previously assigned new-segment callback.", + py::arg("params")); + + m.def("assign_encoder_begin_callback", + [](whisper_full_params * params, py::object callback) { + assign_encoder_begin_callback(params, callback); + }, + "Assign an encoder-begin callback.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("clear_encoder_begin_callback", &clear_encoder_begin_callback, + "Clear any previously assigned encoder-begin callback.", + py::arg("params")); + + m.def("assign_logits_filter_callback", + [](whisper_full_params * params, py::object callback) { + assign_logits_filter_callback(params, callback); + }, + "Assign a logits-filter callback.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("clear_logits_filter_callback", &clear_logits_filter_callback, + "Clear any previously assigned logits-filter callback.", + py::arg("params")); + + m.def("assign_progress_callback", + [](whisper_full_params * params, py::object callback) { + assign_progress_callback(params, callback); + }, + "Assign a progress callback that receives progress updates.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("clear_progress_callback", &clear_progress_callback, + "Clear any previously assigned progress callback while preserving default progress behavior.", + py::arg("params")); + + m.def("assign_abort_callback", + [](whisper_full_params * params, py::object callback) { + assign_abort_callback(params, callback); + }, + "Assign an abort callback that returns True to stop processing.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("clear_abort_callback", &clear_abort_callback, "Clear any previously assigned abort callback.", + py::arg("params")); + + m.def("whisper_log_set", + [](py::object callback) { + whisper_log_set_wrapper(callback); + }, + "Assign a Python log callback or None to restore the default logger.", + py::arg("callback") = py::none()); + + // VAD + py::class_(m,"whisper_vad_params") + .def(py::init<>()) + .def_readwrite("threshold", &whisper_vad_params::threshold) + .def_readwrite("min_speech_duration_ms", &whisper_vad_params::min_speech_duration_ms) + .def_readwrite("min_silence_duration_ms", &whisper_vad_params::min_silence_duration_ms) + .def_readwrite("max_speech_duration_s", &whisper_vad_params::max_speech_duration_s) + .def_readwrite("speech_pad_ms", &whisper_vad_params::speech_pad_ms) + .def_readwrite("samples_overlap", &whisper_vad_params::samples_overlap); + + m.def("whisper_vad_default_params", &whisper_vad_default_params); + + py::class_(m,"whisper_vad_context_params") + .def(py::init<>()) + .def_readwrite("n_threads", &whisper_vad_context_params::n_threads) + .def_readwrite("use_gpu", &whisper_vad_context_params::use_gpu) + .def_readwrite("gpu_device", &whisper_vad_context_params::gpu_device); + + m.def("whisper_vad_default_context_params", &whisper_vad_default_context_params); + m.def("whisper_vad_init_from_file_with_params", &whisper_vad_init_from_file_with_params_wrapper); + m.def("whisper_vad_detect_speech", &whisper_vad_detect_speech_wrapper); + m.def("whisper_vad_n_probs", &whisper_vad_n_probs_wrapper); + m.def("whisper_vad_probs", &whisper_vad_probs_wrapper); + py::class_(m, "whisper_vad_segments"); + m.def("whisper_vad_segments_from_probs", &whisper_vad_segments_from_probs_wrapper); + m.def("whisper_vad_segments_from_samples", &whisper_vad_segments_from_samples_wrapper); + m.def("whisper_vad_segments_n_segments", &whisper_vad_segments_n_segments_wrapper); + m.def("whisper_vad_segments_get_segment_t0", &whisper_vad_segments_get_segment_t0_wrapper); + m.def("whisper_vad_segments_get_segment_t1", &whisper_vad_segments_get_segment_t1_wrapper); + m.def("whisper_vad_free_segments", &whisper_vad_free_segments_wrapper); + m.def("whisper_vad_free", &whisper_vad_free_wrapper); + + +#ifdef VERSION_INFO + m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); +#else + m.attr("__version__") = "dev"; +#endif +} From 48536864ba6a2f6e349d11ec862a3361dd2c9c1f Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Mon, 27 Jul 2026 00:33:16 -0400 Subject: [PATCH 04/14] feat: add parakeet model bindings --- CMakeLists.txt | 2 +- src/bindings_utils.h | 9 + src/main.cpp | 4 + src/parakeet_bindings.cpp | 720 ++++++++++++++++++++++++++++++++++++++ src/whisper_bindings.cpp | 8 +- 5 files changed, 735 insertions(+), 8 deletions(-) create mode 100644 src/bindings_utils.h create mode 100644 src/parakeet_bindings.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 39c16a8..5d287e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,5 +8,5 @@ pybind11_add_module(_pywhispercpp src/main.cpp ) -target_link_libraries (_pywhispercpp PRIVATE whisper) +target_link_libraries (_pywhispercpp PRIVATE whisper parakeet) diff --git a/src/bindings_utils.h b/src/bindings_utils.h new file mode 100644 index 0000000..e69949f --- /dev/null +++ b/src/bindings_utils.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +#define DEF_RELEASE_GIL(name, fn, doc) \ + m.def(name, fn, doc, py::call_guard()) diff --git a/src/main.cpp b/src/main.cpp index 805ffce..93776c1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -10,7 +10,10 @@ ******************************************************************************** */ +#include #include "whisper_bindings.cpp" +#include "parakeet_bindings.cpp" + PYBIND11_MODULE(_pywhispercpp, m) { m.doc() = R"pbdoc( @@ -25,4 +28,5 @@ PYBIND11_MODULE(_pywhispercpp, m) { )pbdoc"; whisper_bindings(m); + parakeet_bindings(m); } diff --git a/src/parakeet_bindings.cpp b/src/parakeet_bindings.cpp new file mode 100644 index 0000000..0864016 --- /dev/null +++ b/src/parakeet_bindings.cpp @@ -0,0 +1,720 @@ +#include +#include +#include +#include + +#include "parakeet.h" +#include "bindings_utils.h" + + +namespace py = pybind11; +using namespace pybind11::literals; + +inline bool has_parakeet_python_user_data(const py::object & obj) { + return obj.ptr() != nullptr && obj.ptr() != Py_None; +} + +py::object py_parakeet_log_callback; + + +// parakeet context wrapper, to solve the incomplete type issue +struct parakeet_context_wrapper { + parakeet_context* ptr; +}; + +struct parakeet_context_wrapper parakeet_init_from_file_with_params_wrapper( + const char * path_model, + struct parakeet_context_params cparams){ + struct parakeet_context * ctx = parakeet_init_from_file_with_params(path_model, cparams); + struct parakeet_context_wrapper ctw_w; + ctw_w.ptr = ctx; + return ctw_w; +} + +struct parakeet_context_wrapper parakeet_init_from_buffer_with_params_wrapper( + void * buffer, + size_t buffer_size, + struct parakeet_context_params cparams){ + struct parakeet_context * ctx = parakeet_init_from_buffer_with_params(buffer, buffer_size, cparams); + struct parakeet_context_wrapper ctw_w; + ctw_w.ptr = ctx; + return ctw_w; +} + +void parakeet_free_wrapper(struct parakeet_context_wrapper * ctx_w){ + parakeet_free(ctx_w->ptr); +}; + +int parakeet_pcm_to_mel_wrapper( + struct parakeet_context_wrapper * ctx, + py::array_t samples, + int n_samples, + int n_threads){ + py::buffer_info buf = samples.request(); + float *samples_ptr = static_cast(buf.ptr); + py::gil_scoped_release release; + return parakeet_pcm_to_mel(ctx->ptr, samples_ptr, n_samples, n_threads); +}; + +int parakeet_set_mel_wrapper( + struct parakeet_context_wrapper * ctx, + py::array_t data, + int n_len, + int n_mel){ + py::buffer_info buf = data.request(); + float *data_ptr = static_cast(buf.ptr); + return parakeet_set_mel(ctx->ptr, data_ptr, n_len, n_mel); +}; + +int parakeet_encode_wrapper( + struct parakeet_context_wrapper * ctx, + int offset, + int n_threads){ + py::gil_scoped_release release; + return parakeet_encode(ctx->ptr, offset, n_threads); +} + +int parakeet_tokenize_wrapper( + struct parakeet_context_wrapper * ctx, + const char * text, + parakeet_token * tokens, + int n_max_tokens){ + return parakeet_tokenize(ctx->ptr, text, tokens, n_max_tokens); +}; + +int parakeet_n_len_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_n_len(ctx_w->ptr); +}; + +int parakeet_n_vocab_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_n_vocab(ctx_w->ptr); +}; + +int parakeet_n_audio_ctx_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_n_audio_ctx(ctx_w->ptr); +} + +float * parakeet_get_logits_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_get_logits(ctx_w->ptr); +}; + +const char * parakeet_token_to_str_wrapper(struct parakeet_context_wrapper * ctx_w, parakeet_token token){ + return parakeet_token_to_str(ctx_w->ptr, token); +}; + +py::bytes parakeet_token_to_bytes_wrapper(struct parakeet_context_wrapper * ctx_w, parakeet_token token){ + const char* str = parakeet_token_to_str(ctx_w->ptr, token); + size_t l = strlen(str); + return py::bytes(str, l); +} + +parakeet_token parakeet_token_blank_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_token_blank(ctx_w->ptr); +} + +parakeet_token parakeet_token_unk_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_token_unk(ctx_w->ptr); +} + +parakeet_token parakeet_token_bos_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_token_bos(ctx_w->ptr); +} + +void parakeet_print_timings_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_print_timings(ctx_w->ptr); +} + +void parakeet_reset_timings_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_reset_timings(ctx_w->ptr); +} + +int parakeet_full_wrapper( + struct parakeet_context_wrapper * ctx_w, + struct parakeet_full_params params, + py::array_t samples, + int n_samples){ + py::buffer_info buf = samples.request(); + float *samples_ptr = static_cast(buf.ptr); + + py::gil_scoped_release release; + return parakeet_full(ctx_w->ptr, params, samples_ptr, n_samples); +} + +int parakeet_full_n_segments_wrapper(struct parakeet_context_wrapper * ctx){ + return parakeet_full_n_segments(ctx->ptr); +} + +int64_t parakeet_full_get_segment_t0_wrapper(struct parakeet_context_wrapper * ctx, int i_segment){ + return parakeet_full_get_segment_t0(ctx->ptr, i_segment); +} + +int64_t parakeet_full_get_segment_t1_wrapper(struct parakeet_context_wrapper * ctx, int i_segment){ + return parakeet_full_get_segment_t1(ctx->ptr, i_segment); +} + +const py::bytes parakeet_full_get_segment_text_wrapper(struct parakeet_context_wrapper * ctx, int i_segment){ + const char * c_array = parakeet_full_get_segment_text(ctx->ptr, i_segment); + size_t length = strlen(c_array); + return py::bytes(c_array, length); +}; + +int parakeet_full_n_tokens_wrapper(struct parakeet_context_wrapper * ctx, int i_segment){ + return parakeet_full_n_tokens(ctx->ptr, i_segment); +} + +const char * parakeet_full_get_token_text_wrapper(struct parakeet_context_wrapper * ctx, int i_segment, int i_token){ + return parakeet_full_get_token_text(ctx->ptr, i_segment, i_token); +} + +parakeet_token parakeet_full_get_token_id_wrapper(struct parakeet_context_wrapper * ctx, int i_segment, int i_token){ + return parakeet_full_get_token_id(ctx->ptr, i_segment, i_token); +} + +parakeet_token_data parakeet_full_get_token_data_wrapper(struct parakeet_context_wrapper * ctx, int i_segment, int i_token){ + return parakeet_full_get_token_data(ctx->ptr, i_segment, i_token); +} + +float parakeet_full_get_token_p_wrapper(struct parakeet_context_wrapper * ctx, int i_segment, int i_token){ + return parakeet_full_get_token_p(ctx->ptr, i_segment, i_token); +} + +int parakeet_model_n_vocab_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_model_n_vocab(ctx_w->ptr); +} + +int parakeet_model_n_audio_ctx_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_model_n_audio_ctx(ctx_w->ptr); +} + +int parakeet_model_n_audio_state_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_model_n_audio_state(ctx_w->ptr); +} + +int parakeet_model_n_audio_head_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_model_n_audio_head(ctx_w->ptr); +} + +int parakeet_model_n_audio_layer_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_model_n_audio_layer(ctx_w->ptr); +} + +int parakeet_model_n_mels_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_model_n_mels(ctx_w->ptr); +} + +int parakeet_model_ftype_wrapper(struct parakeet_context_wrapper * ctx_w){ + return parakeet_model_ftype(ctx_w->ptr); +} + +// forward declarations for callbacks +bool _parakeet_abort_callback(void * user_data); +void _parakeet_new_segment_callback(struct parakeet_context * ctx, struct parakeet_state * state, int n_new, void * user_data); +bool _parakeet_encoder_begin_callback(struct parakeet_context * ctx, struct parakeet_state * state, void * user_data); + +struct ParakeetFullParamsWrapper : public parakeet_full_params { + ParakeetFullParamsWrapper(const parakeet_full_params& params = parakeet_full_params()) + : parakeet_full_params(params), + py_new_segment_callback_user_data(py::none()), + py_encoder_begin_callback_user_data(py::none()), + py_progress_callback_user_data(py::none()), + py_abort_callback_user_data(py::none()) + { + new_segment_callback_user_data = this; + encoder_begin_callback_user_data = this; + abort_callback_user_data = this; + progress_callback_user_data = this; + } + + ParakeetFullParamsWrapper(const ParakeetFullParamsWrapper& other) + : parakeet_full_params(static_cast(other)), + py_new_segment_callback(other.py_new_segment_callback), + py_new_segment_callback_user_data(other.py_new_segment_callback_user_data), + py_encoder_begin_callback(other.py_encoder_begin_callback), + py_encoder_begin_callback_user_data(other.py_encoder_begin_callback_user_data), + py_progress_callback(other.py_progress_callback), + py_progress_callback_user_data(other.py_progress_callback_user_data), + py_abort_callback(other.py_abort_callback), + py_abort_callback_user_data(other.py_abort_callback_user_data) + { + new_segment_callback_user_data = this; + encoder_begin_callback_user_data = this; + abort_callback_user_data = this; + progress_callback_user_data = this; + } + +public: + py::function py_new_segment_callback; + py::object py_new_segment_callback_user_data; + py::function py_encoder_begin_callback; + py::object py_encoder_begin_callback_user_data; + py::function py_progress_callback; + py::object py_progress_callback_user_data; + py::function py_abort_callback; + py::object py_abort_callback_user_data; + + py::object get_new_segment_callback_user_data() const { + return py_new_segment_callback_user_data; + } + void set_new_segment_callback_user_data(py::object user_data) { + py_new_segment_callback_user_data = std::move(user_data); + new_segment_callback_user_data = this; + } + void set_new_segment_callback(py::function callback) { + py_new_segment_callback = std::move(callback); + new_segment_callback_user_data = this; + new_segment_callback = _parakeet_new_segment_callback; + } + void clear_new_segment_callback() { + py_new_segment_callback = py::function(); + new_segment_callback = nullptr; + new_segment_callback_user_data = this; + } + + py::object get_encoder_begin_callback_user_data() const { + return py_encoder_begin_callback_user_data; + } + void set_encoder_begin_callback_user_data(py::object user_data) { + py_encoder_begin_callback_user_data = std::move(user_data); + encoder_begin_callback_user_data = this; + } + void set_encoder_begin_callback(py::function callback) { + py_encoder_begin_callback = std::move(callback); + encoder_begin_callback_user_data = this; + encoder_begin_callback = _parakeet_encoder_begin_callback; + } + void clear_encoder_begin_callback() { + py_encoder_begin_callback = py::function(); + encoder_begin_callback = nullptr; + encoder_begin_callback_user_data = this; + } + + py::object get_progress_callback_user_data() const { + return py_progress_callback_user_data; + } + void set_progress_callback_user_data(py::object user_data) { + py_progress_callback_user_data = std::move(user_data); + progress_callback_user_data = this; + } + void set_progress_callback(py::function callback) { + py_progress_callback = std::move(callback); + progress_callback_user_data = this; + progress_callback = [](struct parakeet_context* ctx, struct parakeet_state* state, int progress, void* user_data) { + (void) ctx; + (void) state; + auto* self = static_cast(user_data); + if (self && self->py_progress_callback) { + py::gil_scoped_acquire gil; + if (!has_parakeet_python_user_data(self->py_progress_callback_user_data)) { + self->py_progress_callback(progress); + } else { + self->py_progress_callback(progress, self->py_progress_callback_user_data); + } + } + }; + } + void clear_progress_callback() { + py_progress_callback = py::function(); + progress_callback = nullptr; + progress_callback_user_data = this; + } + + py::object get_abort_callback_user_data() const { + return py_abort_callback_user_data; + } + void set_abort_callback_user_data(py::object user_data) { + py_abort_callback_user_data = std::move(user_data); + abort_callback_user_data = this; + } + void set_abort_callback(py::function callback) { + py_abort_callback = std::move(callback); + abort_callback_user_data = this; + abort_callback = _parakeet_abort_callback; + } + void clear_abort_callback() { + py_abort_callback = py::function(); + abort_callback = nullptr; + abort_callback_user_data = this; + } +}; + +ParakeetFullParamsWrapper parakeet_full_default_params_wrapper(enum parakeet_sampling_strategy strategy) { + return ParakeetFullParamsWrapper(parakeet_full_default_params(strategy)); +} + +// callbacks mechanism + +void _parakeet_new_segment_callback(struct parakeet_context * ctx, struct parakeet_state * state, int n_new, void * user_data){ + (void) state; + struct parakeet_context_wrapper ctx_w; + ctx_w.ptr = ctx; + auto * params = static_cast(user_data); + if (!params || !params->py_new_segment_callback) { + return; + } + + py::gil_scoped_acquire gil; + py::function callback = params->py_new_segment_callback; + if (!has_parakeet_python_user_data(params->py_new_segment_callback_user_data)) { + callback(ctx_w, n_new); + } else { + callback(ctx_w, n_new, params->py_new_segment_callback_user_data); + } +}; + +void parakeet_assign_new_segment_callback(parakeet_full_params *params_base, py::object callback){ + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_new_segment_callback(); + return; + } + + params->set_new_segment_callback(callback.cast()); +} + +void parakeet_clear_new_segment_callback(parakeet_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_new_segment_callback(); +}; + +bool _parakeet_encoder_begin_callback(struct parakeet_context * ctx, struct parakeet_state * state, void * user_data){ + (void) state; + struct parakeet_context_wrapper ctx_w; + ctx_w.ptr = ctx; + auto * params = static_cast(user_data); + if (!params || !params->py_encoder_begin_callback) { + return false; + } + + py::gil_scoped_acquire gil; + py::function callback = params->py_encoder_begin_callback; + py::object result_py; + if (!has_parakeet_python_user_data(params->py_encoder_begin_callback_user_data)) { + result_py = callback(ctx_w); + } else { + result_py = callback(ctx_w, params->py_encoder_begin_callback_user_data); + } + bool res = result_py.cast(); + return res; +} + +void parakeet_assign_encoder_begin_callback(parakeet_full_params *params_base, py::object callback){ + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_encoder_begin_callback(); + return; + } + + params->set_encoder_begin_callback(callback.cast()); +} + +void parakeet_clear_encoder_begin_callback(parakeet_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_encoder_begin_callback(); +} + +bool _parakeet_abort_callback(void * user_data) { + auto * params = static_cast(user_data); + if (!params || !params->py_abort_callback) { + return false; + } + + py::gil_scoped_acquire gil; + py::function callback = params->py_abort_callback; + py::object result_py; + if (!has_parakeet_python_user_data(params->py_abort_callback_user_data)) { + result_py = callback(); + } else { + result_py = callback(params->py_abort_callback_user_data); + } + return result_py.cast(); +} + +void parakeet_assign_abort_callback(parakeet_full_params *params_base, py::object callback){ + auto * params = static_cast(params_base); + if (callback.is_none()) { + params->clear_abort_callback(); + return; + } + + params->set_abort_callback(callback.cast()); +} + +void parakeet_clear_abort_callback(parakeet_full_params *params_base) { + auto * params = static_cast(params_base); + params->clear_abort_callback(); +} + +void parakeet_log_set_wrapper(py::object callback) { + if (callback.is_none()) { + py_parakeet_log_callback = py::none(); + parakeet_log_set(nullptr, nullptr); + return; + } + + py_parakeet_log_callback = callback.cast(); + parakeet_log_set( + [](enum ggml_log_level level, const char * text, void * user_data) { + (void) user_data; + py::gil_scoped_acquire gil; + py::function log_callback = py_parakeet_log_callback.cast(); + log_callback(py::int_(static_cast(level)), py::str(text ? text : "")); + }, + nullptr); +} + + +////////// + +void parakeet_bindings(py::module_ & m) { + + m.attr("PARAKEET_SAMPLE_RATE") = PARAKEET_SAMPLE_RATE; + m.attr("PARAKEET_HOP_LENGTH") = PARAKEET_HOP_LENGTH; + + py::class_(m, "parakeet_context"); + py::class_(m, "parakeet_context_params") + .def(py::init<>()) + .def_readwrite("use_gpu", ¶keet_context_params::use_gpu) + .def_readwrite("gpu_device", ¶keet_context_params::gpu_device); + + py::class_(m, "parakeet_token_data") + .def(py::init<>()) + .def_readwrite("id", ¶keet_token_data::id) + .def_readwrite("duration_idx", ¶keet_token_data::duration_idx) + .def_readwrite("duration_value", ¶keet_token_data::duration_value) + .def_readwrite("frame_index", ¶keet_token_data::frame_index) + .def_readwrite("p", ¶keet_token_data::p) + .def_readwrite("plog", ¶keet_token_data::plog) + .def_readwrite("t0", ¶keet_token_data::t0) + .def_readwrite("t1", ¶keet_token_data::t1) + .def_readwrite("is_word_start", ¶keet_token_data::is_word_start); + + m.def("parakeet_version", ¶keet_version, "Return the version of the parakeet library."); + + m.def("parakeet_context_default_params", ¶keet_context_default_params, + "Return the default context parameters used during model initialization."); + + DEF_RELEASE_GIL("parakeet_init_from_file_with_params", ¶keet_init_from_file_with_params_wrapper, + "Various functions for loading a ggml parakeet model.\n" + "Allocate (almost) all memory needed for the model.\n" + "Return NULL on failure"); + + DEF_RELEASE_GIL("parakeet_init_from_buffer_with_params", ¶keet_init_from_buffer_with_params_wrapper, + "Various functions for loading a ggml parakeet model.\n" + "Allocate (almost) all memory needed for the model.\n" + "Return NULL on failure"); + + m.def("parakeet_free", ¶keet_free_wrapper, "Frees all memory allocated by the model."); + + m.def("parakeet_pcm_to_mel", ¶keet_pcm_to_mel_wrapper, "Convert RAW PCM audio to log mel spectrogram.\n" + "The resulting spectrogram is stored inside the provided parakeet context.\n" + "Returns 0 on success"); + + m.def("parakeet_set_mel", ¶keet_set_mel_wrapper, "This can be used to set a custom log mel spectrogram inside the provided parakeet context.\n" + "Use this instead of parakeet_pcm_to_mel() if you want to provide your own log mel spectrogram.\n" + "n_mel must be 128\n" + "Returns 0 on success"); + + m.def("parakeet_encode", ¶keet_encode_wrapper, "Run the Parakeet encoder on the log mel spectrogram stored inside the provided parakeet context.\n" + "Make sure to call parakeet_pcm_to_mel() or parakeet_set_mel() first.\n" + "offset can be used to specify the offset of the first frame in the spectrogram.\n" + "Returns 0 on success"); + + m.def("parakeet_tokenize", ¶keet_tokenize_wrapper, "Convert the provided text into tokens.\n" + "The tokens pointer must be large enough to hold the resulting tokens.\n" + "Returns the number of tokens on success, no more than n_max_tokens\n" + "Returns a negative number on failure"); + + m.def("parakeet_n_len", ¶keet_n_len_wrapper, "parakeet_n_len"); + m.def("parakeet_n_vocab", ¶keet_n_vocab_wrapper, "parakeet_n_vocab"); + m.def("parakeet_n_audio_ctx", ¶keet_n_audio_ctx_wrapper, "parakeet_n_audio_ctx"); + + m.def("parakeet_get_logits", ¶keet_get_logits_wrapper, "Token logits obtained from the last call to parakeet_full()\n" + "The logits for the last token are stored in the last row\n" + "Rows: n_tokens\n" + "Cols: n_vocab"); + + m.def("parakeet_token_to_str", ¶keet_token_to_str_wrapper, "parakeet_token_to_str"); + m.def("parakeet_token_to_bytes", ¶keet_token_to_bytes_wrapper, "parakeet_token_to_bytes"); + m.def("parakeet_token_blank", ¶keet_token_blank_wrapper, "parakeet_token_blank"); + m.def("parakeet_token_unk", ¶keet_token_unk_wrapper, "parakeet_token_unk"); + m.def("parakeet_token_bos", ¶keet_token_bos_wrapper, "parakeet_token_bos"); + + m.def("parakeet_print_timings", ¶keet_print_timings_wrapper); + m.def("parakeet_reset_timings", ¶keet_reset_timings_wrapper); + + m.def("parakeet_print_system_info", ¶keet_print_system_info); + + ////////////////////// + + py::enum_(m, "parakeet_sampling_strategy") + .value("PARAKEET_SAMPLING_GREEDY", parakeet_sampling_strategy::PARAKEET_SAMPLING_GREEDY) + .export_values(); + + py::class_(m, "__parakeet_full_params__internal") + .def(py::init<>()) + .def("__repr__", [](const parakeet_full_params& self) { + std::ostringstream oss; + oss << "parakeet_full_params(" + << "strategy=" << self.strategy << ", " + << "n_threads=" << self.n_threads << ", " + << "offset_ms=" << self.offset_ms << ", " + << "duration_ms=" << self.duration_ms << ", " + << "no_context=" << (self.no_context ? "True" : "False") << ", " + << "audio_ctx=" << self.audio_ctx << ", " + << "new_segment_callback=" << (self.new_segment_callback ? "(function pointer)" : "None") << ", " + << "progress_callback=" << (self.progress_callback ? "(function pointer)" : "None") << ", " + << "encoder_begin_callback=" << (self.encoder_begin_callback ? "(function pointer)" : "None") << ", " + << "abort_callback=" << (self.abort_callback ? "(function pointer)" : "None") + << ")"; + return oss.str(); + }); + + py::class_(m, "parakeet_full_params") + .def(py::init<>()) + .def_readwrite("strategy", &ParakeetFullParamsWrapper::strategy) + .def_readwrite("n_threads", &ParakeetFullParamsWrapper::n_threads) + .def_readwrite("offset_ms", &ParakeetFullParamsWrapper::offset_ms) + .def_readwrite("duration_ms", &ParakeetFullParamsWrapper::duration_ms) + .def_readwrite("no_context", &ParakeetFullParamsWrapper::no_context) + .def_readwrite("audio_ctx", &ParakeetFullParamsWrapper::audio_ctx) + .def_readwrite("progress_callback", &ParakeetFullParamsWrapper::py_progress_callback) + .def("set_progress_callback", + [](ParakeetFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_progress_callback(); + } else { + self.set_progress_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign a progress callback that receives progress updates.") + .def("clear_progress_callback", &ParakeetFullParamsWrapper::clear_progress_callback, + "Clear any previously assigned progress callback while preserving default progress behavior.") + .def("set_new_segment_callback", + [](ParakeetFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_new_segment_callback(); + } else { + self.set_new_segment_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign a new-segment callback.") + .def("clear_new_segment_callback", &ParakeetFullParamsWrapper::clear_new_segment_callback, + "Clear any previously assigned new-segment callback.") + .def("set_encoder_begin_callback", + [](ParakeetFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_encoder_begin_callback(); + } else { + self.set_encoder_begin_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign an encoder-begin callback.") + .def("clear_encoder_begin_callback", &ParakeetFullParamsWrapper::clear_encoder_begin_callback, + "Clear any previously assigned encoder-begin callback.") + .def("set_abort_callback", + [](ParakeetFullParamsWrapper &self, py::object callback) { + if (callback.is_none()) { + self.clear_abort_callback(); + } else { + self.set_abort_callback(callback.cast()); + } + }, + py::arg("callback") = py::none(), + "Assign an abort callback that returns True to stop processing.") + .def("clear_abort_callback", &ParakeetFullParamsWrapper::clear_abort_callback, + "Clear any previously assigned abort callback.") + .def_property("new_segment_callback_user_data", + &ParakeetFullParamsWrapper::get_new_segment_callback_user_data, + &ParakeetFullParamsWrapper::set_new_segment_callback_user_data) + .def_property("progress_callback_user_data", + &ParakeetFullParamsWrapper::get_progress_callback_user_data, + &ParakeetFullParamsWrapper::set_progress_callback_user_data) + .def_property("encoder_begin_callback_user_data", + &ParakeetFullParamsWrapper::get_encoder_begin_callback_user_data, + &ParakeetFullParamsWrapper::set_encoder_begin_callback_user_data) + .def_property("abort_callback_user_data", + &ParakeetFullParamsWrapper::get_abort_callback_user_data, + &ParakeetFullParamsWrapper::set_abort_callback_user_data); + + py::implicitly_convertible(); + + m.def("parakeet_full_default_params", ¶keet_full_default_params_wrapper); + + m.def("parakeet_full", ¶keet_full_wrapper, "Run the entire model: PCM -> log mel spectrogram -> encoder -> decoder -> text\n" + "Uses the specified decoding strategy to obtain the text.\n"); + + m.def("parakeet_full_n_segments", ¶keet_full_n_segments_wrapper, "Number of generated text segments.\n" + "A segment can be a few words, a sentence, or even a paragraph.\n"); + + m.def("parakeet_full_get_segment_t0", ¶keet_full_get_segment_t0_wrapper, "Get the start time of the specified segment"); + m.def("parakeet_full_get_segment_t1", ¶keet_full_get_segment_t1_wrapper, "Get the end time of the specified segment"); + m.def("parakeet_full_get_segment_text", ¶keet_full_get_segment_text_wrapper, "Get the text of the specified segment"); + m.def("parakeet_full_n_tokens", ¶keet_full_n_tokens_wrapper, "Get number of tokens in the specified segment."); + + m.def("parakeet_full_get_token_text", ¶keet_full_get_token_text_wrapper, "Get the token text of the specified token in the specified segment."); + m.def("parakeet_full_get_token_id", ¶keet_full_get_token_id_wrapper, "Get the token id of the specified token in the specified segment."); + + m.def("parakeet_full_get_token_data", ¶keet_full_get_token_data_wrapper, "Get token data for the specified token in the specified segment.\n" + "This contains probabilities, timestamps, etc."); + + m.def("parakeet_full_get_token_p", ¶keet_full_get_token_p_wrapper, "Get the probability of the specified token in the specified segment."); + + m.def("parakeet_model_n_vocab", ¶keet_model_n_vocab_wrapper, "Return the model vocabulary size."); + m.def("parakeet_model_n_audio_ctx", ¶keet_model_n_audio_ctx_wrapper, "Return the audio context size baked into the model."); + m.def("parakeet_model_n_audio_state", ¶keet_model_n_audio_state_wrapper, "Return the number of audio state units in the model."); + m.def("parakeet_model_n_audio_head", ¶keet_model_n_audio_head_wrapper, "Return the number of audio attention heads in the model."); + m.def("parakeet_model_n_audio_layer", ¶keet_model_n_audio_layer_wrapper, "Return the number of audio layers in the model."); + m.def("parakeet_model_n_mels", ¶keet_model_n_mels_wrapper, "Return the number of mel bins used by the model."); + m.def("parakeet_model_ftype", ¶keet_model_ftype_wrapper, "Return the model file type identifier."); + + //////////////////////////////////////////////////////////////////////////// + // Helper mechanism to set callbacks from python + + m.def("parakeet_assign_new_segment_callback", + [](parakeet_full_params * params, py::object callback) { + parakeet_assign_new_segment_callback(params, callback); + }, + "Assign a new-segment callback.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("parakeet_clear_new_segment_callback", ¶keet_clear_new_segment_callback, + "Clear any previously assigned new-segment callback.", + py::arg("params")); + + m.def("parakeet_assign_encoder_begin_callback", + [](parakeet_full_params * params, py::object callback) { + parakeet_assign_encoder_begin_callback(params, callback); + }, + "Assign an encoder-begin callback.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("parakeet_clear_encoder_begin_callback", ¶keet_clear_encoder_begin_callback, + "Clear any previously assigned encoder-begin callback.", + py::arg("params")); + + m.def("parakeet_assign_abort_callback", + [](parakeet_full_params * params, py::object callback) { + parakeet_assign_abort_callback(params, callback); + }, + "Assign an abort callback that returns True to stop processing.", + py::arg("params"), py::arg("callback") = py::none()); + + m.def("parakeet_clear_abort_callback", ¶keet_clear_abort_callback, "Clear any previously assigned abort callback.", + py::arg("params")); + + m.def("parakeet_log_set", + [](py::object callback) { + parakeet_log_set_wrapper(callback); + }, + "Assign a Python log callback or None to restore the default logger.", + py::arg("callback") = py::none()); + +#ifdef VERSION_INFO + m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); +#else + m.attr("__version__") = "dev"; +#endif +} diff --git a/src/whisper_bindings.cpp b/src/whisper_bindings.cpp index f32a1dd..a67cc49 100644 --- a/src/whisper_bindings.cpp +++ b/src/whisper_bindings.cpp @@ -4,13 +4,7 @@ #include #include "whisper.h" - - -#define STRINGIFY(x) #x -#define MACRO_STRINGIFY(x) STRINGIFY(x) - -#define DEF_RELEASE_GIL(name, fn, doc) \ - m.def(name, fn, doc, py::call_guard()) +#include "bindings_utils.h" namespace py = pybind11; From 10e6d1a4cb4a16e58e7577b948ae72428200ac25 Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Mon, 27 Jul 2026 20:18:42 -0400 Subject: [PATCH 05/14] refactor: extract BaseModel ABC and load_audio to utils --- pywhispercpp/base.py | 46 +++++++++++++++++ pywhispercpp/model.py | 114 ++---------------------------------------- pywhispercpp/utils.py | 62 +++++++++++++++++++++++ 3 files changed, 112 insertions(+), 110 deletions(-) create mode 100644 pywhispercpp/base.py diff --git a/pywhispercpp/base.py b/pywhispercpp/base.py new file mode 100644 index 0000000..45e1919 --- /dev/null +++ b/pywhispercpp/base.py @@ -0,0 +1,46 @@ +from abc import ABC, abstractmethod +from typing import List, Union + +import numpy as np + + +class Segment: + """ + A small class representing a transcription segment + """ + + def __init__(self, t0: int, t1: int, text: str, probability: float = np.nan): + """ + :param t0: start time + :param t1: end time + :param text: text + :param probability: Confidence score for the segment, computed as the geometric mean of + the token probabilities for the segment (NaN if not calculated). + This makes it interpretable as a probability in [0, 1]. + """ + self.t0 = t0 + self.t1 = t1 + self.text = text + self.probability = probability + + def __str__(self): + return f"t0={self.t0}, t1={self.t1}, text={self.text}, probability={self.probability}" + + def __repr__(self): + return str(self) + + +class BaseModel(ABC): + """ + Abstract base class for all transcription models (whisper, parakeet, etc.). + Defines the public contract that every model must implement. + """ + + @abstractmethod + def transcribe(self, media: Union[str, np.ndarray], **params) -> List[Segment]: + """ + Transcribe audio media and return a list of segments. + :param media: file path or numpy array of audio data + :return: list of Segment objects + """ + ... diff --git a/pywhispercpp/model.py b/pywhispercpp/model.py index 08be6bd..53ed530 100644 --- a/pywhispercpp/model.py +++ b/pywhispercpp/model.py @@ -6,13 +6,8 @@ [whisper.cpp](https://github.com/ggerganov/whisper.cpp) API. """ import importlib.metadata -import subprocess -import os import logging -import shutil import sys -import tempfile -import wave from pathlib import Path from time import time from typing import Any, Union, Callable, List, TextIO, Tuple, Optional, Dict, TypedDict @@ -21,6 +16,7 @@ import numpy as np import pywhispercpp.constants as constants import pywhispercpp.utils as utils +from pywhispercpp.base import BaseModel, Segment __author__ = "absadiki" __copyright__ = "Copyright 2023, " @@ -43,33 +39,7 @@ class ContextParams(TypedDict, total=False): _CONTEXT_PARAM_KEYS = frozenset(ContextParams.__annotations__) -class Segment: - """ - A small class representing a transcription segment - """ - - def __init__(self, t0: int, t1: int, text: str, probability: float = np.nan): - """ - :param t0: start time - :param t1: end time - :param text: text - :param probability: Confidence score for the segment, computed as the geometric mean of - the token probabilities for the segment (NaN if not calculated). - This makes it interpretable as a probability in [0, 1]. - """ - self.t0 = t0 - self.t1 = t1 - self.text = text - self.probability = probability - - def __str__(self): - return f"t0={self.t0}, t1={self.t1}, text={self.text}, probability={self.probability}" - - def __repr__(self): - return str(self) - - -class Model: +class Model(BaseModel): """ This classes defines a Whisper.cpp model. @@ -82,8 +52,6 @@ class Model: ``` """ - - def __init__(self, model: str = 'tiny', models_dir: Optional[str] = None, @@ -204,7 +172,7 @@ def transcribe(self, else: if not Path(media).exists(): raise FileNotFoundError(media) - audio = self._load_audio(media) + audio = utils.load_audio(media, sample_rate=pw.WHISPER_SAMPLE_RATE) # update params if any self._set_params(params) @@ -266,23 +234,6 @@ def _get_segments(ctx, start: int, end: int, extract_probability: bool = False) res.append(Segment(t0, t1, text.strip(), probability=float(avg_prob))) return res - def get_params(self) -> dict: - """ - Returns a `dict` representation of the actual params - - :return: params dict - """ - res = {} - for param in dir(self._params): - if param.startswith('__'): - continue - try: - res[param] = getattr(self._params, param) - except Exception: - # ignore callback functions - continue - return res - @staticmethod def get_params_schema() -> dict: """ @@ -435,63 +386,6 @@ def __call_new_segment_callback(self, ctx, n_new, user_data=None) -> None: if self._new_segment_callback is not None: self._new_segment_callback(segment) - @staticmethod - def _load_audio(media_file_path: str) -> np.ndarray: - """ - Helper method to return a `np.array` object from a media file - If the media file is not a WAV file, it will try to convert it using ffmpeg - - :param media_file_path: Path of the media file - :return: Numpy array - """ - - def wav_to_np(file_path): - with wave.open(file_path, 'rb') as wf: - num_channels = wf.getnchannels() - sample_width = wf.getsampwidth() - sample_rate = wf.getframerate() - num_frames = wf.getnframes() - - if num_channels not in (1, 2): - raise Exception(f"WAV file must be mono or stereo") - - if sample_rate != pw.WHISPER_SAMPLE_RATE: - raise Exception(f"WAV file must be {pw.WHISPER_SAMPLE_RATE} Hz") - - if sample_width != 2: - raise Exception(f"WAV file must be 16-bit") - - raw = wf.readframes(num_frames) - wf.close() - audio = np.frombuffer(raw, dtype=np.int16).astype(np.float32) - n = num_frames - if num_channels == 1: - pcmf32 = audio / 32768.0 - else: - audio = audio.reshape(-1, 2) - # Averaging the two channels - pcmf32 = (audio[:, 0] + audio[:, 1]) / 65536.0 - return pcmf32 - - if media_file_path.endswith('.wav'): - return wav_to_np(media_file_path) - else: - if shutil.which('ffmpeg') is None: - raise Exception( - "FFMPEG is not installed or not in PATH. Please install it, or provide a WAV file or a NumPy array instead!") - - temp_file = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) - temp_file_path = temp_file.name - temp_file.close() - try: - subprocess.run([ - 'ffmpeg', '-i', media_file_path, '-ac', '1', '-ar', '16000', - temp_file_path, '-y' - ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - return wav_to_np(temp_file_path) - finally: - os.remove(temp_file_path) - def auto_detect_language(self, media: Union[str, np.ndarray], offset_ms: Optional[int] = None, n_threads: Optional[int] = None) -> Tuple[Tuple[str, np.float32], Dict[str, np.float32]]: """ Automatic language detection using whisper.cpp/whisper_pcm_to_mel and whisper.cpp/whisper_lang_auto_detect @@ -506,7 +400,7 @@ def auto_detect_language(self, media: Union[str, np.ndarray], offset_ms: Optiona else: if not Path(media).exists(): raise FileNotFoundError(media) - audio = self._load_audio(media) + audio = utils.load_audio(media, sample_rate=pw.WHISPER_SAMPLE_RATE) if offset_ms is None: offset_ms = self._params.offset_ms diff --git a/pywhispercpp/utils.py b/pywhispercpp/utils.py index 3d3b6ff..1e77492 100644 --- a/pywhispercpp/utils.py +++ b/pywhispercpp/utils.py @@ -7,10 +7,15 @@ import contextlib import logging import os +import shutil +import subprocess import sys +import tempfile +import wave from pathlib import Path from typing import TextIO +import numpy as np import requests from tqdm import tqdm @@ -280,3 +285,60 @@ def _resolve_target(target): finally: if should_close: stream.close() + + +def load_audio(media_file_path: str, sample_rate: int = 16000) -> np.ndarray: + """ + Load audio from a media file and return a numpy array. + + If the file is WAV and matches the expected format, it is read directly. + Otherwise, ffmpeg is used to convert it to the required format. + + :param media_file_path: Path to the audio/video file. + :param sample_rate: Expected sample rate in Hz. Default is 16000. + :return: numpy array of float32 audio samples. + """ + + def wav_to_np(file_path): + with wave.open(file_path, 'rb') as wf: + num_channels = wf.getnchannels() + sample_width = wf.getsampwidth() + file_sample_rate = wf.getframerate() + num_frames = wf.getnframes() + + if num_channels not in (1, 2): + raise Exception("WAV file must be mono or stereo") + + if file_sample_rate != sample_rate: + raise Exception(f"WAV file must be {sample_rate} Hz, got {file_sample_rate} Hz") + + if sample_width != 2: + raise Exception("WAV file must be 16-bit") + + raw = wf.readframes(num_frames) + audio = np.frombuffer(raw, dtype=np.int16).astype(np.float32) + if num_channels == 1: + pcmf32 = audio / 32768.0 + else: + audio = audio.reshape(-1, 2) + pcmf32 = (audio[:, 0] + audio[:, 1]) / 65536.0 + return pcmf32 + + if media_file_path.endswith('.wav'): + return wav_to_np(media_file_path) + + if shutil.which('ffmpeg') is None: + raise Exception( + "FFMPEG is not installed or not in PATH. Please install it, or provide a WAV file or a NumPy array instead!") + + temp_file = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) + temp_file_path = temp_file.name + temp_file.close() + try: + subprocess.run([ + 'ffmpeg', '-i', media_file_path, '-ac', '1', '-ar', str(sample_rate), + temp_file_path, '-y' + ], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return wav_to_np(temp_file_path) + finally: + os.remove(temp_file_path) From 7d2fd2416e33437cca96c05f438125cda096f50b Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Mon, 27 Jul 2026 22:39:08 -0400 Subject: [PATCH 06/14] feat: add Python API for parakeet model --- pywhispercpp/parakeet_model.py | 204 +++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 pywhispercpp/parakeet_model.py diff --git a/pywhispercpp/parakeet_model.py b/pywhispercpp/parakeet_model.py new file mode 100644 index 0000000..14c9abc --- /dev/null +++ b/pywhispercpp/parakeet_model.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Python API for Parakeet TDT models +""" +import logging +from dataclasses import dataclass, fields +from pathlib import Path +from time import time +from typing import List, Optional, Union + +import _pywhispercpp as pw +import numpy as np + +import pywhispercpp.utils as utils +from pywhispercpp.base import BaseModel, Segment + +logger = logging.getLogger(__name__) + + +@dataclass +class ParakeetContextParams: + """ + Parameters for model context initialization. + """ + use_gpu: Optional[bool] = None + """Use GPU for inference.""" + + gpu_device: Optional[int] = None + """GPU device index.""" + + def apply(self, c_params) -> None: + """Apply non-None fields to the C ``parakeet_context_params`` struct.""" + for f in fields(self): + val = getattr(self, f.name) + if val is not None: + setattr(c_params, f.name, val) + + +@dataclass +class ParakeetParams: + """ + Parameters for Parakeet transcription. + + These map directly to ``parakeet_full_params`` fields in the C library. + """ + n_threads: Optional[int] = None + """Number of inference threads.""" + + offset_ms: Optional[int] = None + """Start offset in milliseconds.""" + + duration_ms: Optional[int] = None + """Audio duration to process in milliseconds.""" + + no_context: Optional[bool] = None + """Disable reuse of past transcription context.""" + + audio_ctx: Optional[int] = None + """Override audio context size (0 = use default).""" + + def apply(self, c_params) -> None: + """Apply non-None fields to the C ``parakeet_full_params`` struct.""" + for f in fields(self): + val = getattr(self, f.name) + if val is not None: + setattr(c_params, f.name, val) + + +class ParakeetModel(BaseModel): + """ + A Parakeet TDT model backed by parakeet.cpp. + + Example usage:: + + model = ParakeetModel('path/to/parakeet.gguf') + segments = model.transcribe('audio.wav') + for segment in segments: + print(segment.text) + """ + + def __init__(self, + model: str, + models_dir: Optional[str] = None, + redirect_logs_to: Union[bool, None] = False, + context_params: Optional[ParakeetContextParams] = None, + params: Optional[ParakeetParams] = None, + **kwargs): + """ + :param model: path to a Parakeet GGUF model file, or a model name to resolve/download. + :param models_dir: directory to search for local model files. Defaults to ``MODELS_DIR`` + from constants. If the model is not found locally, it will be downloaded and stored + here. Ignored when *model* is a direct file path. + :param redirect_logs_to: log redirection target. ``False`` for no redirection, ``None`` for /dev/null. + :param context_params: optional context params (GPU settings). + :param params: a ``ParakeetParams`` instance for default decode settings. + :param kwargs: override params as keyword arguments (e.g. ``n_threads=8``). + """ + self.model_path = utils.resolve_model_path(model, models_dir) + self._ctx = None + self._context_params = self._resolve_context_params(context_params) + self._params = pw.parakeet_full_default_params( + pw.parakeet_sampling_strategy.PARAKEET_SAMPLING_GREEDY + ) + + if params is None: + params = ParakeetParams(**kwargs) + params.apply(self._params) + + self._redirect_logs_to = redirect_logs_to + self._new_segment_callback = None + self._init_model() + + def transcribe(self, + media: Union[str, np.ndarray], + new_segment_callback=None, + abort_callback=None, + params: Optional[ParakeetParams] = None, + **kwargs) -> List[Segment]: + """ + Transcribe audio and return a list of ``Segment`` objects. + + :param media: file path or numpy array of audio data. + :param new_segment_callback: callback invoked for each new segment. + :param abort_callback: callback returning ``True`` to abort. + :param params: optional ``ParakeetParams`` overrides for this call only. + :param kwargs: override params as keyword arguments. + :return: list of transcription segments. + """ + if isinstance(media, np.ndarray): + audio = media + else: + if not Path(media).exists(): + raise FileNotFoundError(media) + audio = utils.load_audio(media, sample_rate=pw.PARAKEET_SAMPLE_RATE) + + if params is not None: + params.apply(self._params) + elif kwargs: + ParakeetParams(**kwargs).apply(self._params) + + self._new_segment_callback = new_segment_callback + pw.parakeet_assign_new_segment_callback( + self._params, + self._on_new_segment if new_segment_callback is not None else None, + ) + + pw.parakeet_assign_abort_callback(self._params, abort_callback) + + start_time = time() + logger.info("Transcribing ...") + pw.parakeet_full(self._ctx, self._params, audio, audio.size) + n = pw.parakeet_full_n_segments(self._ctx) + segments = self._get_segments(self._ctx, 0, n) + end_time = time() + logger.info(f"Inference time: {end_time - start_time:.3f} s") + return segments + + def print_timings(self) -> None: + pw.parakeet_print_timings(self._ctx) + + def print_system_info(self) -> None: + pw.parakeet_print_system_info() + + def _init_model(self) -> None: + logger.info("Initializing parakeet model ...") + with utils.redirect_stderr(to=self._redirect_logs_to): + self._ctx = pw.parakeet_init_from_file_with_params( + self.model_path, self._context_params + ) + + def _on_new_segment(self, ctx, n_new, user_data=None) -> None: + n = pw.parakeet_full_n_segments(ctx) + start = n - n_new + segments = self._get_segments(ctx, start, n) + for segment in segments: + if self._new_segment_callback is not None: + self._new_segment_callback(segment) + + @staticmethod + def _get_segments(ctx, start: int, end: int) -> List[Segment]: + n = pw.parakeet_full_n_segments(ctx) + assert end <= n, f"{end} > {n}: end index exceeds segment count" + res = [] + for i in range(start, end): + t0 = pw.parakeet_full_get_segment_t0(ctx, i) + t1 = pw.parakeet_full_get_segment_t1(ctx, i) + text = pw.parakeet_full_get_segment_text(ctx, i) + if isinstance(text, bytes): + text = text.decode('utf-8', errors='replace') + res.append(Segment(t0, t1, text.strip())) + return res + + @staticmethod + def _resolve_context_params(context_params: Optional[ParakeetContextParams]): + resolved = pw.parakeet_context_default_params() + if context_params is not None: + context_params.apply(resolved) + return resolved + + def __del__(self): + if self._ctx is not None: + pw.parakeet_free(self._ctx) From 3ed2322d8744cf46e705d3855e8552a37e566e8a Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Tue, 28 Jul 2026 19:44:04 -0400 Subject: [PATCH 07/14] refactor: make model download utilities model-agnostic --- pywhispercpp/utils.py | 55 +++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/pywhispercpp/utils.py b/pywhispercpp/utils.py index 1e77492..7472f3d 100644 --- a/pywhispercpp/utils.py +++ b/pywhispercpp/utils.py @@ -20,7 +20,6 @@ from tqdm import tqdm from pywhispercpp.constants import ( - AVAILABLE_MODELS, MODELS_BASE_URL, MODELS_DIR, MODELS_PREFIX_URL, @@ -29,34 +28,36 @@ logger = logging.getLogger(__name__) -def _get_model_url(model_name: str) -> str: +def _get_model_url(base_url: str, prefix_url: str, model_name: str) -> str: """ - Returns the url of the `ggml` model - :param model_name: name of the model - :return: URL of the model + Build the download URL for a model. + :param base_url: base URL of the model repository. + :param prefix_url: path prefix before the model name. + :param model_name: name of the model. + :return: full download URL. """ - return f"{MODELS_BASE_URL}/{MODELS_PREFIX_URL}-{model_name}.bin" + return f"{base_url}/{prefix_url}-{model_name}.bin" -def download_model(model_name: str, download_dir=None, chunk_size=1024) -> str: +def download_model(model_name: str, download_dir=None, chunk_size=1024, + base_url=MODELS_BASE_URL, prefix_url=MODELS_PREFIX_URL) -> str: """ - Helper function to download the `ggml` models - :param model_name: name of the model, one of ::: constants.AVAILABLE_MODELS - :param download_dir: Where to store the models - :param chunk_size: size of the download chunk - - :return: Absolute path of the downloaded model + Download a model file from a remote repository. + + :param model_name: name of the model (used to build the download URL). + :param download_dir: directory to store the model. Defaults to MODELS_DIR. + :param chunk_size: size of the download chunk. + :param base_url: base URL of the model repository. + :param prefix_url: path prefix before the model name. + :return: Absolute path of the downloaded model. """ - if model_name not in AVAILABLE_MODELS: - logger.error(f"Invalid model name `{model_name}`, available models are: {AVAILABLE_MODELS}") - return if download_dir is None: download_dir = MODELS_DIR logger.info(f"No download directory was provided, models will be downloaded to {download_dir}") os.makedirs(download_dir, exist_ok=True) - url = _get_model_url(model_name=model_name) + url = _get_model_url(base_url=base_url, prefix_url=prefix_url, model_name=model_name) file_path = Path(download_dir) / os.path.basename(url) # check if the file is already there if file_path.exists(): @@ -85,17 +86,21 @@ def download_model(model_name: str, download_dir=None, chunk_size=1024) -> str: return str(file_path.absolute()) -def resolve_model_path(model_name: str, models_dir=None) -> str: +def resolve_model_path(model_name: str, models_dir=None, + base_url=MODELS_BASE_URL, prefix_url=MODELS_PREFIX_URL) -> str: """ Resolve a model name to a local model file. Resolution order: - 1. If `model_name` is an existing file path, return it. - 2. Look for `model_name` and `model_name.bin` in `models_dir`. - 3. If no local file is found, fall back to downloading a built-in model. - - :param model_name: A built-in model name, a custom model name, or a direct path to a model file. - :param models_dir: Directory to search for local models before downloading. Defaults to `MODELS_DIR`. + 1. If ``model_name`` is an existing file path, return it. + 2. Look for ``model_name`` and ``model_name.bin`` in ``models_dir``. + 3. If no local file is found, download it from the remote repository. + + :param model_name: A direct file path or a model name to resolve/download. + :param models_dir: Directory to search for local models before downloading. + Defaults to ``MODELS_DIR``. + :param base_url: Base URL of the model repository for downloading. + :param prefix_url: Path prefix before the model name for downloading. :return: Absolute path to the resolved model file. """ if Path(model_name).is_file(): @@ -112,7 +117,7 @@ def resolve_model_path(model_name: str, models_dir=None) -> str: if candidate.is_file(): return str(candidate.resolve()) - return download_model(model_name, search_dir) + return download_model(model_name, search_dir, base_url=base_url, prefix_url=prefix_url) def to_timestamp(t: int, separator=',') -> str: From 17088179006b3f94e022ed569767eb5665c009b7 Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Tue, 28 Jul 2026 21:06:45 -0400 Subject: [PATCH 08/14] feat: add AvailableModels enum and auto-download URLs to ParakeetModel --- pywhispercpp/parakeet_model.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pywhispercpp/parakeet_model.py b/pywhispercpp/parakeet_model.py index 14c9abc..1b63c54 100644 --- a/pywhispercpp/parakeet_model.py +++ b/pywhispercpp/parakeet_model.py @@ -6,6 +6,7 @@ """ import logging from dataclasses import dataclass, fields +from enum import StrEnum from pathlib import Path from time import time from typing import List, Optional, Union @@ -78,8 +79,23 @@ class ParakeetModel(BaseModel): segments = model.transcribe('audio.wav') for segment in segments: print(segment.text) + + Or use a model name to download automatically:: + + model = ParakeetModel(ParakeetModel.AvailableModels.TDT_0_6B_V3_Q4_0) """ + class AvailableModels(StrEnum): + """Available Parakeet model names for auto-download.""" + TDT_0_6B_V3_F16 = "parakeet-tdt-0.6b-v3-f16" + TDT_0_6B_V3_F32 = "parakeet-tdt-0.6b-v3-f32" + TDT_0_6B_V3_Q4_0 = "parakeet-tdt-0.6b-v3-q4_0" + TDT_0_6B_V3_Q4_K = "parakeet-tdt-0.6b-v3-q4_k" + TDT_0_6B_V3_Q8_0 = "parakeet-tdt-0.6b-v3-q8_0" + + _MODELS_BASE_URL = "https://huggingface.co/ggml-org/parakeet-GGUF" + _MODELS_PREFIX_URL = "resolve/main/ggml" + def __init__(self, model: str, models_dir: Optional[str] = None, @@ -89,6 +105,9 @@ def __init__(self, **kwargs): """ :param model: path to a Parakeet GGUF model file, or a model name to resolve/download. + Use ``ParakeetModel.AvailableModels`` for available model names (e.g. + ``ParakeetModel.AvailableModels.TDT_0_6B_V3_Q4_0``). If the model is not + found locally, it will be downloaded automatically. :param models_dir: directory to search for local model files. Defaults to ``MODELS_DIR`` from constants. If the model is not found locally, it will be downloaded and stored here. Ignored when *model* is a direct file path. @@ -97,7 +116,11 @@ def __init__(self, :param params: a ``ParakeetParams`` instance for default decode settings. :param kwargs: override params as keyword arguments (e.g. ``n_threads=8``). """ - self.model_path = utils.resolve_model_path(model, models_dir) + self.model_path = utils.resolve_model_path( + model, models_dir, + base_url=self._MODELS_BASE_URL, + prefix_url=self._MODELS_PREFIX_URL, + ) self._ctx = None self._context_params = self._resolve_context_params(context_params) self._params = pw.parakeet_full_default_params( From e6bb16a062884d7d0690be320a73dcbae9551903 Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Tue, 28 Jul 2026 21:26:41 -0400 Subject: [PATCH 09/14] test: add tests for Parakeet --- tests/test_parakeet_model.py | 233 +++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 tests/test_parakeet_model.py diff --git a/tests/test_parakeet_model.py b/tests/test_parakeet_model.py new file mode 100644 index 0000000..7fb869c --- /dev/null +++ b/tests/test_parakeet_model.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Test parakeet_model.py +""" +import unittest +from pathlib import Path +from unittest import TestCase + +import _pywhispercpp as pw +import numpy as np +import pywhispercpp.utils as utils +from pywhispercpp.base import Segment +from pywhispercpp.parakeet_model import ( + ParakeetModel, + ParakeetParams, + ParakeetContextParams, +) + +WHISPER_CPP_DIR = Path(__file__).parent.parent / 'whisper.cpp' + + +class TestParakeetParams(TestCase): + """Tests for ParakeetParams dataclass.""" + + def test_default_values(self): + params = ParakeetParams() + self.assertIsNone(params.n_threads) + self.assertIsNone(params.offset_ms) + self.assertIsNone(params.duration_ms) + self.assertIsNone(params.no_context) + self.assertIsNone(params.audio_ctx) + + def test_custom_values(self): + params = ParakeetParams( + n_threads=8, offset_ms=100, duration_ms=5000, + no_context=True, audio_ctx=256, + ) + self.assertEqual(params.n_threads, 8) + self.assertEqual(params.offset_ms, 100) + self.assertEqual(params.duration_ms, 5000) + self.assertTrue(params.no_context) + self.assertEqual(params.audio_ctx, 256) + + def test_apply_to_c_params(self): + c_params = pw.parakeet_full_default_params( + pw.parakeet_sampling_strategy.PARAKEET_SAMPLING_GREEDY + ) + params = ParakeetParams(n_threads=16) + params.apply(c_params) + self.assertEqual(c_params.n_threads, 16) + + def test_apply_partial_params(self): + c_params = pw.parakeet_full_default_params( + pw.parakeet_sampling_strategy.PARAKEET_SAMPLING_GREEDY + ) + params = ParakeetParams(no_context=True) + params.apply(c_params) + self.assertTrue(c_params.no_context) + + def test_apply_ignores_none_values(self): + c_params = pw.parakeet_full_default_params( + pw.parakeet_sampling_strategy.PARAKEET_SAMPLING_GREEDY + ) + original_n_threads = c_params.n_threads + params = ParakeetParams(n_threads=None, offset_ms=None) + params.apply(c_params) + self.assertEqual(c_params.n_threads, original_n_threads) + + +class TestParakeetContextParams(TestCase): + """Tests for ParakeetContextParams dataclass.""" + + def test_default_values(self): + params = ParakeetContextParams() + self.assertIsNone(params.use_gpu) + self.assertIsNone(params.gpu_device) + + def test_custom_values(self): + params = ParakeetContextParams(use_gpu=True, gpu_device=0) + self.assertTrue(params.use_gpu) + self.assertEqual(params.gpu_device, 0) + + def test_apply_to_c_params(self): + c_params = pw.parakeet_context_default_params() + params = ParakeetContextParams(use_gpu=False) + params.apply(c_params) + self.assertFalse(c_params.use_gpu) + + def test_apply_ignores_none_values(self): + c_params = pw.parakeet_context_default_params() + original_use_gpu = c_params.use_gpu + params = ParakeetContextParams(use_gpu=None) + params.apply(c_params) + self.assertEqual(c_params.use_gpu, original_use_gpu) + + +class TestParakeetModel(TestCase): + audio_file = WHISPER_CPP_DIR / 'samples/jfk.wav' + model = ParakeetModel( + str(WHISPER_CPP_DIR / 'models/for-tests-ggml-parakeet-tdt.bin'), + redirect_logs_to=None, + ) + + def test_transcribe_returns_list(self): + segments = self.model.transcribe(str(self.audio_file)) + self.assertIsInstance(segments, list) + + def test_transcribe_numpy_array(self): + audio = utils.load_audio(str(self.audio_file), sample_rate=pw.PARAKEET_SAMPLE_RATE) + segments = self.model.transcribe(audio) + self.assertIsInstance(segments, list) + + def test_transcribe_file_not_found(self): + with self.assertRaises(FileNotFoundError): + self.model.transcribe('/nonexistent/file.wav') + + def test_transcribe_with_params(self): + params = ParakeetParams(n_threads=4) + segments = self.model.transcribe(str(self.audio_file), params=params) + self.assertIsInstance(segments, list) + + def test_transcribe_with_kwargs(self): + segments = self.model.transcribe(str(self.audio_file), n_threads=4) + self.assertIsInstance(segments, list) + + def test_transcribe_with_new_segment_callback(self): + callback_segments = [] + self.model.transcribe( + str(self.audio_file), + new_segment_callback=lambda seg: callback_segments.append(seg) + ) + self.assertIsInstance(callback_segments, list) + + def test_transcribe_with_abort_callback(self): + segments = self.model.transcribe( + str(self.audio_file), + abort_callback=lambda: False + ) + self.assertIsInstance(segments, list) + + def test_model_init_with_params(self): + params = ParakeetParams(n_threads=2) + model = ParakeetModel( + str(WHISPER_CPP_DIR / 'models/for-tests-ggml-parakeet-tdt.bin'), + params=params, + redirect_logs_to=None, + ) + self.assertIsNotNone(model._ctx) + + def test_model_init_with_kwargs(self): + model = ParakeetModel( + str(WHISPER_CPP_DIR / 'models/for-tests-ggml-parakeet-tdt.bin'), + n_threads=2, + redirect_logs_to=None, + ) + self.assertIsNotNone(model._ctx) + + def test_model_init_with_context_params(self): + context_params = ParakeetContextParams(use_gpu=False) + model = ParakeetModel( + str(WHISPER_CPP_DIR / 'models/for-tests-ggml-parakeet-tdt.bin'), + context_params=context_params, + redirect_logs_to=None, + ) + self.assertIsNotNone(model._ctx) + + def test_print_timings(self): + self.model.transcribe(str(self.audio_file)) + self.model.print_timings() + + def test_print_system_info(self): + self.model.print_system_info() + + def test_model_metadata(self): + ctx = self.model._ctx + self.assertGreater(pw.parakeet_model_n_vocab(ctx), 0) + self.assertGreater(pw.parakeet_model_n_audio_ctx(ctx), 0) + self.assertGreater(pw.parakeet_model_n_audio_state(ctx), 0) + self.assertGreater(pw.parakeet_model_n_audio_head(ctx), 0) + self.assertGreater(pw.parakeet_model_n_audio_layer(ctx), 0) + self.assertGreater(pw.parakeet_model_n_mels(ctx), 0) + + +class TestParakeetCAPI(TestCase): + """Tests for low-level parakeet C bindings.""" + + model_file = str(WHISPER_CPP_DIR / 'models/for-tests-ggml-parakeet-tdt.bin') + + def test_parakeet_version(self): + version = pw.parakeet_version() + self.assertIsInstance(version, str) + self.assertGreater(len(version), 0) + + def test_parakeet_context_default_params(self): + params = pw.parakeet_context_default_params() + self.assertTrue(hasattr(params, 'use_gpu')) + self.assertTrue(hasattr(params, 'gpu_device')) + + def test_parakeet_full_default_params(self): + params = pw.parakeet_full_default_params( + pw.parakeet_sampling_strategy.PARAKEET_SAMPLING_GREEDY + ) + self.assertIsNotNone(params) + + def test_parakeet_init_from_file(self): + ctx = pw.parakeet_init_from_file_with_params( + self.model_file, + pw.parakeet_context_default_params(), + ) + self.assertIsNotNone(ctx) + pw.parakeet_free(ctx) + + def test_parakeet_n_vocab(self): + ctx = pw.parakeet_init_from_file_with_params( + self.model_file, + pw.parakeet_context_default_params(), + ) + self.assertGreater(pw.parakeet_n_vocab(ctx), 0) + pw.parakeet_free(ctx) + + def test_parakeet_token_blank(self): + ctx = pw.parakeet_init_from_file_with_params( + self.model_file, + pw.parakeet_context_default_params(), + ) + self.assertIsInstance(pw.parakeet_token_blank(ctx), int) + pw.parakeet_free(ctx) + + +if __name__ == '__main__': + unittest.main() From 468b06e2ec452842f9c11968886a31e98c3c8ce7 Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Tue, 28 Jul 2026 21:46:00 -0400 Subject: [PATCH 10/14] fix: restore get_params and update _load_audio test for whisper Model --- pywhispercpp/model.py | 18 ++++++++++++++++++ tests/test_model.py | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pywhispercpp/model.py b/pywhispercpp/model.py index 53ed530..41d9b67 100644 --- a/pywhispercpp/model.py +++ b/pywhispercpp/model.py @@ -234,6 +234,24 @@ def _get_segments(ctx, start: int, end: int, extract_probability: bool = False) res.append(Segment(t0, t1, text.strip(), probability=float(avg_prob))) return res + + def get_params(self) -> dict: + """ + Returns a `dict` representation of the actual params + + :return: params dict + """ + res = {} + for param in dir(self._params): + if param.startswith('__'): + continue + try: + res[param] = getattr(self._params, param) + except Exception: + # ignore callback functions + continue + return res + @staticmethod def get_params_schema() -> dict: """ diff --git a/tests/test_model.py b/tests/test_model.py index b68f8a6..a493109 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -9,6 +9,7 @@ from unittest import TestCase import _pywhispercpp as pw +import pywhispercpp.utils as utils from pywhispercpp.model import Model, Segment if __name__ == '__main__': @@ -38,7 +39,7 @@ def test_available_languages(self): return self.assertIsInstance(av_langs, list) and self.assertGreater(len(av_langs), 1) def test__load_audio(self): - audio_arr = self.model._load_audio(str(self.audio_file)) + audio_arr = utils.load_audio(str(self.audio_file), sample_rate=pw.WHISPER_SAMPLE_RATE) return self.assertIsNotNone(audio_arr) def test_auto_detect_language(self): From a70b6a02ab8b57f33d699377af538ed88b44313a Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Tue, 28 Jul 2026 23:27:59 -0400 Subject: [PATCH 11/14] feat: add WhisperModel with typed dataclass params and deprecate old Model --- pywhispercpp/model.py | 9 + pywhispercpp/parakeet_model.py | 6 +- pywhispercpp/whisper_model.py | 473 +++++++++++++++++++++++++++++++++ tests/test_whisper_model.py | 323 ++++++++++++++++++++++ 4 files changed, 808 insertions(+), 3 deletions(-) create mode 100644 pywhispercpp/whisper_model.py create mode 100644 tests/test_whisper_model.py diff --git a/pywhispercpp/model.py b/pywhispercpp/model.py index 41d9b67..4d349fb 100644 --- a/pywhispercpp/model.py +++ b/pywhispercpp/model.py @@ -8,6 +8,7 @@ import importlib.metadata import logging import sys +import warnings from pathlib import Path from time import time from typing import Any, Union, Callable, List, TextIO, Tuple, Optional, Dict, TypedDict @@ -43,6 +44,9 @@ class Model(BaseModel): """ This classes defines a Whisper.cpp model. + .. deprecated:: + Use :class:`pywhispercpp.whisper_model.WhisperModel` instead. + Example usage. ```python model = Model('base.en', n_threads=6) @@ -125,6 +129,11 @@ def __init__(self, - `vad`: enable VAD. Default `False`. - `vad_model_path`: path to the VAD model. Default `None`. """ + warnings.warn( + "Model is deprecated, use pywhispercpp.whisper_model.WhisperModel instead.", + DeprecationWarning, + stacklevel=2, + ) self.model_path = utils.resolve_model_path(model, models_dir) self._ctx = None self._context_params = self._resolve_context_params(context_params) diff --git a/pywhispercpp/parakeet_model.py b/pywhispercpp/parakeet_model.py index 1b63c54..eb0b734 100644 --- a/pywhispercpp/parakeet_model.py +++ b/pywhispercpp/parakeet_model.py @@ -9,7 +9,7 @@ from enum import StrEnum from pathlib import Path from time import time -from typing import List, Optional, Union +from typing import Callable, List, Optional, Union import _pywhispercpp as pw import numpy as np @@ -137,8 +137,8 @@ def __init__(self, def transcribe(self, media: Union[str, np.ndarray], - new_segment_callback=None, - abort_callback=None, + new_segment_callback: Optional[Callable[[Segment], None]] = None, + abort_callback: Optional[Callable[[], bool]] = None, params: Optional[ParakeetParams] = None, **kwargs) -> List[Segment]: """ diff --git a/pywhispercpp/whisper_model.py b/pywhispercpp/whisper_model.py new file mode 100644 index 0000000..658cdd7 --- /dev/null +++ b/pywhispercpp/whisper_model.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Python API for Whisper.cpp models +""" +import logging +from dataclasses import dataclass, fields +from enum import StrEnum +from pathlib import Path +from time import time +from typing import Callable, Dict, List, Optional, Tuple, Union + +import _pywhispercpp as pw +import numpy as np + +import pywhispercpp.utils as utils +from pywhispercpp.base import BaseModel, Segment + +logger = logging.getLogger(__name__) + + +@dataclass +class WhisperContextParams: + """ + Parameters for whisper context initialization. + """ + use_gpu: Optional[bool] = None + """Use GPU for inference.""" + + flash_attn: Optional[bool] = None + """Enable Flash Attention.""" + + gpu_device: Optional[int] = None + """GPU device index.""" + + dtw_token_timestamps: Optional[bool] = None + """Enable DTW token timestamps.""" + + dtw_aheads_preset: Optional[int] = None + """DTW aheads preset.""" + + dtw_n_top: Optional[int] = None + """DTW n_top.""" + + dtw_mem_size: Optional[int] = None + """DTW memory size.""" + + def apply(self, c_params) -> None: + """Apply non-None fields to the C ``whisper_context_params`` struct.""" + for f in fields(self): + val = getattr(self, f.name) + if val is not None: + setattr(c_params, f.name, val) + + +@dataclass +class WhisperParams: + """ + Parameters for Whisper transcription. + + These map directly to ``whisper_full_params`` fields in the C-API. + """ + n_threads: Optional[int] = None + """Number of inference threads.""" + + n_max_text_ctx: Optional[int] = None + """Max prompt-text tokens carried into the decoder.""" + + offset_ms: Optional[int] = None + """Start offset in milliseconds.""" + + duration_ms: Optional[int] = None + """Audio duration to process in milliseconds.""" + + translate: Optional[bool] = None + """Translate output to English.""" + + no_context: Optional[bool] = None + """Disable reuse of past transcription context.""" + + no_timestamps: Optional[bool] = None + """Disable timestamp generation.""" + + single_segment: Optional[bool] = None + """Force single segment output.""" + + print_special: Optional[bool] = None + """Print special tokens.""" + + print_progress: Optional[bool] = None + """Print progress information.""" + + print_realtime: Optional[bool] = None + """Print realtime output.""" + + print_timestamps: Optional[bool] = None + """Print timestamps during realtime output.""" + + token_timestamps: Optional[bool] = None + """Enable token-level timestamps.""" + + thold_pt: Optional[float] = None + """Token timestamp probability threshold.""" + + thold_ptsum: Optional[float] = None + """Token timestamp sum threshold.""" + + max_len: Optional[int] = None + """Max segment length in characters.""" + + split_on_word: Optional[bool] = None + """Split on words when max_len is used.""" + + max_tokens: Optional[int] = None + """Max tokens per segment (0 = no limit).""" + + debug_mode: Optional[bool] = None + """Enable whisper.cpp debug mode.""" + + audio_ctx: Optional[int] = None + """Override audio context size (0 = use default).""" + + tdrz_enable: Optional[bool] = None + """Enable tinydiarize speaker-turn detection.""" + + initial_prompt: Optional[str] = None + """Initial text prompt prepended before decoding.""" + + prompt_tokens: Optional[Tuple] = None + """Explicit prompt token sequence.""" + + prompt_n_tokens: Optional[int] = None + """Number of prompt tokens.""" + + carry_initial_prompt: Optional[bool] = None + """Prepend the initial prompt to each decode window.""" + + language: Optional[str] = None + """Language code (e.g. ``"en"``, ``"de"``).""" + + detect_language: Optional[bool] = None + """Enable automatic language detection.""" + + suppress_blank: Optional[bool] = None + """Suppress blank outputs.""" + + suppress_non_speech_tokens: Optional[bool] = None + """Alias for suppress_nst.""" + + suppress_nst: Optional[bool] = None + """Suppress non-speech tokens.""" + + suppress_regex: Optional[str] = None + """Regex pattern used to suppress matching text.""" + + temperature: Optional[float] = None + """Initial decoding temperature.""" + + max_initial_ts: Optional[float] = None + """Maximum initial timestamp.""" + + length_penalty: Optional[float] = None + """Length penalty.""" + + temperature_inc: Optional[float] = None + """Fallback temperature increment.""" + + entropy_thold: Optional[float] = None + """Entropy threshold.""" + + logprob_thold: Optional[float] = None + """Logprob threshold.""" + + no_speech_thold: Optional[float] = None + """No-speech threshold.""" + + greedy: Optional[dict] = None + """Greedy decoder settings (e.g. ``{"best_of": 5}``).""" + + beam_search: Optional[dict] = None + """Beam search settings (e.g. ``{"beam_size": -1, "patience": -1.0}``).""" + + vad: Optional[bool] = None + """Enable VAD.""" + + vad_model_path: Optional[str] = None + """Path to the VAD model.""" + + def apply(self, c_params) -> None: + """Apply non-None fields to the C ``whisper_full_params`` struct.""" + if self.suppress_non_speech_tokens is not None: + c_params.suppress_nst = self.suppress_non_speech_tokens + + if self.prompt_tokens is not None: + c_params.set_prompt_tokens(self.prompt_tokens) + + for f in fields(self): + if f.name in ('suppress_non_speech_tokens', 'prompt_tokens'): + continue + val = getattr(self, f.name) + if val is not None: + setattr(c_params, f.name, val) + + +class WhisperModel(BaseModel): + """ + A Whisper model Python API on top of whisper.cpp C-API. + + Example usage:: + + model = WhisperModel('path/to/tiny.bin', params=WhisperParams(n_threads=8)) + segments = model.transcribe('audio.wav') + for segment in segments: + print(segment.text) + + Or use a model name to download automatically:: + + model = WhisperModel(WhisperModel.AvailableModels.TINY) + """ + + class AvailableModels(StrEnum): + """Available Whisper model names for auto-download.""" + TINY = "tiny" + TINY_Q5_1 = "tiny-q5_1" + TINY_Q8_0 = "tiny-q8_0" + TINY_EN = "tiny.en" + TINY_EN_Q5_1 = "tiny.en-q5_1" + TINY_EN_Q8_0 = "tiny.en-q8_0" + BASE = "base" + BASE_Q5_1 = "base-q5_1" + BASE_Q8_0 = "base-q8_0" + BASE_EN = "base.en" + BASE_EN_Q5_1 = "base.en-q5_1" + BASE_EN_Q8_0 = "base.en-q8_0" + SMALL = "small" + SMALL_Q5_1 = "small-q5_1" + SMALL_Q8_0 = "small-q8_0" + SMALL_EN = "small.en" + SMALL_EN_Q5_1 = "small.en-q5_1" + SMALL_EN_Q8_0 = "small.en-q8_0" + MEDIUM = "medium" + MEDIUM_Q5_0 = "medium-q5_0" + MEDIUM_Q8_0 = "medium-q8_0" + MEDIUM_EN = "medium.en" + MEDIUM_EN_Q5_0 = "medium.en-q5_0" + MEDIUM_EN_Q8_0 = "medium.en-q8_0" + LARGE_V1 = "large-v1" + LARGE_V2 = "large-v2" + LARGE_V2_Q5_0 = "large-v2-q5_0" + LARGE_V2_Q8_0 = "large-v2-q8_0" + LARGE_V3 = "large-v3" + LARGE_V3_Q5_0 = "large-v3-q5_0" + LARGE_V3_TURBO = "large-v3-turbo" + LARGE_V3_TURBO_Q5_0 = "large-v3-turbo-q5_0" + LARGE_V3_TURBO_Q8_0 = "large-v3-turbo-q8_0" + + _MODELS_BASE_URL = "https://huggingface.co/ggerganov/whisper.cpp" + _MODELS_PREFIX_URL = "resolve/main/ggml" + + def __init__(self, + model: str = 'tiny', + models_dir: Optional[str] = None, + redirect_logs_to: Union[bool, None] = False, + context_params: Optional[WhisperContextParams] = None, + params: Optional[WhisperParams] = None, + sampling_strategy: int = 0, + use_openvino: bool = False, + openvino_model_path: Optional[str] = None, + openvino_device: str = 'CPU', + openvino_cache_dir: Optional[str] = None, + **kwargs): + """ + :param model: model name (e.g. ``"tiny"``, ``"base.en"``) or a direct path to a ggml/gguf model file. + Use ``WhisperModel.AvailableModels`` for available model names. If the model is not + found locally, it will be downloaded automatically. + :param models_dir: directory to search for local model files. Defaults to ``MODELS_DIR``. + Ignored when *model* is a direct file path. + :param redirect_logs_to: log redirection target. ``False`` for no redirection, ``None`` for /dev/null. + :param context_params: optional ``WhisperContextParams`` for GPU settings. + :param params: a ``WhisperParams`` instance for default decode settings. + :param sampling_strategy: ``0`` for greedy, ``1`` for beam search. + :param use_openvino: whether to initialize the OpenVINO encoder backend. + :param openvino_model_path: path to the OpenVINO model directory or files. + :param openvino_device: OpenVINO device name, default ``"CPU"``. + :param openvino_cache_dir: OpenVINO cache directory. + :param kwargs: override params as keyword arguments (e.g. ``n_threads=8``). + """ + self.model_path = utils.resolve_model_path( + model, models_dir, + base_url=self._MODELS_BASE_URL, + prefix_url=self._MODELS_PREFIX_URL, + ) + self._ctx = None + self._context_params = self._resolve_context_params(context_params) + self._sampling_strategy = ( + pw.whisper_sampling_strategy.WHISPER_SAMPLING_GREEDY + if sampling_strategy == 0 + else pw.whisper_sampling_strategy.WHISPER_SAMPLING_BEAM_SEARCH + ) + self._params = pw.whisper_full_default_params(self._sampling_strategy) + + if params is None: + params = WhisperParams(**kwargs) + params.apply(self._params) + + self._redirect_logs_to = redirect_logs_to + self._use_openvino = use_openvino + self._openvino_model_path = openvino_model_path + self._openvino_device = openvino_device + self._openvino_cache_dir = openvino_cache_dir + self._new_segment_callback = None + self._init_model() + + def transcribe(self, + media: Union[str, np.ndarray], + new_segment_callback: Optional[Callable[[Segment], None]] = None, + abort_callback: Optional[Callable[[], bool]] = None, + params: Optional[WhisperParams] = None, + n_processors: Optional[int] = None, + extract_probability: bool = False, + **kwargs) -> List[Segment]: + """ + Transcribe audio and return a list of ``Segment`` objects. + + :param media: file path or numpy array of audio data. + :param new_segment_callback: callback invoked for each new segment. + :param abort_callback: callback returning ``True`` to abort. + :param params: optional ``WhisperParams`` overrides for this call only. + :param n_processors: number of worker processes for ``whisper_full_parallel``. + If omitted, runs single-process ``whisper_full()``. + :param extract_probability: if ``True``, computes geometric mean of token probabilities + per segment as a confidence score in [0, 1]. + :param kwargs: override params as keyword arguments. + :return: list of transcription segments. + """ + if isinstance(media, np.ndarray): + audio = media + else: + if not Path(media).exists(): + raise FileNotFoundError(media) + audio = utils.load_audio(media, sample_rate=pw.WHISPER_SAMPLE_RATE) + + if params is not None: + params.apply(self._params) + elif kwargs: + WhisperParams(**kwargs).apply(self._params) + + self._new_segment_callback = new_segment_callback + pw.assign_new_segment_callback( + self._params, + self._on_new_segment if new_segment_callback is not None else None, + ) + + pw.assign_abort_callback(self._params, abort_callback) + + start_time = time() + logger.info("Transcribing ...") + if n_processors: + pw.whisper_full_parallel(self._ctx, self._params, audio, audio.size, n_processors) + else: + pw.whisper_full(self._ctx, self._params, audio, audio.size) + n = pw.whisper_full_n_segments(self._ctx) + segments = self._get_segments(self._ctx, 0, n, extract_probability) + end_time = time() + logger.info(f"Inference time: {end_time - start_time:.3f} s") + return segments + + def auto_detect_language(self, + media: Union[str, np.ndarray], + offset_ms: Optional[int] = None, + n_threads: Optional[int] = None) -> Tuple[Tuple[str, np.float32], Dict[str, np.float32]]: + """ + Automatic language detection using ``whisper_pcm_to_mel`` and ``whisper_lang_auto_detect``. + + :param media: file path or numpy array of audio data. + :param offset_ms: offset in milliseconds; when omitted, uses the model's current ``offset_ms``. + :param n_threads: number of threads; when omitted, uses the model's current ``n_threads``. + :return: ``((detected_language, probability), probabilities_for_all_languages)``. + """ + if isinstance(media, np.ndarray): + audio = media + else: + if not Path(media).exists(): + raise FileNotFoundError(media) + audio = utils.load_audio(media, sample_rate=pw.WHISPER_SAMPLE_RATE) + + if offset_ms is None: + offset_ms = self._params.offset_ms + + if n_threads is None: + n_threads = self._params.n_threads + + pw.whisper_pcm_to_mel(self._ctx, audio, len(audio), n_threads) + lang_count = self.lang_max_id() + 1 + probs = np.zeros(lang_count, dtype=np.float32) + auto_detect = pw.whisper_lang_auto_detect(self._ctx, offset_ms, n_threads, probs) + langs = self.available_languages() + lang_probs = {langs[i]: probs[i] for i in range(lang_count)} + return (langs[auto_detect], np.float32(probs[auto_detect])), lang_probs + + def print_timings(self) -> None: + pw.whisper_print_timings(self._ctx) + + @staticmethod + def print_system_info() -> None: + pw.whisper_print_system_info() + + @staticmethod + def lang_max_id() -> int: + return pw.whisper_lang_max_id() + + @staticmethod + def available_languages() -> List[str]: + n = pw.whisper_lang_max_id() + return [pw.whisper_lang_str(i) for i in range(n + 1)] + + def _init_model(self) -> None: + logger.info("Initializing whisper model ...") + with utils.redirect_stderr(to=self._redirect_logs_to): + self._ctx = pw.whisper_init_from_file_with_params( + self.model_path, self._context_params + ) + if self._use_openvino: + pw.whisper_ctx_init_openvino_encoder( + self._ctx, self._openvino_model_path, + self._openvino_device, self._openvino_cache_dir, + ) + + def _on_new_segment(self, ctx, n_new, user_data=None) -> None: + n = pw.whisper_full_n_segments(ctx) + start = n - n_new + segments = self._get_segments(ctx, start, n, False) + for segment in segments: + if self._new_segment_callback is not None: + self._new_segment_callback(segment) + + @staticmethod + def _get_segments(ctx, start: int, end: int, extract_probability: bool = False) -> List[Segment]: + n = pw.whisper_full_n_segments(ctx) + assert end <= n, f"{end} > {n}: end index exceeds segment count" + res = [] + for i in range(start, end): + t0 = pw.whisper_full_get_segment_t0(ctx, i) + t1 = pw.whisper_full_get_segment_t1(ctx, i) + text = pw.whisper_full_get_segment_text(ctx, i) + if isinstance(text, bytes): + text = text.decode('utf-8', errors='replace') + + avg_prob = np.nan + if extract_probability: + n_tokens = pw.whisper_full_n_tokens(ctx, i) + if n_tokens == 1: + avg_prob = pw.whisper_full_get_token_p(ctx, i, 0) + elif n_tokens > 1: + total_logprob = 0.0 + for j in range(n_tokens): + total_logprob += np.log(pw.whisper_full_get_token_p(ctx, i, j)) + avg_prob = np.exp(total_logprob / n_tokens) + + res.append(Segment(t0, t1, text.strip(), probability=float(avg_prob))) + return res + + @staticmethod + def _resolve_context_params(context_params: Optional[WhisperContextParams]): + resolved = pw.whisper_context_default_params() + if context_params is not None: + context_params.apply(resolved) + return resolved + + def __del__(self): + if self._ctx is not None: + pw.whisper_free(self._ctx) diff --git a/tests/test_whisper_model.py b/tests/test_whisper_model.py new file mode 100644 index 0000000..2967585 --- /dev/null +++ b/tests/test_whisper_model.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Test whisper_model.py +""" +import unittest +import warnings +from pathlib import Path +from unittest import TestCase + +import _pywhispercpp as pw +import numpy as np +import pywhispercpp.utils as utils +from pywhispercpp.base import Segment +from pywhispercpp.whisper_model import ( + WhisperModel, + WhisperParams, + WhisperContextParams, +) + +WHISPER_CPP_DIR = Path(__file__).parent.parent / 'whisper.cpp' + + +class TestWhisperParams(TestCase): + """Tests for WhisperParams dataclass.""" + + def test_default_values(self): + params = WhisperParams() + self.assertIsNone(params.n_threads) + self.assertIsNone(params.translate) + self.assertIsNone(params.language) + self.assertIsNone(params.temperature) + + def test_custom_values(self): + params = WhisperParams( + n_threads=8, translate=True, language='de', + temperature=0.5, no_context=False, + ) + self.assertEqual(params.n_threads, 8) + self.assertTrue(params.translate) + self.assertEqual(params.language, 'de') + self.assertEqual(params.temperature, 0.5) + self.assertFalse(params.no_context) + + def test_apply_to_c_params(self): + c_params = pw.whisper_full_default_params( + pw.whisper_sampling_strategy.WHISPER_SAMPLING_GREEDY + ) + params = WhisperParams(n_threads=16, language='fr') + params.apply(c_params) + self.assertEqual(c_params.n_threads, 16) + self.assertEqual(c_params.language, 'fr') + + def test_apply_partial_params(self): + c_params = pw.whisper_full_default_params( + pw.whisper_sampling_strategy.WHISPER_SAMPLING_GREEDY + ) + params = WhisperParams(no_context=True) + params.apply(c_params) + self.assertTrue(c_params.no_context) + + def test_apply_ignores_none_values(self): + c_params = pw.whisper_full_default_params( + pw.whisper_sampling_strategy.WHISPER_SAMPLING_GREEDY + ) + original_n_threads = c_params.n_threads + params = WhisperParams(n_threads=None) + params.apply(c_params) + self.assertEqual(c_params.n_threads, original_n_threads) + + def test_suppress_non_speech_tokens_alias(self): + c_params = pw.whisper_full_default_params( + pw.whisper_sampling_strategy.WHISPER_SAMPLING_GREEDY + ) + params = WhisperParams(suppress_non_speech_tokens=True) + params.apply(c_params) + self.assertTrue(c_params.suppress_nst) + + def test_prompt_tokens(self): + c_params = pw.whisper_full_default_params( + pw.whisper_sampling_strategy.WHISPER_SAMPLING_GREEDY + ) + params = WhisperParams(prompt_tokens=(1, 2, 3)) + params.apply(c_params) + self.assertEqual(c_params.prompt_n_tokens, 3) + + +class TestWhisperContextParams(TestCase): + """Tests for WhisperContextParams dataclass.""" + + def test_default_values(self): + params = WhisperContextParams() + self.assertIsNone(params.use_gpu) + self.assertIsNone(params.flash_attn) + self.assertIsNone(params.gpu_device) + + def test_custom_values(self): + params = WhisperContextParams(use_gpu=True, flash_attn=False, gpu_device=1) + self.assertTrue(params.use_gpu) + self.assertFalse(params.flash_attn) + self.assertEqual(params.gpu_device, 1) + + def test_apply_to_c_params(self): + c_params = pw.whisper_context_default_params() + params = WhisperContextParams(use_gpu=False) + params.apply(c_params) + self.assertFalse(c_params.use_gpu) + + def test_apply_ignores_none_values(self): + c_params = pw.whisper_context_default_params() + original_use_gpu = c_params.use_gpu + params = WhisperContextParams(use_gpu=None) + params.apply(c_params) + self.assertEqual(c_params.use_gpu, original_use_gpu) + + +class TestWhisperModel(TestCase): + audio_file = WHISPER_CPP_DIR / 'samples/jfk.wav' + model = WhisperModel("tiny", models_dir=str(WHISPER_CPP_DIR / 'models'), redirect_logs_to=None) + + def test_transcribe_returns_list(self): + segments = self.model.transcribe(str(self.audio_file)) + self.assertIsInstance(segments, list) + + def test_transcribe_returns_segments(self): + segments = self.model.transcribe(str(self.audio_file)) + if len(segments) > 0: + self.assertIsInstance(segments[0], Segment) + + def test_transcribe_jfk_content(self): + segments = self.model.transcribe(str(self.audio_file)) + full_text = ' '.join(seg.text for seg in segments).lower() + self.assertIn('fellow americans', full_text) + + def test_transcribe_numpy_array(self): + audio = utils.load_audio(str(self.audio_file), sample_rate=pw.WHISPER_SAMPLE_RATE) + segments = self.model.transcribe(audio) + self.assertIsInstance(segments, list) + if len(segments) > 0: + self.assertIsInstance(segments[0], Segment) + + def test_transcribe_file_not_found(self): + with self.assertRaises(FileNotFoundError): + self.model.transcribe('/nonexistent/file.wav') + + def test_transcribe_with_params(self): + params = WhisperParams(n_threads=4) + segments = self.model.transcribe(str(self.audio_file), params=params) + self.assertIsInstance(segments, list) + + def test_transcribe_with_kwargs(self): + segments = self.model.transcribe(str(self.audio_file), n_threads=4) + self.assertIsInstance(segments, list) + + def test_transcribe_with_new_segment_callback(self): + callback_segments = [] + self.model.transcribe( + str(self.audio_file), + new_segment_callback=lambda seg: callback_segments.append(seg) + ) + self.assertGreater(len(callback_segments), 0) + self.assertIsInstance(callback_segments[0], Segment) + + def test_transcribe_with_abort_callback(self): + segments = self.model.transcribe( + str(self.audio_file), + abort_callback=lambda: False + ) + self.assertIsInstance(segments, list) + + def test_transcribe_with_extract_probability(self): + segments = self.model.transcribe( + str(self.audio_file), + extract_probability=True + ) + self.assertIsInstance(segments, list) + if len(segments) > 0: + self.assertFalse(np.isnan(segments[0].probability)) + + def test_model_init_with_params(self): + params = WhisperParams(n_threads=2, language='en') + model = WhisperModel( + "tiny", + models_dir=str(WHISPER_CPP_DIR / 'models'), + params=params, + redirect_logs_to=None, + ) + self.assertIsNotNone(model._ctx) + + def test_model_init_with_kwargs(self): + model = WhisperModel( + "tiny", + models_dir=str(WHISPER_CPP_DIR / 'models'), + n_threads=2, + redirect_logs_to=None, + ) + self.assertIsNotNone(model._ctx) + + def test_model_init_with_context_params(self): + context_params = WhisperContextParams(use_gpu=False) + model = WhisperModel( + "tiny", + models_dir=str(WHISPER_CPP_DIR / 'models'), + context_params=context_params, + redirect_logs_to=None, + ) + self.assertIsNotNone(model._ctx) + + def test_model_init_beam_search(self): + model = WhisperModel( + "tiny", + models_dir=str(WHISPER_CPP_DIR / 'models'), + sampling_strategy=1, + redirect_logs_to=None, + ) + self.assertIsNotNone(model._ctx) + + def test_print_timings(self): + self.model.transcribe(str(self.audio_file)) + self.model.print_timings() + + def test_print_system_info(self): + self.model.print_system_info() + + def test_lang_max_id(self): + n = WhisperModel.lang_max_id() + self.assertGreater(n, 0) + + def test_available_languages(self): + langs = WhisperModel.available_languages() + self.assertIsInstance(langs, list) + self.assertGreater(len(langs), 1) + self.assertIn('en', langs) + + def test_auto_detect_language(self): + detected_language, probs = self.model.auto_detect_language(str(self.audio_file)) + self.assertIsInstance(detected_language, tuple) + self.assertEqual(detected_language[0], 'en') + + def test_auto_detect_language_numpy(self): + audio = utils.load_audio(str(self.audio_file), sample_rate=pw.WHISPER_SAMPLE_RATE) + detected_language, probs = self.model.auto_detect_language(audio) + self.assertEqual(detected_language[0], 'en') + + def test_model_metadata(self): + ctx = self.model._ctx + self.assertIsInstance(pw.whisper_model_type_readable(ctx), str) + self.assertGreater(pw.whisper_model_n_vocab(ctx), 0) + self.assertGreater(pw.whisper_model_n_audio_ctx(ctx), 0) + self.assertGreater(pw.whisper_model_n_text_ctx(ctx), 0) + + def test_speaker_turn_accessor(self): + self.model.transcribe(str(self.audio_file)) + segment_count = pw.whisper_full_n_segments(self.model._ctx) + self.assertGreater(segment_count, 0) + self.assertIsInstance( + pw.whisper_full_get_segment_speaker_turn_next(self.model._ctx, 0), + bool, + ) + + def test_compat_alias_for_non_speech_tokens(self): + model = WhisperModel( + "tiny", + models_dir=str(WHISPER_CPP_DIR / 'models'), + suppress_non_speech_tokens=True, + redirect_logs_to=None, + ) + self.assertTrue(model._params.suppress_nst) + + +class TestWhisperCAPI(TestCase): + """Tests for low-level whisper C bindings.""" + + model_file = str(WHISPER_CPP_DIR / 'models/for-tests-ggml-tiny.en.bin') + + def test_whisper_init_from_file(self): + ctx = pw.whisper_init_from_file_with_params( + self.model_file, + pw.whisper_context_default_params(), + ) + self.assertIsInstance(ctx, pw.whisper_context) + pw.whisper_free(ctx) + + def test_whisper_lang_str(self): + self.assertEqual(pw.whisper_lang_str(0), 'en') + + def test_whisper_lang_id(self): + self.assertEqual(pw.whisper_lang_id('en'), 0) + + def test_whisper_full_default_params(self): + params = pw.whisper_full_default_params( + pw.whisper_sampling_strategy.WHISPER_SAMPLING_GREEDY + ) + self.assertIsInstance(params, pw.whisper_full_params) + self.assertEqual(params.suppress_regex, "") + + def test_whisper_full_params_language_set(self): + params = pw.whisper_full_params() + params.language = 'de' + self.assertEqual(params.language, 'de') + + def test_whisper_full_params_prompt_tokens(self): + params = pw.whisper_full_params() + params.set_prompt_tokens((1, 2, 3)) + self.assertEqual(params.prompt_n_tokens, 3) + + +class TestDeprecationWarning(TestCase): + """Tests for deprecation warning on old Model class.""" + + def test_old_model_emits_deprecation_warning(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + from pywhispercpp.model import Model + Model("tiny", models_dir=str(WHISPER_CPP_DIR / 'models')) + deprecation_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)] + self.assertGreater(len(deprecation_warnings), 0) + self.assertIn("WhisperModel", str(deprecation_warnings[0].message)) + + +if __name__ == '__main__': + unittest.main() From b452e03eee5d98210b1a169420042204c16a5170 Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Wed, 29 Jul 2026 19:43:55 -0400 Subject: [PATCH 12/14] docs: add new models to mkdocs and fix docstrings --- docs/index.md | 4 ++++ pywhispercpp/parakeet_model.py | 12 ++++++------ pywhispercpp/whisper_model.py | 8 ++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/index.md b/docs/index.md index 9f4fdc5..921f728 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,10 @@ # PyWhisperCpp API Reference +::: pywhispercpp.whisper_model + +::: pywhispercpp.parakeet_model + ::: pywhispercpp.model ::: pywhispercpp.constants diff --git a/pywhispercpp/parakeet_model.py b/pywhispercpp/parakeet_model.py index eb0b734..28e4681 100644 --- a/pywhispercpp/parakeet_model.py +++ b/pywhispercpp/parakeet_model.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -Python API for Parakeet TDT models +Python API for the Parakeet TDT model on top of parakeet.h C-API """ import logging from dataclasses import dataclass, fields @@ -23,7 +23,7 @@ @dataclass class ParakeetContextParams: """ - Parameters for model context initialization. + Parameters for Parakeet context initialization. """ use_gpu: Optional[bool] = None """Use GPU for inference.""" @@ -71,11 +71,11 @@ def apply(self, c_params) -> None: class ParakeetModel(BaseModel): """ - A Parakeet TDT model backed by parakeet.cpp. + Python API for the Parakeet TDT model on top of parakeet.h C-API Example usage:: - model = ParakeetModel('path/to/parakeet.gguf') + model = ParakeetModel('path/to/parakeet_model.bin') segments = model.transcribe('audio.wav') for segment in segments: print(segment.text) @@ -112,8 +112,8 @@ def __init__(self, from constants. If the model is not found locally, it will be downloaded and stored here. Ignored when *model* is a direct file path. :param redirect_logs_to: log redirection target. ``False`` for no redirection, ``None`` for /dev/null. - :param context_params: optional context params (GPU settings). - :param params: a ``ParakeetParams`` instance for default decode settings. + :param context_params: optional ``ParakeetContextParams`` for context params. + :param params: a ``ParakeetParams`` instance for parakeet settings. :param kwargs: override params as keyword arguments (e.g. ``n_threads=8``). """ self.model_path = utils.resolve_model_path( diff --git a/pywhispercpp/whisper_model.py b/pywhispercpp/whisper_model.py index 658cdd7..1d39f2f 100644 --- a/pywhispercpp/whisper_model.py +++ b/pywhispercpp/whisper_model.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ -Python API for Whisper.cpp models +Python API for The Whisper model on top of whisper.h C-API. """ import logging from dataclasses import dataclass, fields @@ -205,7 +205,7 @@ def apply(self, c_params) -> None: class WhisperModel(BaseModel): """ - A Whisper model Python API on top of whisper.cpp C-API. + Python API for The Whisper model on top of whisper.h C-API. Example usage:: @@ -277,8 +277,8 @@ def __init__(self, :param models_dir: directory to search for local model files. Defaults to ``MODELS_DIR``. Ignored when *model* is a direct file path. :param redirect_logs_to: log redirection target. ``False`` for no redirection, ``None`` for /dev/null. - :param context_params: optional ``WhisperContextParams`` for GPU settings. - :param params: a ``WhisperParams`` instance for default decode settings. + :param context_params: optional ``WhisperContextParams`` for context settings. + :param params: a ``WhisperParams`` instance for whisper settings. :param sampling_strategy: ``0`` for greedy, ``1`` for beam search. :param use_openvino: whether to initialize the OpenVINO encoder backend. :param openvino_model_path: path to the OpenVINO model directory or files. From b040c3e494cf5a277083f0c54ef74b6dff5a2919 Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Wed, 29 Jul 2026 19:59:01 -0400 Subject: [PATCH 13/14] chore: export WhisperModel, ParakeetModel and params from package root --- pywhispercpp/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pywhispercpp/__init__.py b/pywhispercpp/__init__.py index 8b13789..c826001 100644 --- a/pywhispercpp/__init__.py +++ b/pywhispercpp/__init__.py @@ -1 +1,3 @@ - +from pywhispercpp.base import Segment +from pywhispercpp.whisper_model import WhisperModel, WhisperParams, WhisperContextParams +from pywhispercpp.parakeet_model import ParakeetModel, ParakeetParams, ParakeetContextParams From fa25df92d09ba8937e51f05d969f09b900f29a7e Mon Sep 17 00:00:00 2001 From: abdeladim-s Date: Wed, 29 Jul 2026 20:18:05 -0400 Subject: [PATCH 14/14] chore: update README --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 018f18e..bcac5bf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # pywhispercpp -Python bindings for [whisper.cpp](https://github.com/ggerganov/whisper.cpp) with a simple Pythonic API on top of it. +Python bindings for [whisper.cpp](https://github.com/ggerganov/whisper.cpp) with a simple Pythonic API on top of it. Supports both Whisper and Parakeet models. [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![Wheels](https://github.com/absadiki/pywhispercpp/actions/workflows/wheels.yml/badge.svg?branch=main&event=push)](https://github.com/absadiki/pywhispercpp/actions/workflows/wheels.yml) @@ -15,6 +15,8 @@ Python bindings for [whisper.cpp](https://github.com/ggerganov/whisper.cpp) with * [CoreML support](#coreml-support) * [Vulkan support](#vulkan-support) * [Quick start](#quick-start) + * [Whisper](#whisper) + * [Parakeet](#parakeet) * [Examples](#examples) * [CLI](#cli) * [GUI](#gui) @@ -103,28 +105,30 @@ Note that the toolkit for Ubuntu22 works on Ubuntu24 # Quick start +### Whisper + ```python -from pywhispercpp.model import Model +from pywhispercpp import WhisperModel, WhisperParams -model = Model('base.en') +model = WhisperModel(WhisperModel.AvailableModels.BASE_EN, params=WhisperParams(n_threads=2)) segments = model.transcribe('file.wav') for segment in segments: print(segment.text) ``` -You can also assign a custom `new_segment_callback` +### Parakeet ```python -from pywhispercpp.model import Model +from pywhispercpp import ParakeetModel, ParakeetParams -model = Model('base.en', print_realtime=False, print_progress=False) -segments = model.transcribe('file.mp3', new_segment_callback=print) +model = ParakeetModel(ParakeetModel.AvailableModels.TDT_0_6B_V3_Q4_0, params=ParakeetParams(n_threads=2)) +segments = model.transcribe('file.wav') +for segment in segments: + print(segment.text) ``` - * The model will be downloaded automatically, or you can use the path to a local model. -* You can pass any `whisper.cpp` [parameter](https://absadiki.github.io/pywhispercpp/#pywhispercpp.constants.PARAMS_SCHEMA) as a keyword argument to the `Model` class or to the `transcribe` function. -* Check the [Model](https://absadiki.github.io/pywhispercpp/#pywhispercpp.model.Model) class documentation for more details. +* Check the [WhisperModel](https://absadiki.github.io/pywhispercpp/#pywhispercpp.whisper_model.WhisperModel) and [ParakeetModel](https://absadiki.github.io/pywhispercpp/#pywhispercpp.parakeet_model.ParakeetModel) documentation for more details. # Examples