Skip to content

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
singalsu wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Audio: MFCC: Add Voice Activity Detection based on Mel spectrum #10782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d282976
Audio: MFCC: Add Voice Activity Detection based on Mel spectrum
singalsu May 12, 2026
e703a93
Audio: MFCC: Add Python script for speech to text with Whisper model
singalsu May 19, 2026
d6b5dcb
Audio: MFCC: Add VAD switch control notification to user space
singalsu May 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/audio/mfcc/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@ if(CONFIG_COMP_MFCC STREQUAL "m" AND DEFINED CONFIG_LLEXT)
add_subdirectory(llext ${PROJECT_BINARY_DIR}/mfcc_llext)
add_dependencies(app mfcc)
else()
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c)
add_local_sources(sof mfcc.c mfcc_setup.c mfcc_common.c mfcc_generic.c mfcc_hifi4.c mfcc_hifi3.c mfcc_vad.c)
if(CONFIG_IPC_MAJOR_4)
add_local_sources(sof mfcc_ipc4.c)
endif()
Comment thread
singalsu marked this conversation as resolved.
endif()
46 changes: 15 additions & 31 deletions src/audio/mfcc/mfcc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,36 +97,13 @@ static int mfcc_free(struct processing_module *mod)
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");
ipc_msg_free(cd->msg);
mod_data_blob_handler_free(mod, cd->model_handler);
mfcc_free_buffers(mod);
mod_free(mod, cd);
return 0;
}

static int mfcc_get_config(struct processing_module *mod,
uint32_t config_id, uint32_t *data_offset_size,
uint8_t *fragment, size_t fragment_size)
{
struct sof_ipc_ctrl_data *cdata = (struct sof_ipc_ctrl_data *)fragment;
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");

return comp_data_blob_get_cmd(cd->model_handler, cdata, fragment_size);
}

static int mfcc_set_config(struct processing_module *mod, uint32_t config_id,
enum module_cfg_fragment_position pos, uint32_t data_offset_size,
const uint8_t *fragment, size_t fragment_size, uint8_t *response,
size_t response_size)
{
struct mfcc_comp_data *cd = module_get_private_data(mod);

comp_info(mod->dev, "entry");

return comp_data_blob_set(cd->model_handler, pos, data_offset_size,
fragment, fragment_size);
}

static int mfcc_process(struct processing_module *mod,
struct input_stream_buffer *input_buffers, int num_input_buffers,
Expand Down Expand Up @@ -187,22 +164,29 @@ static int mfcc_prepare(struct processing_module *mod,
audio_stream_get_channels(&sourceb->stream));
if (ret < 0) {
comp_err(dev, "setup failed.");
goto err;
return ret;
}
} else {
comp_err(dev, "configuration is missing.");
return -EINVAL;
}

cd->mfcc_func = mfcc_find_func(source_format, sink_format, mfcc_fm, ARRAY_SIZE(mfcc_fm));
if (!cd->mfcc_func) {
comp_err(dev, "No proc func");
ret = -EINVAL;
goto err;
return -EINVAL;
}

return 0;
/* Initialize VAD switch control notification if enabled */
if (cd->config->enable_vad && cd->config->update_controls && !cd->msg) {
ret = mfcc_ipc_notification_init(mod);
if (ret < 0)
return ret;
Comment on lines +182 to +184
Copy link
Copy Markdown
Collaborator Author

@singalsu singalsu May 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, can prepare() really be called again? But ok, adding this.


cd->vad_prev = false;
}
Comment on lines +180 to +187
Copy link
Copy Markdown
Collaborator Author

@singalsu singalsu May 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok


err:
comp_set_state(dev, COMP_TRIGGER_RESET);
return ret;
return 0;
}

static int mfcc_reset(struct processing_module *mod)
Expand Down
92 changes: 61 additions & 31 deletions src/audio/mfcc/mfcc_common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,17 @@
#include <stddef.h>
#include <stdint.h>

#include <sof/audio/mfcc/mfcc_vad.h>

LOG_MODULE_REGISTER(mfcc_common, CONFIG_SOF_LOG_LEVEL);

/*
* The main processing function for MFCC
*/

static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *cd)
static int mfcc_stft_process(struct processing_module *mod, struct mfcc_comp_data *cd)
{
const struct comp_dev *dev = mod->dev;
struct sof_mfcc_config *config = cd->config;
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &state->buf;
Expand Down Expand Up @@ -169,6 +172,32 @@ static int mfcc_stft_process(const struct comp_dev *dev, struct mfcc_comp_data *

cc_count += state->dct.num_out;
}

/* Use hop counter for frame numbering (independent of VAD enable) */
state->header.frame_number = state->hop_count;

/* Run VAD on the mel log spectrum (available in both modes) */
if (config->enable_vad) {
mfcc_vad_update(&cd->vad, state->mel_log_32);

/* Populate data header for this output frame */
state->header.energy = cd->vad.energy;
state->header.noise_energy = cd->vad.noise_energy;
state->header.vad_flag = cd->vad.is_speech ? 1 : 0;
}

/* Increment hop counter at end of hop processing */
state->hop_count++;

/* Send notification when VAD state changes */
if (config->enable_vad && config->update_controls) {
bool vad_now = cd->vad.is_speech;

if (vad_now != cd->vad_prev) {
mfcc_send_vad_notification(mod, vad_now ? 1 : 0);
cd->vad_prev = vad_now;
}
}
}

return cc_count;
Expand Down Expand Up @@ -267,9 +296,8 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int16_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 2;
const int num_header_s16 = sizeof(state->header) / sizeof(int16_t);
int num_ceps;
int sink_samples;
int to_copy;
Expand All @@ -278,27 +306,29 @@ void mfcc_s16_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s16(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT: Mel auditory filter and DCT. */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data and mark magic pending */
/* If new output produced, set up pointer into scratch data and mark header pending */
if (num_ceps > 0) {
if (state->mel_only)
if (state->mel_only) {
state->out_data_ptr = state->mel_spectra->data;
else
} else {
state->out_data_ptr = state->cepstral_coef->data;
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_magic, (int16_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s16) {
w_ptr = mfcc_sink_copy_data_s16(sink, w_ptr, num_header_s16,
(int16_t *)&state->header);
sink_samples -= num_header_s16;
state->header_pending = false;
}

/* Write cepstral/mel data from scratch buffer */
Expand Down Expand Up @@ -363,9 +393,8 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -376,7 +405,7 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s24(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -391,17 +420,18 @@ void mfcc_s24_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s32) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down Expand Up @@ -443,9 +473,8 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
struct mfcc_comp_data *cd = module_get_private_data(mod);
struct mfcc_state *state = &cd->state;
struct mfcc_buffer *buf = &cd->state.buf;
uint32_t magic = MFCC_MAGIC;
int32_t *w_ptr = audio_stream_get_wptr(sink);
const int num_magic = 1; /* one int32_t word for magic */
const int num_header_s32 = sizeof(state->header) / sizeof(int32_t);
int num_ceps;
int sink_samples;
int remain_s32;
Expand All @@ -455,7 +484,7 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
mfcc_source_copy_s32(bsource, buf, &state->emph, frames, state->source_channel);

/* Run STFT and processing after FFT */
num_ceps = mfcc_stft_process(mod->dev, cd);
num_ceps = mfcc_stft_process(mod, cd);

/* If new output produced, set up pointer into scratch data */
if (num_ceps > 0) {
Expand All @@ -466,17 +495,18 @@ void mfcc_s32_default(struct processing_module *mod, struct input_stream_buffer
}

state->out_remain = num_ceps;
state->magic_pending = true;
state->header_pending = true;
}

/* Write to sink, limited by period size */
sink_samples = frames * audio_stream_get_channels(sink);

/* Write magic word first if pending */
if (state->magic_pending && sink_samples >= num_magic) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_magic, (int32_t *)&magic);
sink_samples -= num_magic;
state->magic_pending = false;
/* Write data header first if pending */
if (state->header_pending && sink_samples >= num_header_s32) {
w_ptr = mfcc_sink_copy_data_s32(sink, w_ptr, num_header_s32,
(int32_t *)&state->header);
sink_samples -= num_header_s32;
state->header_pending = false;
}

if (state->mel_only) {
Expand Down
Loading
Loading