From 07bb3f01b4d93bebe02b05f618036d8a8dc0272a Mon Sep 17 00:00:00 2001 From: Jean-Marc Valin Date: Wed, 21 Jun 2023 17:48:21 -0400 Subject: [PATCH] cleanup: get rid of non-causal PLC and DC handling --- dnn/include/lpcnet.h | 2 - dnn/lpcnet_demo.c | 9 +- dnn/lpcnet_plc.c | 202 +------------------------------------------ dnn/lpcnet_private.h | 8 -- 4 files changed, 4 insertions(+), 217 deletions(-) diff --git a/dnn/include/lpcnet.h b/dnn/include/lpcnet.h index fd12044c7..fc9978f5f 100644 --- a/dnn/include/lpcnet.h +++ b/dnn/include/lpcnet.h @@ -189,10 +189,8 @@ LPCNET_EXPORT void lpcnet_synthesize(LPCNetState *st, const float *features, sho #define LPCNET_PLC_CAUSAL 0 -#define LPCNET_PLC_NONCAUSAL 1 #define LPCNET_PLC_CODEC 2 -#define LPCNET_PLC_DC_FILTER 4 LPCNET_EXPORT int lpcnet_plc_get_size(void); diff --git a/dnn/lpcnet_demo.c b/dnn/lpcnet_demo.c index 1c2cd95f2..a0ee3c9cf 100644 --- a/dnn/lpcnet_demo.c +++ b/dnn/lpcnet_demo.c @@ -90,9 +90,7 @@ void usage(void) { fprintf(stderr, " lpcnet_demo -addlpc \n\n"); fprintf(stderr, " plc_options:\n"); fprintf(stderr, " causal: normal (causal) PLC\n"); - fprintf(stderr, " causal_dc: normal (causal) PLC with DC offset compensation\n"); - fprintf(stderr, " noncausal: non-causal PLC\n"); - fprintf(stderr, " noncausal_dc: non-causal PLC with DC offset compensation\n"); + fprintf(stderr, " codec: normal (causal) PLC without cross-fade (will glitch)\n"); exit(1); } @@ -134,9 +132,7 @@ int main(int argc, char **argv) { } if (mode == MODE_PLC) { if (strcmp(plc_options, "causal")==0) plc_flags = LPCNET_PLC_CAUSAL; - else if (strcmp(plc_options, "causal_dc")==0) plc_flags = LPCNET_PLC_CAUSAL | LPCNET_PLC_DC_FILTER; - else if (strcmp(plc_options, "noncausal")==0) plc_flags = LPCNET_PLC_NONCAUSAL; - else if (strcmp(plc_options, "noncausal_dc")==0) plc_flags = LPCNET_PLC_NONCAUSAL | LPCNET_PLC_DC_FILTER; + else if (strcmp(plc_options, "codec")==0) plc_flags = LPCNET_PLC_CODEC; else usage(); } if (argc != 4) usage(); @@ -191,7 +187,6 @@ int main(int argc, char **argv) { int loss=0; int skip=0, extra=0; LPCNetPLCState *net; - if ((plc_flags&0x3) == LPCNET_PLC_NONCAUSAL) skip=extra=80; net = lpcnet_plc_create(plc_flags); #ifdef USE_WEIGHTS_FILE lpcnet_plc_load_model(net, data, len); diff --git a/dnn/lpcnet_plc.c b/dnn/lpcnet_plc.c index a104c1d9d..99605987d 100644 --- a/dnn/lpcnet_plc.c +++ b/dnn/lpcnet_plc.c @@ -54,8 +54,6 @@ LPCNET_EXPORT void lpcnet_plc_reset(LPCNetPLCState *st) { st->skip_analysis = 0; st->blend = 0; st->loss_count = 0; - st->dc_mem = 0; - st->queued_update = 0; } LPCNET_EXPORT int lpcnet_plc_init(LPCNetPLCState *st, int options) { @@ -64,17 +62,11 @@ LPCNET_EXPORT int lpcnet_plc_init(LPCNetPLCState *st, int options) { lpcnet_encoder_init(&st->enc); if ((options&0x3) == LPCNET_PLC_CAUSAL) { st->enable_blending = 1; - st->non_causal = 0; - } else if ((options&0x3) == LPCNET_PLC_NONCAUSAL) { - st->enable_blending = 1; - st->non_causal = 1; } else if ((options&0x3) == LPCNET_PLC_CODEC) { st->enable_blending = 0; - st->non_causal = 0; } else { return -1; } - st->remove_dc = !!(options&LPCNET_PLC_DC_FILTER); #ifndef USE_WEIGHTS_FILE ret = init_plc_model(&st->model, lpcnet_plc_arrays); #else @@ -180,28 +172,15 @@ void clear_state(LPCNetPLCState *st) { RNN_CLEAR(st->lpcnet.nnet.gru_b_state, GRU_B_STATE_SIZE); } -#define DC_CONST 0.003 - /* In this causal version of the code, the DNN model implemented by compute_plc_pred() needs to generate two feature vectors to conceal the first lost packet.*/ -static int lpcnet_plc_update_causal(LPCNetPLCState *st, short *pcm) { +int lpcnet_plc_update(LPCNetPLCState *st, short *pcm) { int i; float x[FRAME_SIZE]; short output[FRAME_SIZE]; float plc_features[2*NB_BANDS+NB_FEATURES+1]; - short lp[FRAME_SIZE]={0}; int delta = 0; - if (st->remove_dc) { - st->dc_mem += st->syn_dc; - delta = st->syn_dc; - st->syn_dc = 0; - for (i=0;idc_mem); - st->dc_mem += DC_CONST*(pcm[i] - st->dc_mem); - pcm[i] -= lp[i]; - } - } for (i=0;ienc.pcount = 0; @@ -280,17 +259,12 @@ static int lpcnet_plc_update_causal(LPCNetPLCState *st, short *pcm) { RNN_MOVE(st->pcm, &st->pcm[FRAME_SIZE], PLC_BUF_SIZE); } st->loss_count = 0; - if (st->remove_dc) { - for (i=0;iblend = 0; return 0; } static const float att_table[10] = {0, 0, -.2, -.2, -.4, -.4, -.8, -.8, -1.6, -1.6}; -static int lpcnet_plc_conceal_causal(LPCNetPLCState *st, short *pcm) { +int lpcnet_plc_conceal(LPCNetPLCState *st, short *pcm) { int i; short output[FRAME_SIZE]; run_frame_network_flush(&st->lpcnet); @@ -327,177 +301,5 @@ static int lpcnet_plc_conceal_causal(LPCNetPLCState *st, short *pcm) { process_single_frame(&st->enc, NULL); } st->blend = 1; - if (st->remove_dc) { - for (i=0;isyn_dc += DC_CONST*(pcm[i] - st->syn_dc); - pcm[i] += (int)floor(.5 + st->dc_mem); - } - } return 0; } - -/* In this non-causal version of the code, the DNN model implemented by compute_plc_pred() - is always called once per frame. We process audio up to the current position minus TRAINING_OFFSET. */ - -void process_queued_update(LPCNetPLCState *st) { - if (st->queued_update) { - lpcnet_synthesize_impl(&st->lpcnet, st->features, st->queued_samples, FRAME_SIZE, FRAME_SIZE); - st->queued_update=0; - } -} - -static int lpcnet_plc_update_non_causal(LPCNetPLCState *st, short *pcm) { - int i; - float x[FRAME_SIZE]; - short pcm_save[FRAME_SIZE]; - float plc_features[2*NB_BANDS+NB_FEATURES+1]; - short lp[FRAME_SIZE]={0}; - double mem_bak=0; - int delta = st->syn_dc; - if (FEATURES_DELAY != 0) { - fprintf(stderr, "Non-causal PLC cannot work with non-zero FEATURES_DELAY\n"); - fprintf(stderr, "Recompile with a no-lookahead model (see README.md)\n"); - exit(1); - } - process_queued_update(st); - if (st->remove_dc) { - st->dc_mem += st->syn_dc; - st->syn_dc = 0; - mem_bak = st->dc_mem; - for (i=0;idc_mem); - st->dc_mem += DC_CONST*(pcm[i] - st->dc_mem); - pcm[i] -= lp[i]; - } - } - RNN_COPY(pcm_save, pcm, FRAME_SIZE); - for (i=0;ienc.pcount = 0; - if (st->loss_count > 0) { - LPCNetState copy; - /* Handle blending. */ - float zeros[2*NB_BANDS+NB_FEATURES+1] = {0}; - RNN_COPY(zeros, plc_features, 2*NB_BANDS); - zeros[2*NB_BANDS+NB_FEATURES] = 1; - compute_plc_pred(st, st->features, zeros); - copy = st->lpcnet; - lpcnet_synthesize_impl(&st->lpcnet, st->features, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, 0); - /* Undo initial DC offset removal so that we can take into account the last 5ms of synthesis. */ - if (st->remove_dc) { - for (i=0;idc_mem = mem_bak; - for (i=0;isyn_dc += DC_CONST*(st->pcm[FRAME_SIZE-TRAINING_OFFSET+i] - st->syn_dc); - st->dc_mem += st->syn_dc; - delta += st->syn_dc; - st->syn_dc = 0; - for (i=0;idc_mem); - st->dc_mem += DC_CONST*(pcm[i] - st->dc_mem); - pcm[i] -= lp[i]; - } - RNN_COPY(pcm_save, pcm, FRAME_SIZE); - } - { - short rev[FRAME_SIZE]; - for (i=0;ilpcnet, st->features, rev, FRAME_SIZE, FRAME_SIZE); - lpcnet_synthesize_tail_impl(&st->lpcnet, rev, TRAINING_OFFSET, 0); - for (i=0;ipcm[FRAME_SIZE-1-i] = (int)floor(.5 + w*st->pcm[FRAME_SIZE-1-i] + (1-w)*(rev[i]+delta)); - } - - } - st->lpcnet = copy; -#if 1 - st->queued_update = 1; - RNN_COPY(&st->queued_samples[0], &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET); - RNN_COPY(&st->queued_samples[TRAINING_OFFSET], pcm, FRAME_SIZE-TRAINING_OFFSET); -#else - lpcnet_synthesize_impl(&st->lpcnet, st->features, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, TRAINING_OFFSET); - lpcnet_synthesize_tail_impl(&st->lpcnet, pcm, FRAME_SIZE-TRAINING_OFFSET, FRAME_SIZE-TRAINING_OFFSET); -#endif - for (i=0;ipcm[i]; - preemphasis(x, &st->enc.mem_preemph, x, PREEMPHASIS, FRAME_SIZE); - compute_frame_features(&st->enc, x); - process_single_frame(&st->enc, NULL); - - } - for (i=0;ienc.mem_preemph, x, PREEMPHASIS, FRAME_SIZE); - compute_frame_features(&st->enc, x); - process_single_frame(&st->enc, NULL); - if (st->loss_count == 0) { - RNN_COPY(&plc_features[2*NB_BANDS], st->enc.features[0], NB_FEATURES); - plc_features[2*NB_BANDS+NB_FEATURES] = 1; - compute_plc_pred(st, st->features, plc_features); - lpcnet_synthesize_impl(&st->lpcnet, st->enc.features[0], &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, TRAINING_OFFSET); - lpcnet_synthesize_tail_impl(&st->lpcnet, pcm, FRAME_SIZE-TRAINING_OFFSET, FRAME_SIZE-TRAINING_OFFSET); - } - RNN_COPY(&pcm[FRAME_SIZE-TRAINING_OFFSET], pcm, TRAINING_OFFSET); - RNN_COPY(pcm, &st->pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET); - RNN_COPY(st->pcm, pcm_save, FRAME_SIZE); - st->loss_count = 0; - if (st->remove_dc) { - for (i=0;idc_buf[i]; - for (;idc_buf[i] = lp[FRAME_SIZE-TRAINING_OFFSET+i]; - } - return 0; -} - -static int lpcnet_plc_conceal_non_causal(LPCNetPLCState *st, short *pcm) { - int i; - float x[FRAME_SIZE]; - float zeros[2*NB_BANDS+NB_FEATURES+1] = {0}; - process_queued_update(st); - st->enc.pcount = 0; - - compute_plc_pred(st, st->features, zeros); - if (st->loss_count >= 10) st->features[0] = MAX16(-10, st->features[0]+att_table[9] - 2*(st->loss_count-9)); - else st->features[0] = MAX16(-10, st->features[0]+att_table[st->loss_count]); - - if (st->loss_count == 0) { - RNN_COPY(pcm, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET); - lpcnet_synthesize_impl(&st->lpcnet, st->features, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, TRAINING_OFFSET); - lpcnet_synthesize_tail_impl(&st->lpcnet, &pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET, 0); - } else { - lpcnet_synthesize_impl(&st->lpcnet, st->features, pcm, TRAINING_OFFSET, 0); - lpcnet_synthesize_tail_impl(&st->lpcnet, &pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET, 0); - - RNN_COPY(&st->pcm[FRAME_SIZE-TRAINING_OFFSET], pcm, TRAINING_OFFSET); - for (i=0;ipcm[i]; - preemphasis(x, &st->enc.mem_preemph, x, PREEMPHASIS, FRAME_SIZE); - compute_frame_features(&st->enc, x); - process_single_frame(&st->enc, NULL); - } - RNN_COPY(st->pcm, &pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET); - - if (st->remove_dc) { - int dc = (int)floor(.5 + st->dc_mem); - if (st->loss_count == 0) { - for (i=TRAINING_OFFSET;isyn_dc += DC_CONST*(pcm[i] - st->syn_dc); - } else { - for (i=0;isyn_dc += DC_CONST*(pcm[i] - st->syn_dc); - } - for (i=0;idc_buf[i]; - for (;idc_buf[i] = dc; - } - st->loss_count++; - return 0; -} - - -LPCNET_EXPORT int lpcnet_plc_update(LPCNetPLCState *st, short *pcm) { - if (st->non_causal) return lpcnet_plc_update_non_causal(st, pcm); - else return lpcnet_plc_update_causal(st, pcm); -} - -LPCNET_EXPORT int lpcnet_plc_conceal(LPCNetPLCState *st, short *pcm) { - if (st->non_causal) return lpcnet_plc_conceal_non_causal(st, pcm); - else return lpcnet_plc_conceal_causal(st, pcm); -} diff --git a/dnn/lpcnet_private.h b/dnn/lpcnet_private.h index e6fb57ed4..a3eea0a6d 100644 --- a/dnn/lpcnet_private.h +++ b/dnn/lpcnet_private.h @@ -80,8 +80,6 @@ struct LPCNetPLCState { LPCNetState lpcnet; LPCNetEncState enc; int enable_blending; - int non_causal; - int remove_dc; #define LPCNET_PLC_RESET_START fec float fec[PLC_MAX_FEC][NB_FEATURES]; @@ -97,12 +95,6 @@ struct LPCNetPLCState { int loss_count; PLCNetState plc_net; PLCNetState plc_copy[FEATURES_DELAY+1]; - double dc_mem; - double syn_dc; - - short dc_buf[TRAINING_OFFSET]; - int queued_update; - short queued_samples[FRAME_SIZE]; }; void preemphasis(float *y, float *mem, const float *x, float coef, int N);