Skip to content

Commit

Permalink
posix shm, can not unmap shmem_ptr until encoder is done with it
Browse files Browse the repository at this point in the history
  • Loading branch information
@jsorg71
jsorg71 committed Sep 9, 2023
1 parent def567c commit bfdcdb0
Show file tree
Hide file tree
Showing 6 changed files with 47 additions and 13 deletions.
3 changes: 2 additions & 1 deletion xrdp/xrdp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,7 +528,8 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
int num_crects, short *crects,
char *data, int left, int top,
int width, int height,
int flags, int frame_id);
int flags, int frame_id,
void *shmem_ptr, int shmem_bytes);
int
server_palette(struct xrdp_mod *mod, int *palette);
int
Expand Down
2 changes: 2 additions & 0 deletions xrdp/xrdp_encoder.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ struct xrdp_enc_data
int height;
int flags;
int frame_id;
void *shmem_ptr;
int shmem_bytes;
};

typedef struct xrdp_enc_data XRDP_ENC_DATA;
Expand Down
42 changes: 34 additions & 8 deletions xrdp/xrdp_mm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2928,6 +2928,7 @@ xrdp_mm_update_module_frame_ack(struct xrdp_mm *self)
static int
xrdp_mm_process_enc_done(struct xrdp_mm *self)
{
XRDP_ENC_DATA *enc;
XRDP_ENC_DATA_DONE *enc_done;
int x;
int y;
Expand Down Expand Up @@ -2976,21 +2977,26 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
/* free enc_done */
if (enc_done->last)
{
enc = enc_done->enc;
LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: last set");
if (self->wm->client_info->use_frame_acks == 0)
{
self->mod->mod_frame_ack(self->mod,
enc_done->enc->flags,
enc_done->enc->frame_id);
enc->flags,
enc->frame_id);
}
else
{
self->encoder->frame_id_server = enc_done->enc->frame_id;
self->encoder->frame_id_server = enc->frame_id;
xrdp_mm_update_module_frame_ack(self);
}
g_free(enc_done->enc->drects);
g_free(enc_done->enc->crects);
g_free(enc_done->enc);
g_free(enc->drects);
g_free(enc->crects);
if (enc->shmem_ptr != NULL)
{
g_munmap(enc->shmem_ptr, enc->shmem_bytes);
}
g_free(enc);
}
g_free(enc_done->comp_pad_data);
g_free(enc_done);
Expand Down Expand Up @@ -3360,15 +3366,17 @@ server_paint_rects(struct xrdp_mod *mod, int num_drects, short *drects,
int height, int flags, int frame_id)
{
return server_paint_rects_ex(mod, num_drects, drects, num_crects, crects,
data, 0, 0, width, height, flags, frame_id);
data, 0, 0, width, height, flags, frame_id,
NULL, 0);
}

/*****************************************************************************/
int
server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
int num_crects, short *crects, char *data,
int left, int top, int width, int height,
int flags, int frame_id)
int flags, int frame_id,
void *shmem_ptr, int shmem_bytes)
{
struct xrdp_wm *wm;
struct xrdp_mm *mm;
Expand All @@ -3389,13 +3397,21 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
enc_data = (XRDP_ENC_DATA *) g_malloc(sizeof(XRDP_ENC_DATA), 1);
if (enc_data == 0)
{
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
return 1;
}

enc_data->drects = (short *)
g_malloc(sizeof(short) * num_drects * 4, 0);
if (enc_data->drects == 0)
{
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
g_free(enc_data);
return 1;
}
Expand All @@ -3404,6 +3420,10 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
g_malloc(sizeof(short) * num_crects * 4, 0);
if (enc_data->crects == 0)
{
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
g_free(enc_data->drects);
g_free(enc_data);
return 1;
Expand All @@ -3422,6 +3442,8 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
enc_data->height = height;
enc_data->flags = flags;
enc_data->frame_id = frame_id;
enc_data->shmem_ptr = shmem_ptr;
enc_data->shmem_bytes = shmem_bytes;
if (width == 0 || height == 0)
{
LOG_DEVEL(LOG_LEVEL_WARNING, "server_paint_rects: error");
Expand Down Expand Up @@ -3456,6 +3478,10 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
}
xrdp_bitmap_delete(b);
mm->mod->mod_frame_ack(mm->mod, flags, frame_id);
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
return 0;
}

Expand Down
3 changes: 2 additions & 1 deletion xrdp/xrdp_types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,8 @@ struct xrdp_mod
int num_crects, short *crects,
char *data, int left, int top,
int width, int height,
int flags, int frame_id);
int flags, int frame_id,
void *shmem_ptr, int shmem_bytes);
tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server
functions above */
/* common */
Expand Down
7 changes: 5 additions & 2 deletions xup/xup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1386,11 +1386,14 @@ process_server_paint_rect_shmfd(struct mod *amod, struct stream *s)
{
bmpdata = (char *)shmem_ptr;
bmpdata += shmem_offset;
/* we give up ownership of shmem_ptr
will get cleaned up in server_paint_rects_ex or
xrdp_mm_process_enc_done(rfx, gfx) */
rv = amod->server_paint_rects_ex(amod, num_drects, ldrects,
num_crects, lcrects, bmpdata,
left, top, width, height,
flags, frame_id);
g_munmap(shmem_ptr, shmem_bytes);
flags, frame_id,
shmem_ptr, shmem_bytes);
}
g_file_close(fd);
}
Expand Down
3 changes: 2 additions & 1 deletion xup/xup.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ struct mod
int num_crects, short *crects,
char *data, int left, int top,
int width, int height,
int flags, int frame_id);
int flags, int frame_id,
void *shmem_ptr, int shmem_bytes);
tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server
functions above */
/* common */
Expand Down

0 comments on commit bfdcdb0

Please sign in to comment.