From 958c2ca7cbe8d4f6c23ee3eb51eec8d672f9c140 Mon Sep 17 00:00:00 2001 From: Aaron Kling Date: Sat, 20 Apr 2024 02:00:11 -0500 Subject: [PATCH] Stream mjpeg using avcodec and yuv420p --- src/zm_eventstream.cpp | 9 +++-- src/zm_image.cpp | 53 ++++++++++++++++++++++++++++++ src/zm_image.h | 1 + src/zm_monitorstream.cpp | 20 +++++++---- src/zm_stream.cpp | 71 +++++++++++++++++++++++++++++++++------- src/zm_stream.h | 4 +++ 6 files changed, 137 insertions(+), 21 deletions(-) diff --git a/src/zm_eventstream.cpp b/src/zm_eventstream.cpp index 2d413d0c87..519297598c 100644 --- a/src/zm_eventstream.cpp +++ b/src/zm_eventstream.cpp @@ -870,14 +870,19 @@ bool EventStream::sendFrame(Microseconds delta_us) { } Image *send_image = prepareImage(image); - reserveTempImgBuffer(send_image->Size()); + int l_width = floor(send_image->Width() * scale / ZM_SCALE_BASE); + int l_height = floor(send_image->Height() * scale / ZM_SCALE_BASE); + reserveTempImgBuffer(av_image_get_buffer_size(AV_PIX_FMT_YUVJ420P, l_width, l_height, 32)); int img_buffer_size = 0; uint8_t *img_buffer = temp_img_buffer; fprintf(stdout, "--" BOUNDARY "\r\n"); switch ( type ) { case STREAM_JPEG : - send_image->EncodeJpeg(img_buffer, &img_buffer_size); + if (mJpegCodecContext->width != l_width || mJpegCodecContext->height != l_height) { + initContexts(l_width, l_height); + } + send_image->EncodeJpeg(img_buffer, &img_buffer_size, mJpegCodecContext, mJpegSwsContext); fputs("Content-Type: image/jpeg\r\n", stdout); break; case STREAM_ZIP : diff --git a/src/zm_image.cpp b/src/zm_image.cpp index 0023d55d7e..bb23bbee20 100644 --- a/src/zm_image.cpp +++ b/src/zm_image.cpp @@ -1600,6 +1600,59 @@ bool Image::EncodeJpeg(JOCTET *outbuffer, int *outbuffer_size, int quality_overr return true; } +bool Image::EncodeJpeg(JOCTET *outbuffer, int *outbuffer_size, AVCodecContext *p_jpegcodeccontext, SwsContext *p_jpegswscontext) const { + if ( config.colour_jpeg_files && (colours == ZM_COLOUR_GRAY8) ) { + Image temp_image(*this); + temp_image.Colourise(ZM_COLOUR_RGB24, ZM_SUBPIX_ORDER_RGB); + return temp_image.EncodeJpeg(outbuffer, outbuffer_size, p_jpegcodeccontext, p_jpegswscontext); + } + + if (p_jpegcodeccontext == NULL) { + Error("Jpeg codec context is not initialized"); + return false; + } + + std::unique_lock lck(jpeg_mutex); + + av_frame_ptr frame = av_frame_ptr{zm_av_frame_alloc()}; + AVPacket *pkt; + + if (av_image_get_buffer_size(AV_PIX_FMT_YUVJ420P, width, height, 32) > static_cast(Size())) { + Error("Output buffer not large enough"); + return false; + } + + if ( p_jpegswscontext ) { + av_frame_ptr temp_frame = av_frame_ptr{zm_av_frame_alloc()}; + PopulateFrame(temp_frame.get()); + + frame.get()->width = width; + frame.get()->height = height; + frame.get()->format = AV_PIX_FMT_YUV420P; + av_image_fill_linesizes(frame.get()->linesize, AV_PIX_FMT_YUV420P, width); + av_frame_get_buffer(frame.get(), 32); + + sws_scale(p_jpegswscontext, temp_frame.get()->data, temp_frame.get()->linesize, 0, height, frame.get()->data, frame.get()->linesize); + + av_frame_unref(temp_frame.get()); + } else { + PopulateFrame(frame.get()); + } + + pkt = av_packet_alloc(); + + avcodec_send_frame(p_jpegcodeccontext, frame.get()); + if (avcodec_receive_packet(p_jpegcodeccontext, pkt) == 0) { + memcpy(outbuffer, pkt->data, pkt->size); + *outbuffer_size = pkt->size; + } + + av_packet_free(&pkt); + av_frame_unref(frame.get()); + + return true; +} + #if HAVE_ZLIB_H bool Image::Unzip( const Bytef *inbuffer, unsigned long inbuffer_size ) { unsigned long zip_size = size; diff --git a/src/zm_image.h b/src/zm_image.h index dd3681a449..3f391cf1fe 100644 --- a/src/zm_image.h +++ b/src/zm_image.h @@ -241,6 +241,7 @@ class Image { bool DecodeJpeg(const JOCTET *inbuffer, int inbuffer_size, unsigned int p_colours, unsigned int p_subpixelorder); bool EncodeJpeg(JOCTET *outbuffer, int *outbuffer_size, int quality_override=0) const; + bool EncodeJpeg(JOCTET *outbuffer, int *outbuffer_size, AVCodecContext *p_jpegcodeccontext, SwsContext *p_jpegswscontext) const; #if HAVE_ZLIB_H bool Unzip(const Bytef *inbuffer, unsigned long inbuffer_size); diff --git a/src/zm_monitorstream.cpp b/src/zm_monitorstream.cpp index c114ca72e4..450e472636 100644 --- a/src/zm_monitorstream.cpp +++ b/src/zm_monitorstream.cpp @@ -393,14 +393,20 @@ bool MonitorStream::sendFrame(Image *image, SystemTimePoint timestamp) { /* double pts = */ vid_stream->EncodeFrame(send_image->Buffer(), send_image->Size(), config.mpeg_timed_frames, delta_time.count()); } else { - reserveTempImgBuffer(send_image->Size()); + int l_width = floor(send_image->Width() * scale / ZM_SCALE_BASE); + int l_height = floor(send_image->Height() * scale / ZM_SCALE_BASE); + + reserveTempImgBuffer(av_image_get_buffer_size(AV_PIX_FMT_YUVJ420P, l_width, l_height, 32)); int img_buffer_size = 0; unsigned char *img_buffer = temp_img_buffer; switch (type) { case STREAM_JPEG : - send_image->EncodeJpeg(img_buffer, &img_buffer_size); + if (mJpegCodecContext->width != l_width || mJpegCodecContext->height != l_height) { + initContexts(l_width, l_height); + } + send_image->EncodeJpeg(img_buffer, &img_buffer_size, mJpegCodecContext, mJpegSwsContext); fputs("Content-Type: image/jpeg\r\n", stdout); break; case STREAM_RAW : @@ -937,12 +943,12 @@ void MonitorStream::SingleImage(int scale) { SystemTimePoint(zm::chrono::duration_cast(monitor->shared_timestamps[index]))); } - if ( scale != ZM_SCALE_BASE ) { - scaled_image.Assign(*snap_image); - scaled_image.Scale(scale); - snap_image = &scaled_image; + int l_width = floor(snap_image->Width() * scale / ZM_SCALE_BASE); + int l_height = floor(snap_image->Height() * scale / ZM_SCALE_BASE); + if (mJpegCodecContext->width != l_width || mJpegCodecContext->height != l_height) { + initContexts(l_width, l_height); } - snap_image->EncodeJpeg(img_buffer, &img_buffer_size); + snap_image->EncodeJpeg(img_buffer, &img_buffer_size, mJpegCodecContext, mJpegSwsContext); fprintf(stdout, "Content-Length: %d\r\n" diff --git a/src/zm_stream.cpp b/src/zm_stream.cpp index 1f55ad3503..4cebec894c 100644 --- a/src/zm_stream.cpp +++ b/src/zm_stream.cpp @@ -36,6 +36,61 @@ StreamBase::~StreamBase() { delete vid_stream; delete[] temp_img_buffer; closeComms(); + + if (mJpegCodecContext) { + avcodec_free_context(&mJpegCodecContext); + } + + if (mJpegSwsContext) { + sws_freeContext(mJpegSwsContext); + } +} + +bool StreamBase::initContexts(int p_width, int p_height) { + if (mJpegCodecContext) avcodec_free_context(&mJpegCodecContext); + if (mJpegSwsContext) sws_freeContext(mJpegSwsContext); + + const AVCodec* mJpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG); + if (!mJpegCodec) { + Error("MJPEG codec not found"); + return false; + } + + mJpegCodecContext = avcodec_alloc_context3(mJpegCodec); + if (!mJpegCodecContext) { + Error("Could not allocate jpeg codec context"); + return false; + } + + mJpegCodecContext->bit_rate = 400000; + mJpegCodecContext->width = p_width; + mJpegCodecContext->height = p_height; + mJpegCodecContext->time_base= (AVRational) {1,25}; + mJpegCodecContext->pix_fmt = AV_PIX_FMT_YUVJ420P; + + if (avcodec_open2(mJpegCodecContext, mJpegCodec, NULL) < 0) { + Error("Could not open mjpeg codec"); + return false; + } + + AVPixelFormat format; + switch (monitor->Colours()) { + case ZM_COLOUR_RGB24: + format = (monitor->SubpixelOrder() == ZM_SUBPIX_ORDER_BGR ? AV_PIX_FMT_BGR24 : AV_PIX_FMT_RGB24); + break; + case ZM_COLOUR_GRAY8: + format = AV_PIX_FMT_GRAY8; + break; + default: + format = AV_PIX_FMT_RGBA; + break; + }; + mJpegSwsContext = sws_getContext( + monitor->Width(), monitor->Height(), format, + p_width, p_height, AV_PIX_FMT_YUV420P, + SWS_BICUBIC, nullptr, nullptr, nullptr); + + return true; } bool StreamBase::loadMonitor(int p_monitor_id) { @@ -64,7 +119,9 @@ bool StreamBase::loadMonitor(int p_monitor_id) { return false; } - return true; + mJpegCodecContext = nullptr; + mJpegSwsContext = nullptr; + return initContexts(monitor->Width(), monitor->Height()); } bool StreamBase::checkInitialised() { @@ -150,9 +207,7 @@ Image *StreamBase::prepareImage(Image *image) { if (zoom != 100) { int base_image_width = image->Width(), - base_image_height = image->Height(), - disp_image_width = image->Width() * scale/ZM_SCALE_BASE, - disp_image_height = image->Height() * scale / ZM_SCALE_BASE; + base_image_height = image->Height(); /* x and y are scaled by web UI to base dimensions units. * When zooming, we blow up the image by the amount 150 for first zoom, right? 150%, then cut out a base sized chunk * However if we have zoomed before, then we are zooming into the previous cutout @@ -229,14 +284,6 @@ Image *StreamBase::prepareImage(Image *image) { image_copied = true; } image->Crop(last_crop); - image->Scale(disp_image_width, disp_image_height); - } else if (scale != ZM_SCALE_BASE) { - Debug(3, "scaling by %d from %dx%d", scale, image->Width(), image->Height()); - static Image copy_image; - copy_image.Assign(*image); - image = ©_image; - image_copied = true; - image->Scale(scale); } Debug(3, "Sending %dx%d", image->Width(), image->Height()); diff --git a/src/zm_stream.h b/src/zm_stream.h index c668c0bcbb..59393b0048 100644 --- a/src/zm_stream.h +++ b/src/zm_stream.h @@ -153,6 +153,9 @@ class StreamBase { uint8_t *temp_img_buffer; // Used when encoding or sending file data size_t temp_img_buffer_size; + AVCodecContext *mJpegCodecContext; + SwsContext *mJpegSwsContext; + protected: bool loadMonitor(int monitor_id); bool checkInitialised(); @@ -161,6 +164,7 @@ class StreamBase { void checkCommandQueue(); virtual void processCommand(const CmdMsg *msg)=0; void reserveTempImgBuffer(size_t size); + bool initContexts(int p_width, int p_height); public: StreamBase():