Skip to content

Commit 75f8c33

Browse files
awawa-devawawa-dev
committed
30% improved performance for MJPEG HDR mode
We can do JPEG YUV to RGB by ourselves since we have got proper YUV/HDR LUT table.
1 parent 36a3198 commit 75f8c33

File tree

6 files changed

+150
-111
lines changed

6 files changed

+150
-111
lines changed

‎include/base/Grabber.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232

3333
#define LUT_FILE_SIZE 50331648
3434

35+
#define UNSUPPORTED_DECODER "UNSUPPORTED YUV DECODER"
36+
3537
class Grabber : public DetectionAutomatic, public DetectionManual
3638
{
3739
Q_OBJECT

‎sources/grabber/MF/MFGrabber.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ void MFGrabber::setHdrToneMappingEnabled(int mode)
180180
{
181181
Debug(_log, "setHdrToneMappingMode replacing LUT and restarting");
182182
_MFWorkerManager.Stop();
183-
if ((_actualVideoFormat == PixelFormat::YUYV) || (_actualVideoFormat == PixelFormat::I420) || (_actualVideoFormat == PixelFormat::NV12))
183+
if ((_actualVideoFormat == PixelFormat::YUYV) || (_actualVideoFormat == PixelFormat::I420) || (_actualVideoFormat == PixelFormat::NV12) || (_actualVideoFormat == PixelFormat::MJPEG))
184184
loadLutFile(PixelFormat::YUYV);
185185
else
186186
loadLutFile(PixelFormat::RGB24);
@@ -874,9 +874,8 @@ bool MFGrabber::init_device(QString selectedDeviceName, DevicePropertiesItem pro
874874

875875
case PixelFormat::MJPEG:
876876
{
877-
loadLutFile(PixelFormat::RGB24);
877+
loadLutFile(PixelFormat::YUYV);
878878
_lineLength = props.x * 3;
879-
880879
}
881880
break;
882881
}
@@ -1021,6 +1020,10 @@ void MFGrabber::newWorkerFrameError(unsigned int workerIndex, QString error, qui
10211020
{
10221021

10231022
frameStat.badFrame++;
1023+
if (error.indexOf(QString(UNSUPPORTED_DECODER)) == 0)
1024+
{
1025+
Error(_log, "Unsupported MJPEG/YUV format. Please contact HyperHDR developers! (info: %s)", QSTRING_CSTR(error));
1026+
}
10241027
//Debug(_log, "Error occured while decoding mjpeg frame %d = %s", sourceCount, QSTRING_CSTR(error));
10251028

10261029
// get next frame

‎sources/grabber/MF/MFWorker.cpp‎

Lines changed: 56 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -211,11 +211,7 @@ void MFWorker::runMe()
211211
if (_isActive && _width > 0 && _height > 0)
212212
{
213213
if (_pixelFormat == PixelFormat::MJPEG)
214-
{
215-
if (_qframe)
216-
{
217-
_cropLeft = _cropRight = _cropTop = _cropBottom = 0;
218-
}
214+
{
219215
process_image_jpg_mt();
220216
}
221217
else
@@ -265,75 +261,78 @@ void MFWorker::noBusy()
265261
_isBusy = false;
266262
}
267263

268-
269264
void MFWorker::process_image_jpg_mt()
270265
{
271266
if (_decompress == nullptr)
272267
_decompress = tjInitDecompress();
273268

274-
if (tjDecompressHeader2(_decompress, const_cast<uint8_t*>(_localData), _size, &_width, &_height, &_subsamp) != 0)
269+
if (tjDecompressHeader2(_decompress, const_cast<uint8_t*>(_localData), _size, &_width, &_height, &_subsamp) != 0 &&
270+
tjGetErrorCode(_decompress) == TJERR_FATAL)
275271
{
276-
if (tjGetErrorCode(_decompress) == TJERR_FATAL)
277-
{
278-
QString info = QString(tjGetErrorStr());
279-
emit newFrameError(_workerIndex, info, _currentFrame);
280-
return;
281-
}
272+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
273+
return;
274+
}
275+
276+
if (_subsamp != TJSAMP_422 && _hdrToneMappingEnabled > 0)
277+
{
278+
emit newFrameError(_workerIndex, QString("%1: %2").arg(UNSUPPORTED_DECODER).arg(_subsamp), _currentFrame);
279+
return;
282280
}
283281

284282
tjscalingfactor sca{ 1, 2 };
285-
Image<ColorRgb> srcImage((_qframe) ? TJSCALED(_width, sca) : _width,
286-
(_qframe) ? TJSCALED(_height, sca) : _height);
287-
_width = srcImage.width();
288-
_height = srcImage.height();
289283

290-
if (tjDecompress2(_decompress, const_cast<uint8_t*>(_localData), _size, (uint8_t*)srcImage.memptr(), _width, 0, _height, TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0)
291-
{
292-
if (tjGetErrorCode(_decompress) == TJERR_FATAL)
293-
{
294-
QString info = QString(tjGetErrorStr());
295-
emit newFrameError(_workerIndex, info, _currentFrame);
296-
return;
297-
}
298-
}
284+
_width = (_qframe) ? TJSCALED(_width, sca) : _width;
285+
_height = (_qframe) ? TJSCALED(_height, sca) : _height;
299286

300-
// got image, process it
301-
if (!(_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0))
302-
{
303-
// apply LUT
304-
FrameDecoder::applyLUT((unsigned char*)srcImage.memptr(), srcImage.width(), srcImage.height(), _lutBuffer, _hdrToneMappingEnabled);;
287+
Image<ColorRgb> image(_width - _cropLeft - _cropRight, _height - _cropTop - _cropBottom);
305288

306-
// exit
307-
emit newFrame(_workerIndex, srcImage, _currentFrame, _frameBegin);
308-
}
309-
else
289+
if (_hdrToneMappingEnabled > 0)
310290
{
311-
// calculate the output size
312-
int outputWidth = (_width - _cropLeft - _cropRight);
313-
int outputHeight = (_height - _cropTop - _cropBottom);
291+
size_t yuvSize = tjBufSizeYUV2(_width, 2, _height, _subsamp);
292+
uint8_t* jpegBuffer = (uint8_t*)malloc(yuvSize);
314293

315-
if (outputWidth <= 0 || outputHeight <= 0)
316-
{
317-
QString info = QString("Invalid cropping");
318-
emit newFrameError(_workerIndex, info, _currentFrame);
319-
return;
320-
}
321-
322-
Image<ColorRgb> destImage(outputWidth, outputHeight);
294+
if (tjDecompressToYUV2(_decompress, const_cast<uint8_t*>(_localData), _size, jpegBuffer, _width, 2, _height, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0 &&
295+
tjGetErrorCode(_decompress) == TJERR_FATAL)
296+
{
297+
free(jpegBuffer);
323298

324-
for (unsigned int y = 0; y < destImage.height(); y++)
325-
{
326-
unsigned char* source = (unsigned char*)srcImage.memptr() + (static_cast<size_t>(y) + _cropTop) * srcImage.width() * 3 + static_cast<size_t>(_cropLeft) * 3;
327-
unsigned char* dest = (unsigned char*)destImage.memptr() + static_cast<size_t>(y) * destImage.width() * 3;
328-
memcpy(dest, source, static_cast<size_t>(destImage.width()) * 3);
329-
}
299+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
300+
return;
301+
}
330302

331-
// apply LUT
332-
FrameDecoder::applyLUT((unsigned char*)destImage.memptr(), destImage.width(), destImage.height(), _lutBuffer, _hdrToneMappingEnabled);
303+
FrameDecoder::processImage(_cropLeft, _cropRight, _cropTop, _cropBottom,
304+
jpegBuffer, _width, _height, _width, PixelFormat::MJPEG, _lutBuffer, image);
333305

334-
// exit
335-
emit newFrame(_workerIndex, destImage, _currentFrame, _frameBegin);
306+
free(jpegBuffer);
336307
}
337-
}
308+
else if (image.width() != (uint)_width || image.height() != (uint)_height)
309+
{
310+
uint8_t* jpegBuffer = (uint8_t*)malloc(_width * _height * 3);
311+
312+
if (tjDecompress2(_decompress, const_cast<uint8_t*>(_localData), _size, (uint8_t*)jpegBuffer, _width, 0, _height, TJPF_BGR, TJFLAG_BOTTOMUP | TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0 &&
313+
tjGetErrorCode(_decompress) == TJERR_FATAL)
314+
{
315+
free(jpegBuffer);
316+
317+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
318+
return;
319+
}
320+
321+
FrameDecoder::processImage(_cropLeft, _cropRight, _cropTop, _cropBottom,
322+
jpegBuffer, _width, _height, _width * 3, PixelFormat::RGB24, nullptr, image);
338323

324+
free(jpegBuffer);
325+
}
326+
else
327+
{
328+
if (tjDecompress2(_decompress, const_cast<uint8_t*>(_localData), _size, (uint8_t*)image.memptr(), _width, 0, _height, TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0 &&
329+
tjGetErrorCode(_decompress) == TJERR_FATAL)
330+
{
331+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
332+
return;
333+
}
334+
}
335+
339336

337+
emit newFrame(_workerIndex, image, _currentFrame, _frameBegin);
338+
}

‎sources/grabber/v4l2/V4L2Grabber.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ void V4L2Grabber::setHdrToneMappingEnabled(int mode)
130130
{
131131
Debug(_log, "setHdrToneMappingMode replacing LUT and restarting");
132132
_V4L2WorkerManager.Stop();
133-
if ((_actualVideoFormat == PixelFormat::YUYV) || (_actualVideoFormat == PixelFormat::I420) || (_actualVideoFormat == PixelFormat::NV12))
133+
if ((_actualVideoFormat == PixelFormat::YUYV) || (_actualVideoFormat == PixelFormat::I420) || (_actualVideoFormat == PixelFormat::NV12) || (_actualVideoFormat == PixelFormat::MJPEG))
134134
loadLutFile(PixelFormat::YUYV);
135135
else
136136
loadLutFile(PixelFormat::RGB24);
@@ -975,7 +975,7 @@ bool V4L2Grabber::init_device(QString selectedDeviceName, DevicePropertiesItem p
975975

976976
case V4L2_PIX_FMT_MJPEG:
977977
{
978-
loadLutFile(PixelFormat::RGB24);
978+
loadLutFile(PixelFormat::YUYV);
979979
_actualVideoFormat = PixelFormat::MJPEG;
980980
Info(_log, "Video pixel format is set to: MJPEG");
981981
}
@@ -1196,6 +1196,10 @@ bool V4L2Grabber::process_image(v4l2_buffer* buf, const void* frameImageBuffer,
11961196
void V4L2Grabber::newWorkerFrameError(unsigned int workerIndex, QString error, quint64 sourceCount)
11971197
{
11981198
frameStat.badFrame++;
1199+
if (error.indexOf(QString(UNSUPPORTED_DECODER)) == 0)
1200+
{
1201+
Error(_log, "Unsupported MJPEG/YUV format. Please contact HyperHDR developers! (info: %s)", QSTRING_CSTR(error));
1202+
}
11991203
//Debug(_log, "Error occured while decoding mjpeg frame %d = %s", sourceCount, QSTRING_CSTR(error));
12001204

12011205
// get next frame

‎sources/grabber/v4l2/V4L2Worker.cpp‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -191,12 +191,7 @@ void V4L2Worker::runMe()
191191
if (_isActive)
192192
{
193193
if (_pixelFormat == PixelFormat::MJPEG)
194-
{
195-
if (_qframe)
196-
{
197-
_cropLeft = _cropRight = _cropTop = _cropBottom = 0;
198-
}
199-
194+
{
200195
process_image_jpg_mt();
201196
}
202197
else
@@ -253,68 +248,73 @@ void V4L2Worker::process_image_jpg_mt()
253248
if (_decompress == nullptr)
254249
_decompress = tjInitDecompress();
255250

256-
if (tjDecompressHeader2(_decompress, const_cast<uint8_t*>(_sharedData), _size, &_width, &_height, &_subsamp) != 0)
251+
if (tjDecompressHeader2(_decompress, const_cast<uint8_t*>(_sharedData), _size, &_width, &_height, &_subsamp) != 0 &&
252+
tjGetErrorCode(_decompress) == TJERR_FATAL)
257253
{
258-
if (tjGetErrorCode(_decompress) == TJERR_FATAL)
259-
{
260-
QString info = QString(tjGetErrorStr());
261-
emit newFrameError(_workerIndex, info, _currentFrame);
262-
return;
263-
}
254+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
255+
return;
256+
}
257+
258+
if (_subsamp != TJSAMP_422 && _hdrToneMappingEnabled > 0)
259+
{
260+
emit newFrameError(_workerIndex, QString("%1: %2").arg(UNSUPPORTED_DECODER).arg(_subsamp), _currentFrame);
261+
return;
264262
}
265263

266264
tjscalingfactor sca{ 1, 2 };
267-
Image<ColorRgb> srcImage((_qframe) ? TJSCALED(_width, sca) : _width,
268-
(_qframe) ? TJSCALED(_height, sca) : _height);
269-
_width = srcImage.width();
270-
_height = srcImage.height();
271265

272-
if (tjDecompress2(_decompress, const_cast<uint8_t*>(_sharedData), _size, (uint8_t*)srcImage.memptr(), _width, 0, _height, TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0)
266+
_width = (_qframe) ? TJSCALED(_width, sca) : _width;
267+
_height = (_qframe) ? TJSCALED(_height, sca) : _height;
268+
269+
Image<ColorRgb> image(_width - _cropLeft - _cropRight, _height - _cropTop - _cropBottom);
270+
271+
if (_hdrToneMappingEnabled > 0)
273272
{
274-
if (tjGetErrorCode(_decompress) == TJERR_FATAL)
273+
size_t yuvSize = tjBufSizeYUV2(_width, 2, _height, _subsamp);
274+
uint8_t* jpegBuffer = (uint8_t*)malloc(yuvSize);
275+
276+
if (tjDecompressToYUV2(_decompress, const_cast<uint8_t*>(_sharedData), _size, jpegBuffer, _width, 2, _height, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0 &&
277+
tjGetErrorCode(_decompress) == TJERR_FATAL)
275278
{
276-
QString info = QString(tjGetErrorStr());
277-
emit newFrameError(_workerIndex, info, _currentFrame);
279+
free(jpegBuffer);
280+
281+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
278282
return;
279283
}
280-
}
281284

285+
FrameDecoder::processImage(_cropLeft, _cropRight, _cropTop, _cropBottom,
286+
jpegBuffer, _width, _height, _width, PixelFormat::MJPEG, _lutBuffer, image);
282287

283-
// got image, process it
284-
if (!(_cropLeft > 0 || _cropTop > 0 || _cropBottom > 0 || _cropRight > 0))
285-
{
286-
// apply LUT
287-
FrameDecoder::applyLUT((uint8_t*)srcImage.memptr(), srcImage.width(), srcImage.height(), _lutBuffer, _hdrToneMappingEnabled);
288-
289-
// exit
290-
emit newFrame(_workerIndex, srcImage, _currentFrame, _frameBegin);
288+
free(jpegBuffer);
291289
}
292-
else
290+
else if (image.width() != (uint)_width || image.height() != (uint)_height)
293291
{
294-
// calculate the output size
295-
int outputWidth = (_width - _cropLeft - _cropRight);
296-
int outputHeight = (_height - _cropTop - _cropBottom);
292+
uint8_t* jpegBuffer = (uint8_t*)malloc(_width * _height * 3);
297293

298-
if (outputWidth <= 0 || outputHeight <= 0)
294+
if (tjDecompress2(_decompress, const_cast<uint8_t*>(_sharedData), _size, (uint8_t*)jpegBuffer, _width, 0, _height, TJPF_BGR, TJFLAG_BOTTOMUP | TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0 &&
295+
tjGetErrorCode(_decompress) == TJERR_FATAL)
299296
{
300-
QString info = QString("Invalid cropping");
301-
emit newFrameError(_workerIndex, info, _currentFrame);
297+
free(jpegBuffer);
298+
299+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
302300
return;
303301
}
304302

305-
Image<ColorRgb> destImage(outputWidth, outputHeight);
303+
FrameDecoder::processImage(_cropLeft, _cropRight, _cropTop, _cropBottom,
304+
jpegBuffer, _width, _height, _width * 3, PixelFormat::RGB24, nullptr, image);
306305

307-
for (uint32_t y = 0; y < destImage.height(); y++)
306+
free(jpegBuffer);
307+
}
308+
else
309+
{
310+
if (tjDecompress2(_decompress, const_cast<uint8_t*>(_sharedData), _size, (uint8_t*)image.memptr(), _width, 0, _height, TJPF_RGB, TJFLAG_FASTDCT | TJFLAG_FASTUPSAMPLE) != 0 &&
311+
tjGetErrorCode(_decompress) == TJERR_FATAL)
308312
{
309-
uint8_t* source = (uint8_t*)srcImage.memptr() + (static_cast<size_t>(y) + _cropTop) * srcImage.width() * 3 + static_cast<size_t>(_cropLeft) * 3;
310-
uint8_t* dest = (uint8_t*)destImage.memptr() + static_cast<size_t>(y) * destImage.width() * 3;
311-
memcpy(dest, source, static_cast<size_t>(destImage.width()) * 3);
313+
emit newFrameError(_workerIndex, QString(tjGetErrorStr()), _currentFrame);
314+
return;
312315
}
316+
}
313317

314-
// apply LUT
315-
FrameDecoder::applyLUT((uint8_t*)destImage.memptr(), destImage.width(), destImage.height(), _lutBuffer, _hdrToneMappingEnabled);
316318

317-
// exit
318-
emit newFrame(_workerIndex, destImage, _currentFrame, _frameBegin);
319-
}
319+
emit newFrame(_workerIndex, image, _currentFrame, _frameBegin);
320320
}

‎sources/utils/FrameDecoder.cpp‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,14 @@ void FrameDecoder::processImage(
4747
// validate format
4848
if (pixelFormat != PixelFormat::YUYV &&
4949
pixelFormat != PixelFormat::XRGB && pixelFormat != PixelFormat::RGB24 &&
50-
pixelFormat != PixelFormat::I420 && pixelFormat != PixelFormat::NV12)
50+
pixelFormat != PixelFormat::I420 && pixelFormat != PixelFormat::NV12 && pixelFormat != PixelFormat::MJPEG)
5151
{
5252
Error(Logger::getInstance("FrameDecoder"), "Invalid pixel format given");
5353
return;
5454
}
5555

5656
// validate format LUT
57-
if ((pixelFormat == PixelFormat::YUYV || pixelFormat == PixelFormat::I420 ||
57+
if ((pixelFormat == PixelFormat::YUYV || pixelFormat == PixelFormat::I420 || pixelFormat == PixelFormat::MJPEG ||
5858
pixelFormat == PixelFormat::NV12) && lutBuffer == NULL)
5959
{
6060
Error(Logger::getInstance("FrameDecoder"), "Missing LUT table for YUV colorspace");
@@ -207,6 +207,37 @@ void FrameDecoder::processImage(
207207
return;
208208
}
209209

210+
if (pixelFormat == PixelFormat::MJPEG)
211+
{
212+
int deltaU = lineLength * height;
213+
int deltaV = lineLength * height * 6 / 4;
214+
for (int yDest = 0, ySource = _cropTop; yDest < outputHeight; ++ySource, ++yDest)
215+
{
216+
uint8_t* currentDest = destMemory + ((uint64_t)destLineSize) * yDest;
217+
uint8_t* endDest = currentDest + destLineSize;
218+
uint8_t* currentSource = (uint8_t*)data + (((uint64_t)lineLength * ySource) + ((uint64_t)_cropLeft));
219+
uint8_t* currentSourceU = (uint8_t*)data + deltaU + ((((uint64_t)ySource) * lineLength) + ((uint64_t)_cropLeft)) / 2;
220+
uint8_t* currentSourceV = (uint8_t*)data + deltaV + ((((uint64_t)ySource) * lineLength) + ((uint64_t)_cropLeft)) / 2;
221+
222+
while (currentDest < endDest)
223+
{
224+
*((uint16_t*)&buffer) = *((uint16_t*)currentSource);
225+
currentSource += 2;
226+
buffer[2] = *(currentSourceU++);
227+
buffer[3] = *(currentSourceV++);
228+
229+
ind_lutd = LUT_INDEX(buffer[0], buffer[2], buffer[3]);
230+
ind_lutd2 = LUT_INDEX(buffer[1], buffer[2], buffer[3]);
231+
232+
*((uint32_t*)currentDest) = *((uint32_t*)(&lutBuffer[ind_lutd]));
233+
currentDest += 3;
234+
*((uint32_t*)currentDest) = *((uint32_t*)(&lutBuffer[ind_lutd2]));
235+
currentDest += 3;
236+
}
237+
}
238+
return;
239+
}
240+
210241
if (pixelFormat == PixelFormat::NV12)
211242
{
212243
int deltaU = lineLength * height;

0 commit comments

Comments
 (0)