From 458cc7e314e1a00017dd090db388db6659d1bb5f Mon Sep 17 00:00:00 2001 From: Leo Izen Date: Sun, 17 Apr 2022 09:22:34 -0400 Subject: avcodec/libjxl: add Jpeg XL decoding via libjxl This commit adds decoding support to libavcodec for Jpeg XL images via the external library libjxl. --- MAINTAINERS | 1 + configure | 5 + doc/general_contents.texi | 7 ++ libavcodec/Makefile | 1 + libavcodec/allcodecs.c | 1 + libavcodec/libjxl.c | 70 ++++++++++++ libavcodec/libjxl.h | 48 ++++++++ libavcodec/libjxldec.c | 280 ++++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 413 insertions(+) create mode 100644 libavcodec/libjxl.c create mode 100644 libavcodec/libjxl.h create mode 100644 libavcodec/libjxldec.c diff --git a/MAINTAINERS b/MAINTAINERS index 859a5005d4..faea84ebf1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -194,6 +194,7 @@ Codecs: libcodec2.c Tomas Härdin libdirac* David Conrad libdavs2.c Huiwen Ren + libjxl*.c, libjxl.h Leo Izen libgsm.c Michel Bardiaux libkvazaar.c Arttu Ylä-Outinen libopenh264enc.c Martin Storsjo, Linjie Fu diff --git a/configure b/configure index 358a614854..df4f6d242d 100755 --- a/configure +++ b/configure @@ -240,6 +240,7 @@ External library support: --enable-libiec61883 enable iec61883 via libiec61883 [no] --enable-libilbc enable iLBC de/encoding via libilbc [no] --enable-libjack enable JACK audio sound server [no] + --enable-libjxl enable JPEG XL decoding via libjxl [no] --enable-libklvanc enable Kernel Labs VANC processing [no] --enable-libkvazaar enable HEVC encoding via libkvazaar [no] --enable-liblensfun enable lensfun lens correction [no] @@ -1833,6 +1834,7 @@ EXTERNAL_LIBRARY_LIST=" libiec61883 libilbc libjack + libjxl libklvanc libkvazaar libmodplug @@ -3331,6 +3333,7 @@ libgsm_ms_decoder_deps="libgsm" libgsm_ms_encoder_deps="libgsm" libilbc_decoder_deps="libilbc" libilbc_encoder_deps="libilbc" +libjxl_decoder_deps="libjxl libjxl_threads" libkvazaar_encoder_deps="libkvazaar" libmodplug_demuxer_deps="libmodplug" libmp3lame_encoder_deps="libmp3lame" @@ -6545,6 +6548,8 @@ enabled libgsm && { for gsm_hdr in "gsm.h" "gsm/gsm.h"; do check_lib libgsm "${gsm_hdr}" gsm_create -lgsm && break; done || die "ERROR: libgsm not found"; } enabled libilbc && require libilbc ilbc.h WebRtcIlbcfix_InitDecode -lilbc $pthreads_extralibs +enabled libjxl && require_pkg_config libjxl "libjxl >= 0.7.0" jxl/decode.h JxlDecoderVersion && + require_pkg_config libjxl_threads "libjxl_threads >= 0.7.0" jxl/thread_parallel_runner.h JxlThreadParallelRunner enabled libklvanc && require libklvanc libklvanc/vanc.h klvanc_context_create -lklvanc enabled libkvazaar && require_pkg_config libkvazaar "kvazaar >= 0.8.1" kvazaar.h kvz_api_get enabled liblensfun && require_pkg_config liblensfun lensfun lensfun.h lf_db_new diff --git a/doc/general_contents.texi b/doc/general_contents.texi index 238568f2bb..93a90a5e52 100644 --- a/doc/general_contents.texi +++ b/doc/general_contents.texi @@ -171,6 +171,13 @@ Go to @url{https://github.com/TimothyGu/libilbc} and follow the instructions for installing the library. Then pass @code{--enable-libilbc} to configure to enable it. +@section libjxl + +JPEG XL is an image format intended to fully replace legacy JPEG for an extended +period of life. See @url{https://jpegxl.info/} for more information, and see +@url{https://github.com/libjxl/libjxl} for the library source. You can pass +@code{--enable-libjxl} to configure in order enable the libjxl wrapper. + @section libvpx FFmpeg can make use of the libvpx library for VP8/VP9 decoding and encoding. diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 4b12228070..eba9835c52 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1062,6 +1062,7 @@ OBJS-$(CONFIG_LIBGSM_MS_DECODER) += libgsmdec.o OBJS-$(CONFIG_LIBGSM_MS_ENCODER) += libgsmenc.o OBJS-$(CONFIG_LIBILBC_DECODER) += libilbc.o OBJS-$(CONFIG_LIBILBC_ENCODER) += libilbc.o +OBJS-$(CONFIG_LIBJXL_DECODER) += libjxldec.o libjxl.o OBJS-$(CONFIG_LIBKVAZAAR_ENCODER) += libkvazaar.o OBJS-$(CONFIG_LIBMP3LAME_ENCODER) += libmp3lame.o OBJS-$(CONFIG_LIBOPENCORE_AMRNB_DECODER) += libopencore-amr.o diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c index 585918da93..07f5bafd27 100644 --- a/libavcodec/allcodecs.c +++ b/libavcodec/allcodecs.c @@ -751,6 +751,7 @@ extern const FFCodec ff_libgsm_ms_encoder; extern const FFCodec ff_libgsm_ms_decoder; extern const FFCodec ff_libilbc_encoder; extern const FFCodec ff_libilbc_decoder; +extern const FFCodec ff_libjxl_decoder; extern const FFCodec ff_libmp3lame_encoder; extern const FFCodec ff_libopencore_amrnb_encoder; extern const FFCodec ff_libopencore_amrnb_decoder; diff --git a/libavcodec/libjxl.c b/libavcodec/libjxl.c new file mode 100644 index 0000000000..204d91d8a8 --- /dev/null +++ b/libavcodec/libjxl.c @@ -0,0 +1,70 @@ +/* + * JPEG XL de/encoding via libjxl, common support implementation + * Copyright (c) 2021 Leo Izen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * JPEG XL via libjxl common support implementation + */ + +#include "libavutil/cpu.h" +#include "libavutil/mem.h" + +#include +#include "libjxl.h" + +size_t ff_libjxl_get_threadcount(int threads) +{ + if (threads <= 0) + return av_cpu_count(); + if (threads == 1) + return 0; + return threads; +} + +/** + * Wrapper around av_malloc used as a jpegxl_alloc_func. + * + * @param opaque opaque pointer for jpegxl_alloc_func, always ignored + * @param size Size in bytes for the memory block to be allocated + * @return Pointer to the allocated block, or `NULL` if it cannot be allocated + */ +static void *libjxl_av_malloc(void *opaque, size_t size) +{ + return av_malloc(size); +} + +/** + * Wrapper around av_free used as a jpegxl_free_func. + * + * @param opaque opaque pointer for jpegxl_free_func, always ignored + * @param address Pointer to the allocated block, to free. `NULL` permitted as a no-op. + */ +static void libjxl_av_free(void *opaque, void *address) +{ + av_free(address); +} + +void ff_libjxl_init_memory_manager(JxlMemoryManager *manager) +{ + manager->opaque = NULL; + manager->alloc = &libjxl_av_malloc; + manager->free = &libjxl_av_free; +} diff --git a/libavcodec/libjxl.h b/libavcodec/libjxl.h new file mode 100644 index 0000000000..5387c438fd --- /dev/null +++ b/libavcodec/libjxl.h @@ -0,0 +1,48 @@ +/* + * JPEG XL de/encoding via libjxl, common support header + * Copyright (c) 2021 Leo Izen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * JPEG XL via libjxl common support header + */ + +#ifndef AVCODEC_LIBJXL_H +#define AVCODEC_LIBJXL_H + +#include + +/** + * Transform threadcount in ffmpeg to one used by libjxl. + * + * @param threads ffmpeg's threads AVOption + * @return thread count for libjxl's parallel runner + */ +size_t ff_libjxl_get_threadcount(int threads); + +/** + * Initialize and populate a JxlMemoryManager + * with av_malloc() and av_free() so libjxl will use these + * functions. + * @param manager a pointer to a JxlMemoryManager struct + */ +void ff_libjxl_init_memory_manager(JxlMemoryManager *manager); + +#endif /* AVCODEC_LIBJXL_H */ diff --git a/libavcodec/libjxldec.c b/libavcodec/libjxldec.c new file mode 100644 index 0000000000..cd4bca3343 --- /dev/null +++ b/libavcodec/libjxldec.c @@ -0,0 +1,280 @@ +/* + * JPEG XL decoding support via libjxl + * Copyright (c) 2021 Leo Izen + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * JPEG XL decoder using libjxl + */ + +#include "libavutil/avassert.h" +#include "libavutil/buffer.h" +#include "libavutil/common.h" +#include "libavutil/error.h" +#include "libavutil/mem.h" +#include "libavutil/pixdesc.h" +#include "libavutil/pixfmt.h" +#include "libavutil/frame.h" + +#include "avcodec.h" +#include "codec_internal.h" +#include "internal.h" + +#include +#include +#include "libjxl.h" + +typedef struct LibJxlDecodeContext { + void *runner; + JxlDecoder *decoder; + JxlBasicInfo basic_info; + JxlPixelFormat jxl_pixfmt; + JxlDecoderStatus events; + AVBufferRef *iccp; +} LibJxlDecodeContext; + +static int libjxl_init_jxl_decoder(AVCodecContext *avctx) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + + ctx->events = JXL_DEC_BASIC_INFO | JXL_DEC_FULL_IMAGE | JXL_DEC_COLOR_ENCODING; + if (JxlDecoderSubscribeEvents(ctx->decoder, ctx->events) != JXL_DEC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Error subscribing to JXL events\n"); + return AVERROR_EXTERNAL; + } + + if (JxlDecoderSetParallelRunner(ctx->decoder, JxlThreadParallelRunner, ctx->runner) != JXL_DEC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to set JxlThreadParallelRunner\n"); + return AVERROR_EXTERNAL; + } + + memset(&ctx->basic_info, 0, sizeof(JxlBasicInfo)); + memset(&ctx->jxl_pixfmt, 0, sizeof(JxlPixelFormat)); + + return 0; +} + +static av_cold int libjxl_decode_init(AVCodecContext *avctx) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + JxlMemoryManager manager; + + ff_libjxl_init_memory_manager(&manager); + ctx->decoder = JxlDecoderCreate(&manager); + if (!ctx->decoder) { + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlDecoder\n"); + return AVERROR_EXTERNAL; + } + + ctx->runner = JxlThreadParallelRunnerCreate(&manager, ff_libjxl_get_threadcount(avctx->thread_count)); + if (!ctx->runner) { + av_log(avctx, AV_LOG_ERROR, "Failed to create JxlThreadParallelRunner\n"); + return AVERROR_EXTERNAL; + } + + return libjxl_init_jxl_decoder(avctx); +} + +static enum AVPixelFormat libjxl_get_pix_fmt(AVCodecContext *avctx, JxlBasicInfo *basic_info, JxlPixelFormat *format) +{ + format->endianness = JXL_NATIVE_ENDIAN; + format->num_channels = basic_info->num_color_channels + (basic_info->alpha_bits > 0); + /* Gray */ + if (basic_info->num_color_channels == 1) { + if (basic_info->bits_per_sample <= 8) { + format->data_type = JXL_TYPE_UINT8; + return basic_info->alpha_bits ? AV_PIX_FMT_YA8 : AV_PIX_FMT_GRAY8; + } + if (basic_info->exponent_bits_per_sample || basic_info->bits_per_sample > 16) { + if (basic_info->alpha_bits) + return AV_PIX_FMT_NONE; + format->data_type = JXL_TYPE_FLOAT; + return AV_PIX_FMT_GRAYF32; + } + format->data_type = JXL_TYPE_UINT16; + return basic_info->alpha_bits ? AV_PIX_FMT_YA16 : AV_PIX_FMT_GRAY16; + } + /* rgb only */ + /* libjxl only supports packed RGB and gray output at the moment */ + if (basic_info->num_color_channels == 3) { + if (basic_info->bits_per_sample <= 8) { + format->data_type = JXL_TYPE_UINT8; + return basic_info->alpha_bits ? AV_PIX_FMT_RGBA : AV_PIX_FMT_RGB24; + } + if (basic_info->bits_per_sample > 16) + av_log(avctx, AV_LOG_WARNING, "Downsampling larger integer to 16-bit via libjxl\n"); + if (basic_info->exponent_bits_per_sample) + av_log(avctx, AV_LOG_WARNING, "Downsampling float to 16-bit integer via libjxl\n"); + format->data_type = JXL_TYPE_UINT16; + return basic_info->alpha_bits ? AV_PIX_FMT_RGBA64 : AV_PIX_FMT_RGB48; + } + + return AV_PIX_FMT_NONE; +} + +static int libjxl_decode_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame, AVPacket *avpkt) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + uint8_t *buf = avpkt->data; + size_t remaining = avpkt->size, iccp_len; + JxlDecoderStatus jret; + int ret; + *got_frame = 0; + + while (1) { + + jret = JxlDecoderSetInput(ctx->decoder, buf, remaining); + + if (jret == JXL_DEC_ERROR) { + /* this should never happen here unless there's a bug in libjxl */ + av_log(avctx, AV_LOG_ERROR, "Unknown libjxl decode error\n"); + return AVERROR_EXTERNAL; + } + + jret = JxlDecoderProcessInput(ctx->decoder); + /* + * JxlDecoderReleaseInput returns the number + * of bytes remaining to be read, rather than + * the number of bytes that it did read + */ + remaining = JxlDecoderReleaseInput(ctx->decoder); + buf = avpkt->data + avpkt->size - remaining; + + switch(jret) { + case JXL_DEC_ERROR: + av_log(avctx, AV_LOG_ERROR, "Unknown libjxl decode error\n"); + return AVERROR_INVALIDDATA; + case JXL_DEC_NEED_MORE_INPUT: + if (remaining == 0) { + av_log(avctx, AV_LOG_ERROR, "Unexpected end of JXL codestream\n"); + return AVERROR_INVALIDDATA; + } + av_log(avctx, AV_LOG_DEBUG, "NEED_MORE_INPUT event emitted\n"); + continue; + case JXL_DEC_BASIC_INFO: + av_log(avctx, AV_LOG_DEBUG, "BASIC_INFO event emitted\n"); + if (JxlDecoderGetBasicInfo(ctx->decoder, &ctx->basic_info) != JXL_DEC_SUCCESS) { + /* + * this should never happen + * if it does it is likely a libjxl decoder bug + */ + av_log(avctx, AV_LOG_ERROR, "Bad libjxl basic info event\n"); + return AVERROR_EXTERNAL; + } + avctx->pix_fmt = libjxl_get_pix_fmt(avctx, &ctx->basic_info, &ctx->jxl_pixfmt); + if (avctx->pix_fmt == AV_PIX_FMT_NONE) { + av_log(avctx, AV_LOG_ERROR, "Bad libjxl pixel format\n"); + return AVERROR_EXTERNAL; + } + ret = ff_set_dimensions(avctx, ctx->basic_info.xsize, ctx->basic_info.ysize); + if (ret < 0) + return ret; + continue; + case JXL_DEC_COLOR_ENCODING: + av_log(avctx, AV_LOG_DEBUG, "COLOR_ENCODING event emitted\n"); + jret = JxlDecoderGetICCProfileSize(ctx->decoder, &ctx->jxl_pixfmt, JXL_COLOR_PROFILE_TARGET_ORIGINAL, &iccp_len); + if (jret == JXL_DEC_SUCCESS && iccp_len > 0) { + av_buffer_unref(&ctx->iccp); + ctx->iccp = av_buffer_alloc(iccp_len); + if (!ctx->iccp) + return AVERROR(ENOMEM); + jret = JxlDecoderGetColorAsICCProfile(ctx->decoder, &ctx->jxl_pixfmt, JXL_COLOR_PROFILE_TARGET_ORIGINAL, ctx->iccp->data, iccp_len); + if (jret != JXL_DEC_SUCCESS) + av_buffer_unref(&ctx->iccp); + } + continue; + case JXL_DEC_NEED_IMAGE_OUT_BUFFER: + av_log(avctx, AV_LOG_DEBUG, "NEED_IMAGE_OUT_BUFFER event emitted\n"); + ret = ff_get_buffer(avctx, frame, 0); + if (ret < 0) + return ret; + ctx->jxl_pixfmt.align = frame->linesize[0]; + if (JxlDecoderSetImageOutBuffer(ctx->decoder, &ctx->jxl_pixfmt, frame->data[0], frame->buf[0]->size) != JXL_DEC_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Bad libjxl dec need image out buffer event\n"); + return AVERROR_EXTERNAL; + } + continue; + case JXL_DEC_FULL_IMAGE: + /* full image is one frame, even if animated */ + av_log(avctx, AV_LOG_DEBUG, "FULL_IMAGE event emitted\n"); + frame->pict_type = AV_PICTURE_TYPE_I; + frame->key_frame = 1; + if (ctx->iccp) { + AVFrameSideData *sd = av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_ICC_PROFILE, ctx->iccp); + if (!sd) + return AVERROR(ENOMEM); + /* ownership is transfered, and it is not ref-ed */ + ctx->iccp = NULL; + } + *got_frame = 1; + return avpkt->size - remaining; + case JXL_DEC_SUCCESS: + av_log(avctx, AV_LOG_DEBUG, "SUCCESS event emitted\n"); + /* + * The SUCCESS event isn't fired until after JXL_DEC_FULL_IMAGE. If this + * stream only contains one JXL image then JXL_DEC_SUCCESS will never fire. + * If the image2 sequence being decoded contains several JXL files, then + * libjxl will fire this event after the next AVPacket has been passed, + * which means the current packet is actually the next image in the sequence. + * This is why we reset the decoder and populate the packet data now, since + * this is the next packet and it has not been decoded yet. The decoder does + * have to be reset to allow us to use it for the next image, or libjxl + * will become very confused if the header information is not identical. + */ + JxlDecoderReset(ctx->decoder); + libjxl_init_jxl_decoder(avctx); + buf = avpkt->data; + remaining = avpkt->size; + continue; + default: + av_log(avctx, AV_LOG_ERROR, "Bad libjxl event: %d\n", jret); + return AVERROR_EXTERNAL; + } + } +} + +static av_cold int libjxl_decode_close(AVCodecContext *avctx) +{ + LibJxlDecodeContext *ctx = avctx->priv_data; + + if (ctx->runner) + JxlThreadParallelRunnerDestroy(ctx->runner); + ctx->runner = NULL; + if (ctx->decoder) + JxlDecoderDestroy(ctx->decoder); + ctx->decoder = NULL; + av_buffer_unref(&ctx->iccp); + + return 0; +} + +const FFCodec ff_libjxl_decoder = { + .p.name = "libjxl", + .p.long_name = NULL_IF_CONFIG_SMALL("libjxl JPEG XL"), + .p.type = AVMEDIA_TYPE_VIDEO, + .p.id = AV_CODEC_ID_JPEGXL, + .priv_data_size = sizeof(LibJxlDecodeContext), + .init = libjxl_decode_init, + FF_CODEC_DECODE_CB(libjxl_decode_frame), + .close = libjxl_decode_close, + .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_OTHER_THREADS, + .caps_internal = FF_CODEC_CAP_AUTO_THREADS | FF_CODEC_CAP_INIT_CLEANUP, + .p.wrapper_name = "libjxl", +}; -- cgit v1.2.1