Rewrite mythtranscode to use ffmpeg instead of libmpeg2

[marillat]

• Platform: Linux

• MythTV version: Up to date fixes/31

• Package version: From DMO

libmpeg2-4-dev 0.5.1-9 is installed

If the include stdint.h is put before the mpeg2dec/mpeg2.h include the build doesn"t fail.

original test.c generated by configure

#include <mpeg2dec/mpeg2.h>
#include <stdint.h>
long check_mpeg2_init(void) { return (long) mpeg2_init; }
int main(void) { int ret = 0;
ret |= ((intptr_t)check_mpeg2_init) & 0xFFFF;
return ret; }

Working test.c

#include <stdint.h>
#include <mpeg2dec/mpeg2.h>
long check_mpeg2_init(void) { return (long) mpeg2_init; }
int main(void) { int ret = 0;
ret |= ((intptr_t)check_mpeg2_init) & 0xFFFF;
return ret; }

Error log from config.ep

/usr/lib/ccache/cc -Wdate-time -D_FORTIFY_SOURCE=2 -D_ISOC99_SOURCE -D_POSIX_C_SOURCE=200112 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_POSIX_C_SOURCE=200112 -D_XOPEN_SOURCE=600 -DPIC -g -O2 -ffile-prefix-map=/home/marillat/src/mythtv-31.0+fixes20210801.git5824c588db=. -fstack-protector-strong -Wformat -Werror=format-security -std=c11 -fomit-frame-pointer -fPIC -pthread -I/usr/include/libxml2 -I/usr/include/uuid -I/usr/include/freetype2 -I/usr/include/libpng16 -DX264_API_IMPORTS -I/usr/include/libxml2 -I/usr/include/libdrm -c -o /tmp/ffconf.VerPqlGs/test.o /tmp/ffconf.VerPqlGs/test.c
In file included from /tmp/ffconf.VerPqlGs/test.c:1:
/usr/include/mpeg2dec/mpeg2.h:49:5: error: unknown type name 'uint32_t'
   49 |     uint32_t flags;
      |     ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:56:5: error: unknown type name 'uint8_t'
   56 |     uint8_t profile_level_id;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:57:5: error: unknown type name 'uint8_t'
   57 |     uint8_t colour_primaries;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:58:5: error: unknown type name 'uint8_t'
   58 |     uint8_t transfer_characteristics;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:59:5: error: unknown type name 'uint8_t'
   59 |     uint8_t matrix_coefficients;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:67:5: error: unknown type name 'uint8_t'
   67 |     uint8_t hours;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:68:5: error: unknown type name 'uint8_t'
   68 |     uint8_t minutes;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:69:5: error: unknown type name 'uint8_t'
   69 |     uint8_t seconds;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:70:5: error: unknown type name 'uint8_t'
   70 |     uint8_t pictures;
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:71:5: error: unknown type name 'uint32_t'
   71 |     uint32_t flags;
      |     ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:91:5: error: unknown type name 'uint32_t'
   91 |     uint32_t tag, tag2;
      |     ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:92:5: error: unknown type name 'uint32_t'
   92 |     uint32_t flags;
      |     ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:99:5: error: unknown type name 'uint8_t'
   99 |     uint8_t * buf[3];
      |     ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:113:11: error: unknown type name 'uint8_t'
  113 |     const uint8_t * user_data;
      |           ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:140:31: error: unknown type name 'uint8_t'
  140 |     void (* copy) (void * id, uint8_t * const * src, unsigned int v_offset);
      |                               ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:149:9: error: unknown type name 'uint32_t'
  149 |         uint32_t accel, void * arg,
      |         ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:151:43: error: unknown type name 'mpeg2_convert_t'; did you mean 'mpeg2_convert_init_t'?
  151 | int mpeg2_convert (mpeg2dec_t * mpeg2dec, mpeg2_convert_t convert, void * arg);
      |                                           ^~~~~~~~~~~~~~~
      |                                           mpeg2_convert_init_t
/usr/include/mpeg2dec/mpeg2.h:153:44: error: unknown type name 'uint8_t'
  153 | void mpeg2_set_buf (mpeg2dec_t * mpeg2dec, uint8_t * buf[3], void * id);
      |                                            ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:169:1: error: unknown type name 'uint32_t'
  169 | uint32_t mpeg2_accel (uint32_t accel);
      | ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:169:23: error: unknown type name 'uint32_t'
  169 | uint32_t mpeg2_accel (uint32_t accel);
      |                       ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:174:43: error: unknown type name 'uint8_t'
  174 | void mpeg2_buffer (mpeg2dec_t * mpeg2dec, uint8_t * start, uint8_t * end);
      |                                           ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:174:60: error: unknown type name 'uint8_t'
  174 | void mpeg2_buffer (mpeg2dec_t * mpeg2dec, uint8_t * start, uint8_t * end);
      |                                                            ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:182:48: error: unknown type name 'uint32_t'
  182 | void mpeg2_tag_picture (mpeg2dec_t * mpeg2dec, uint32_t tag, uint32_t tag2);
      |                                                ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:182:62: error: unknown type name 'uint32_t'
  182 | void mpeg2_tag_picture (mpeg2dec_t * mpeg2dec, uint32_t tag, uint32_t tag2);
      |                                                              ^~~~~~~~
/usr/include/mpeg2dec/mpeg2.h:184:50: error: unknown type name 'uint8_t'
  184 | void mpeg2_init_fbuf (mpeg2_decoder_t * decoder, uint8_t * current_fbuf[3],
      |                                                  ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:185:9: error: unknown type name 'uint8_t'
  185 |         uint8_t * forward_fbuf[3], uint8_t * backward_fbuf[3]);
      |         ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:185:36: error: unknown type name 'uint8_t'
  185 |         uint8_t * forward_fbuf[3], uint8_t * backward_fbuf[3]);
      |                                    ^~~~~~~
/usr/include/mpeg2dec/mpeg2.h:186:62: error: unknown type name 'uint8_t'
  186 | void mpeg2_slice (mpeg2_decoder_t * decoder, int code, const uint8_t * buffer);
      |                                                              ^~~~~~~
Warning! No suitable external libmpeg2 found, use internal copy.

[ulmus-scott]

The libmpeg2 headers are not self-sufficient, i.e. they will not compile on their own. To work around this, replace /mythtv/configure line 6066 with: check_lib libmpeg2external "stdint.h mpeg2dec/mpeg2.h" mpeg2_init -lmpeg2 || disable libmpeg2external

This should make the --enable-libmpeg2external flag work. However, this also disables mythtranscode, which is the only user of libmpeg2, specifically the files mpeg2fix.(h|cpp).

Notably, mpeg2fix.cpp uses information only exposed in mpeg2_internal.h, specifically in the MPEG2fixup::BuildFrame() function https://github.com/MythTV/mythtv/blob/94f61679d476bd0b415c393a704d855def1b8a07/mythtv/programs/mythtranscode/mpeg2fix.cpp#L1202.

Because of this, it would be better to rewrite mpeg2fix to use only FFmpeg.

(As an aside, mpeg2fix is a terrible name; lossless transcode is probably better.)

[stuarta]

@ulmus-scott mythtranscode needs rewriting anyway to be able to losslessly transcode all the newer formats (eg. H264, H265 etc) and not just mpeg2!!!!

[garybuhrmaster]

@ulmus-scott mythtranscode needs rewriting anyway to be able to losslessly transcode all the newer formats (eg. H264, H265 etc) and not just mpeg2!!!!

Or just remove mythtranscode entirely, and recommend using other tools (ffmpeg, handbrake, other) for transcode, and tools such as the Lossless cut scripts on the wiki for commercial removal which support other codecs. Perhaps keeping around script named mythtranscode to invoke those other programs using the existing command line flags for compatibility?

[Steve-Goodey]

Or just remove mythtranscode entirely, ...

Excuse me for butting in, but would this break/cause problems with MythArchive?

[ulmus-scott]

@stuarta mythtranscode needs rewriting anyway to be able to losslessly transcode all the newer formats (eg. H264, H265 etc) and not just mpeg2!!!!

BuildFrame() seems to decode a video packet with libmpeg2 and then add it to the FFmpeg output, encoding the frames as intra-frames only using the highest quality setting. I see no inherent reason that would prevent this from being done in a codec-agnostic manner with FFmpeg.

@garybuhrmaster Or just remove mythtranscode entirely, ...

@Steve-Goodey Excuse me for butting in, but would this break/cause problems with MythArchive?

Mythtranscode also appears to support HTTP Live Streaming.

Since MythTV already requires FFmpeg for decoding, I see no reason to not continue offering an interface in MythTV to transcode with FFmpeg.

[ulmus-scott]

codec-agnostic manner

Re-reading this, I think a truly agnostic approach would probably be difficult to impossible, due to the different settings for each codec. However, what I really meant was: the general idea is codec agnostic (although using i-frames only seems rather wasteful and inefficient) but would probably need an explicit specific implementation for each codec. At this time, the supported codecs for virtually lossless cutting should be at least: H.262, H.264, H.265, and maybe H.263.

I wonder if the audio cutting is similarly virtually lossless.

Could someone rename this issue to "Rewrite mythtranscode to use ffmpeg instead of libmpeg2" since that is what actually needs to be done?

[Jpilk]

I have never used this, which is in 'python' and appears to output .mkv, but it seems worth mentioning its existence.

"A python wrapper for calling external transcode utilities on recordings. Automatically handles all database operations for transcoding a file in place. "

https://www.mythtv.org/wiki/Transcode_wrapper_stub

[ulmus-scott]

As a second, related rewrite, mythtranscode should maybe use FFmpeg's libavformat instead of replex.

However, replex does seem to be used by MythArchive:

mythplugins/mytharchive/mytharchive/archivesettings.cpp:155:                                        "mythreplex."));
mythplugins/mytharchive/mytharchive/archivesettings.cpp:412:                                        "mythreplex."));
mythplugins/mytharchive/mythburn/scripts/mythburn.py:2458:# using mythreplex
mythplugins/mytharchive/mythburn/scripts/mythburn.py:2463:        command = "mythreplex --demux --fix_sync -t TS -o %s " % quoteCmdArg(folder + "/stream")
mythplugins/mytharchive/mythburn/scripts/mythburn.py:2483:        command = "mythreplex --demux --fix_sync -o %s " % quoteCmdArg(folder + "/stream")
mythplugins/mytharchive/mythburn/scripts/mythburn.py:2510:        fatalError("Failed while running mythreplex. Command was %s" % command)
mythplugins/mytharchive/mythburn/scripts/mythburn.py:4494:# mythtranscode/mythreplex or ProjectX as the cutter/demuxer
mythplugins/mytharchive/mythburn/scripts/mythburn.py:4505:# process a single file ready for burning using mythtranscode/mythreplex

[Jpilk]

I used to be an intensive user of MythArchive. I used ProjectX as the cutter/demuxer/sync fixer - and still do, in my current cutting script for mpeg2video recordings. It appears to have been unmaintained for years, and fails with some varieties of GOP or non-monotonic timestamps, but mostly it just works.