From f94427318ca477d72df7be518020a070ebb0f78c Mon Sep 17 00:00:00 2001 From: louis Date: Sat, 17 Apr 2021 03:12:14 -0400 Subject: [PATCH] vivy: tv: common: i regret everything i've ever done --- Vivy/03/03.vpy | 33 ++- Vivy/vivy_common/__init__.py | 2 +- Vivy/vivy_common/dehardsub.py | 2 +- Vivy/vivy_common/final-settings | 1 + Vivy/vivy_common/util.py | 330 +++++++++++++++++++++++++++++- Vivy/vivy_common/workraw-settings | 2 + Vivy/x265-settings | 1 - 7 files changed, 353 insertions(+), 18 deletions(-) create mode 100644 Vivy/vivy_common/final-settings create mode 100644 Vivy/vivy_common/workraw-settings delete mode 100644 Vivy/x265-settings diff --git a/Vivy/03/03.vpy b/Vivy/03/03.vpy index 505400a..30b31d3 100644 --- a/Vivy/03/03.vpy +++ b/Vivy/03/03.vpy @@ -6,7 +6,7 @@ import os import sys sys.path.append("..") -from vivy_common import (HardsubSign, Range, bounded_dehardsub, antialias, deband, denoise, # noqa: E402 +from vivy_common import (HardsubSign, Range, SelfRunner, bounded_dehardsub, antialias, deband, denoise, # noqa: E402 finalize, fsrcnnx_rescale, letterbox_edgefix, source) core = vs.core @@ -31,13 +31,26 @@ NOSCALE: List[Range] = [] NOAA: List[Range] = [] LETTERBOX: List[Range] = [(0, 432)] -waka, ref = source(EPNUM) -src = bounded_dehardsub(waka, ref, SIGNS_RU) -rescale = fsrcnnx_rescale(src, NOSCALE) -den = denoise(rescale) -deb = deband(den) -aa = antialias(deb, NOAA) -edgefix = letterbox_edgefix(aa, LETTERBOX) -final = finalize(edgefix) -final.set_output() +def filter_basic() -> vs.VideoNode: + waka, ref = source(EPNUM) + src = bounded_dehardsub(waka, ref, SIGNS_RU) + return src + + +def filter() -> vs.VideoNode: + src = filter_basic() + rescale = fsrcnnx_rescale(src, NOSCALE) + den = denoise(rescale) + deb = deband(den) + aa = antialias(deb, NOAA) + edgefix = letterbox_edgefix(aa, LETTERBOX) + final = finalize(edgefix) + final.set_output() + return final + + +if __name__ == "__main__": + SelfRunner(EPNUM, filter, filter_basic) +else: + filter() diff --git a/Vivy/vivy_common/__init__.py b/Vivy/vivy_common/__init__.py index 3bc9238..6789f14 100644 --- a/Vivy/vivy_common/__init__.py +++ b/Vivy/vivy_common/__init__.py @@ -1,3 +1,3 @@ from .dehardsub import HardsubSign, bounded_dehardsub, get_all_masks # noqa: F401 from .filter import antialias, deband, denoise, finalize, fsrcnnx_rescale, letterbox_edgefix # noqa: F401 -from .util import Range, glob_crc, source # noqa: F401 +from .util import Range, SelfRunner, glob_crc, source # noqa: F401 diff --git a/Vivy/vivy_common/dehardsub.py b/Vivy/vivy_common/dehardsub.py index 4cc44fa..d03787c 100644 --- a/Vivy/vivy_common/dehardsub.py +++ b/Vivy/vivy_common/dehardsub.py @@ -81,7 +81,7 @@ def get_all_masks(hrdsb: vs.VideoNode, ref: vs.VideoNode, signs: List[HardsubSig Scenefiltering helper, not used in encode """ assert ref.format is not None - mask = core.std.BlankClip(ref, format=ref.format.replace(color_family=vs.GRAY, subsampling_w=0, subsampling_h=0)) + mask = core.std.BlankClip(ref, format=ref.format.replace(color_family=vs.GRAY, subsampling_w=0, subsampling_h=0).id) for sign in signs: mask = lvf.misc.replace_ranges(mask, core.std.Expr([mask, sign.get_mask(hrdsb, ref)], 'x y +'), [sign.range]) return mask diff --git a/Vivy/vivy_common/final-settings b/Vivy/vivy_common/final-settings new file mode 100644 index 0000000..00c4ec4 --- /dev/null +++ b/Vivy/vivy_common/final-settings @@ -0,0 +1 @@ +x265 --input - --y4m --input-depth 10 --output-depth 10 --input-csp i420 --profile main10 --colormatrix bt709 --colorprim bt709 --transfer bt709 --preset slower --rc-lookahead 72 --keyint 360 --min-keyint 23 --subme 5 --qcomp 0.7 --crf 15 --aq-mode 3 --aq-strength 0.9 --bframes 16 --psy-rd 0.95 --psy-rdoq 1.8 --rdoq-level 1 --deblock -2:-2 --no-sao --frames {frames:d} --output {filename:s}.h265 diff --git a/Vivy/vivy_common/util.py b/Vivy/vivy_common/util.py index fb8e7fc..59f8a51 100644 --- a/Vivy/vivy_common/util.py +++ b/Vivy/vivy_common/util.py @@ -1,17 +1,43 @@ import vapoursynth as vs +import acsuite +import argparse +import os +import functools import glob +import signal +import string +import subprocess import vsutil -from typing import Tuple, Union +from typing import Any, BinaryIO, Callable, List, Optional, Sequence, Tuple, Union, cast core = vs.core Range = Union[int, Tuple[int, int]] -FUNI_FILENAME: str = "[SubsPlease] Vivy - Fluorite Eye's Song - {epnum:02d} (1080p) [$CRC].mkv" +TITLE: str = "Vivy" +TITLE_LONG: str = f"{TITLE} - Fluorite Eye's Song" +RESOLUTION: int = 1080 + +SUBGROUP = "YameteTomete" + +SUBSPLS_FILENAME: str = f"[SubsPlease] {TITLE_LONG} - {{epnum:02d}} ({RESOLUTION}p) [$CRC].mkv" +ER_FILENAME: str = f"[Erai-raws] {TITLE_LONG} - {{epnum:02d}} [{RESOLUTION}p][Multiple Subtitle].mkv" FUNI_INTRO: int = 289 -WAKA_FILENAME: str = "Vivy_{epnum:02d}_RU_HD.mp4" +WAKA_FILENAME: str = f"{TITLE}_{{epnum:02d}}_RU_HD.mp4" +AMAZON_FILENAME: str = f"{TITLE_LONG} - {{epnum:02d}} (Amazon Prime CBR {RESOLUTION}p).mkv" + +AUDIO_OVERRIDE: str = "audio.mka" +AUDIO_FMT: List[str] = [".mka", ".aac", ".wav", ".flac", ".mp3", ".ogg", ".opus", ".m4a"] +AFV_FMT: List[str] = [".mkv", ".mp4", ".webm"] +AUDIO_CUT: str = "_audiogetter_cut.mka" + +STATUS: str = '\033[94m' +WARNING: str = '\033[93m' +ERROR: str = '\033[91m' +SUCCESS: str = '\033[92m' +RESET: str = '\033[0m' def glob_crc(pattern: str) -> str: @@ -21,7 +47,301 @@ def glob_crc(pattern: str) -> str: return res[0] +def get_ref(epnum: int) -> vs.VideoNode: + if epnum >= 4: + if os.path.isfile(AMAZON_FILENAME.format(epnum=epnum)): + return core.ffms2.Source(AMAZON_FILENAME.format(epnum=epnum)) + else: + print(f"{WARNING}Amazon video not found, dehardsubbing with new funi encode{RESET}") + + try: + return core.ffms2.Source(glob_crc(SUBSPLS_FILENAME.format(epnum=epnum)))[FUNI_INTRO:] + except FileNotFoundError: + pass + + if not os.path.isfile(ER_FILENAME.format(epnum=epnum)): + raise FileNotFoundError("Failed to find valid reference video") + + return core.ffms2.Source(ER_FILENAME.format(epnum=epnum))[FUNI_INTRO:] + + def source(epnum: int) -> Tuple[vs.VideoNode, vs.VideoNode]: waka = vsutil.depth(core.ffms2.Source(WAKA_FILENAME.format(epnum=epnum)), 16) - funi = vsutil.depth(core.ffms2.Source(glob_crc(FUNI_FILENAME.format(epnum=epnum))), 16)[FUNI_INTRO:] - return waka, funi + ref = vsutil.depth(get_ref(epnum), 16) + return waka, ref + + +def bin_to_plat(binary: str) -> str: + if os.name == "nt": + return binary if binary.lower().endswith(".exe") else f"{binary}.exe" + else: + return binary if not binary.lower().endswith(".exe") else binary[:-len(".exe")] + + +def forward_signal(signum: int, frame: Any, process: Any) -> None: + print(f"{WARNING}Forwarding SIGINT{RESET}") + process.send_signal(signum) + + +class Encoder(): + clip: vs.VideoNode + + binary: str + params: Sequence[str] + force: bool + + out_template: str + + cleanup: List[str] + + def __init__(self, epnum: int, settings_path: str, + binary: Optional[str] = None, force: bool = False) -> None: + self.binary = binary if binary is not None else "" + self.force = force + self.cleanup = [] + + self._get_encoder_settings(settings_path) + + def encode(self, clip: vs.VideoNode, filename: str, start: int = 0, end: int = 0) -> str: + end = end if end != 0 else clip.num_frames + + outfile = self.out_template.format(filename=filename) + + if os.path.isfile(outfile) and not self.force: + print(f"{WARNING}Existing output detected, skipping encode!{RESET}") + return outfile + + params = [p.format(frames=end-start, filename=filename) for p in self.params] + + print(f"{STATUS}--- RUNNING ENCODE ---{RESET}") + + print("+ " + " ".join([self.binary] + list(params))) + + process = subprocess.Popen([self.binary] + list(params), stdin=subprocess.PIPE) + + # i want the encoder to handle any ctrl-c so it exits properly + forward_to_proc = functools.partial(forward_signal, process=process) + signal.signal(signal.SIGINT, forward_to_proc) + + clip[start:end].output(cast(BinaryIO, process.stdin), y4m=True) + process.communicate() + + # vapoursynth should handle this itself but just in case + if process.returncode != 0: + print(f"{ERROR}--- ENCODE FAILED ---{RESET}") + raise BrokenPipeError(f"Pipe to {self.binary} broken") + + print(f"{SUCCESS}--- ENCODE FINISHED ---{RESET}") + self.cleanup.append(outfile) + return outfile + + def _get_encoder_settings(self, settings_path: str) -> None: + with open(settings_path, "r") as settings: + keys = " ".join([line.strip() for line in settings if not line.strip().startswith("#")]).split(" ") + + # verify that the settings contain an output file template + outputs = [k for k in keys[1:] if any([name == "filename" for _, name, _, _ in string.Formatter().parse(k)])] + if not outputs or len(outputs) > 1: + raise Exception("Failed to find unambiguous output file for encoder!") + self.out_template = outputs[0] + + self.binary = bin_to_plat(keys[0]) if not self.binary else self.binary + self.params = keys[1:] + + def do_cleanup(self) -> None: + for f in self.cleanup: + os.remove(f) + self.cleanup = [] + + +class AudioGetter(): + audio_file: str + audio_start: int + video_src: Optional[vs.VideoNode] + + cleanup: List[str] + + def __init__(self, epnum: int, override: Optional[str] = None) -> None: + self.audio_start = 0 + self.video_src = None + self.cleanup = [] + + if override is not None: + if os.path.isfile(override): + self.audio_file = override + else: + raise FileNotFoundError(f"Audio file {override} not found!") + + # drop "audio.m4a" into the folder and it'll get used + if os.path.isfile(AUDIO_OVERRIDE): + self.audio_file = AUDIO_OVERRIDE + return + + # look for amazon first + if os.path.isfile(AMAZON_FILENAME.format(epnum=epnum)): + self.audio_file = AMAZON_FILENAME.format(epnum=epnum) + self.video_src = core.ffms2.Source(AMAZON_FILENAME.format(epnum=epnum)) + print(f"{SUCCESS}Found Amazon audio{RESET}") + return + + try: + if os.path.isfile(glob_crc(SUBSPLS_FILENAME.format(epnum=epnum))): + self.audio_file = glob_crc(SUBSPLS_FILENAME.format(epnum=epnum)) + self.video_src = core.ffms2.Source(glob_crc(SUBSPLS_FILENAME.format(epnum=epnum))) + elif os.path.isfile(ER_FILENAME.format(epnum=epnum)): + self.audio_file = ER_FILENAME.format(epnum=epnum) + self.video_src = core.ffms2.Source(ER_FILENAME.format(epnum=epnum)) + else: + raise FileNotFoundError() + except FileNotFoundError: + print(f"{ERROR}Could not find audio{RESET}") + raise + + self.audio_start = FUNI_INTRO + print(f"{WARNING}No Amazon audio, falling back to Funi{RESET}") + + def trim_audio(self, src: vs.VideoNode, + trims: Union[acsuite.Trim, List[acsuite.Trim], None] = None) -> str: + if isinstance(trims, tuple): + trims = [trims] + + if trims is None or len(trims) == 0: + if self.audio_start == 0: + trims = [(None, None)] + else: + trims = [(self.audio_start, None)] + else: + if self.audio_start != 0: + trims = [(s+self.audio_start if s is not None and s >= 0 else s, + e+self.audio_start if e is not None and e > 0 else e) + for s, e in trims] + + if os.path.isfile(AUDIO_CUT): + os.remove(AUDIO_CUT) + + acsuite.eztrim(self.video_src if self.video_src else src, trims, + self.audio_file, AUDIO_CUT, quiet=True) + + self.cleanup.append(AUDIO_CUT) + + return AUDIO_CUT + + def do_cleanup(self) -> None: + for f in self.cleanup: + os.remove(f) + self.cleanup = [] + + +class SelfRunner(): + clip: vs.VideoNode + epnum: int + + workraw: bool + + video_file: str + audio_file: str + + encoder: Encoder + audio: AudioGetter + + def __init__(self, epnum: int, final_filter: Callable[[], vs.VideoNode], + workraw_filter: Optional[Callable[[], vs.VideoNode]] = None) -> None: + self.epnum = epnum + self.video_clean = False + self.audio_clean = False + + parser = argparse.ArgumentParser(description=f"Encode {TITLE} Episode {epnum:02d}") + if workraw_filter: + parser.add_argument("-w", "--workraw", help="Encode workraw, fast x264", action="store_true") + parser.add_argument("-s", "--start", nargs='?', type=int, help="Start encode at frame START") + parser.add_argument("-e", "--end", nargs='?', type=int, help="Stop encode at frame END (inclusive)") + parser.add_argument("-k", "--keep", help="Keep raw video", action="store_true") + parser.add_argument("-c", "--encoder", type=str, help="Override detected encoder binary") + parser.add_argument("-f", "--force", help="Overwrite existing intermediaries", action="store_true") + parser.add_argument("-a", "--audio", type=str, help="Force audio file") + parser.add_argument("-x", "--suffix", type=str, default="premux", help="Change the suffix of the mux") + parser.add_argument("-d", "--no-metadata", help="No extra metadata in premux", action="store_true") + args = parser.parse_args() + + self.workraw = args.workraw if workraw_filter else False + self.suffix = args.suffix if not self.workraw else "workraw" + + self.clip = workraw_filter() if workraw_filter and self.workraw else final_filter() + + basename = "workraw-settings" if self.workraw else "final-settings" + settings_path = os.path.join(os.path.dirname(__file__), basename) + + if not os.path.isfile(settings_path): + raise FileNotFoundError("Failed to find {basename}!") + + start = args.start if args.start is not None else 0 + if args.end is not None: + if args.end < 0: + end = self.clip.num_frames + args.end + else: + end = args.end + 1 + else: + end = self.clip.num_frames + + if start < 0: + raise ValueError("Start frame cannot be less than 0!") + if start > self.clip.num_frames: + raise ValueError("Start frame exceeds clip length!") + if end < 0: + raise ValueError("End frame cannot be less than 0!") + if end > self.clip.num_frames: + raise ValueError("End frame exceeds clip length!") + if start >= end: + raise ValueError("Start frame must be before end frame!") + + self.encoder = Encoder(epnum, settings_path, args.encoder, args.force) + self.video_file = self.encoder.encode(self.clip, f"{epnum:02d}_{start}_{end}", start, end) + + print(f"{STATUS}--- LOOKING FOR AUDIO ---{RESET}") + self.audio = AudioGetter(self.epnum, args.audio) + + print(f"{STATUS}--- TRIMMING AUDIO ---{RESET}") + self.audio_file = self.audio.trim_audio(self.clip, (start, end)) + + try: + print(f"{STATUS}--- MUXING FILE ---{RESET}") + if self._mux(f"{TITLE.lower()}_{epnum:02d}_{args.suffix}.mkv", not args.no_metadata) != 0: + raise Exception("mkvmerge failed") + except Exception: + print(f"{ERROR}--- MUXING FAILED ---{RESET}") + self.audio.do_cleanup() + raise + + print(f"{SUCCESS}--- MUXING SUCCESSFUL ---{RESET}") + + self.audio.do_cleanup() + + if not args.keep: + self.encoder.do_cleanup() + + print(f"{SUCCESS}--- ENCODE COMPLETE ---{RESET}") + + def _mux(self, name: str, metadata: bool = True) -> int: + mkvtoolnix_args = [ + "mkvmerge", + "--output", name, + "--no-chapters", "--no-track-tags", "--no-global-tags", "--track-name", "0:", + "--default-track", "0:yes", + "(", self.video_file, ")", + "--no-chapters", "--no-track-tags", "--no-global-tags", "--track-name", "0:", + "--default-track", "0:yes", "--language", "0:jpn", + "(", self.audio_file, ")", + "--track-order", "0:0,0:1", + ] + if metadata: + mkvtoolnix_args += [ + "--title", f"[{SUBGROUP}] {TITLE_LONG} - {self.epnum:02d}", + ] + chapters = [f for f in ["{self.epnum:02d}.xml", "chapters.xml"] if os.path.isfile(f)] + if len(chapters) != 0: + mkvtoolnix_args += [ + "--chapters", chapters[0], + ] + + print("+ " + " ".join(mkvtoolnix_args)) + return subprocess.call(mkvtoolnix_args) diff --git a/Vivy/vivy_common/workraw-settings b/Vivy/vivy_common/workraw-settings new file mode 100644 index 0000000..3750a26 --- /dev/null +++ b/Vivy/vivy_common/workraw-settings @@ -0,0 +1,2 @@ +# these don't have to make sense +x264 - --demuxer y4m --input-depth 10 --output-depth 10 --colormatrix bt709 --colorprim bt709 --transfer bt709 --preset fast --crf 16 --aq-mode 3 --aq-strength 0.85 --qcomp 0.70 --psy-rd 0.85:0.0 --deblock -1:-1 --frames {frames:d} --output {filename:s}.h264 diff --git a/Vivy/x265-settings b/Vivy/x265-settings deleted file mode 100644 index 9ec68bf..0000000 --- a/Vivy/x265-settings +++ /dev/null @@ -1 +0,0 @@ ---y4m --input-depth 10 --output-depth 10 --input-csp i420 --profile main10 --colormatrix bt709 --colorprim bt709 --transfer bt709 --preset slower --rc-lookahead 72 --keyint 360 --min-keyint 23 --subme 5 --qcomp 0.7 --crf 15 --aq-mode 3 --aq-strength 0.9 --bframes 16 --psy-rd 0.95 --psy-rdoq 1.8 --rdoq-level 1 --deblock -2:-2 --no-sao