diff --git a/Makefile b/Makefile index d8a5a31d..7ef7a2cd 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,8 @@ DEUTEX_BASIC_ARGS=-v0 -rate accept DEUTEX_ARGS=$(DEUTEX_BASIC_ARGS) -doom2 bootstrap/ NODE_BUILDER=ZenNode NODE_BUILDER_LEVELS=e?m? dm?? map?? +LEGACY_TRANSPARENCY_INDEX=255 +LEGACY_TRANSPARENCY_REPLACEMENT=133 FREEDOOM1=$(WADS)/freedoom1.wad FREEDOOM2=$(WADS)/freedoom2.wad @@ -18,7 +20,7 @@ FREEDM=$(WADS)/freedm.wad OBJS=$(FREEDM) $(FREEDOOM1) $(FREEDOOM2) -.PHONY: clean dist +.PHONY: clean dist pngs-modified-check all: deutex-check $(OBJS) @@ -57,6 +59,12 @@ deutex-check: echo "DEUTEX=/the/path/to/deutex to make when building Freedoom."; \ exit 1; } +# Make sure that no PNG files are modified if scripts are to modify them. +pngs-modified-check: + @{ ! git status -s | grep -q \\.png$ ; } || { \ + echo "PNG fix targets can not be run if there are modified PNGs." ; \ + exit 1; } + #--------------------------------------------------------- # freedm iwad @@ -180,8 +188,34 @@ fix-vanilla-compliance: fix-gfx-offsets: scripts/fix-gfx-offsets sprites/*.png +# Overwrite PNGs with what deutex extracts from the WADs produced. +fix-deutex-pngs: pngs-modified-check + scripts/fix-deutex-pngs $(OBJS) + +# For each PNG replace the legacy transparency index with a similar color, but +# only if the PNG matches the playpal palette specified. It may be helpful to +# run target fix-deutex-pngs before this one. +fix-legacy-transparency-pngs: pngs-modified-check + scripts/map-color-index -p lumps/playpal/playpal-base.lmp . \ + $(LEGACY_TRANSPARENCY_INDEX) $(LEGACY_TRANSPARENCY_REPLACEMENT) + +# Fix targets that fix PNGs. Note that because of the interaction between the +# scripts that are run it can be necessary to run this more than once: +# make # Optional, but make sure the IWADs (wads dir) is up-to-date. +# make fix-pngs +# git commit -m 'Fix the PNGs' -- '*.png' # So the tree is clean for the next run. +# make # Required - so the IWADs are updated. +# make fix-pngs +# git commit --amend --no-edit -- '*.png' +# make # Optional - the PNGs should be stable. +# make fix-pngs # Optional - should not have an effect. +# The final invocation of "fix-pngs" is not needed, but doing so, and seeing +# a clean build tree, is reassuring. If "fix-pngs" needs to be run more than +# twice then something is wrong. +fix-pngs: fix-deutex-pngs fix-legacy-transparency-pngs + # Run all fixes. Add fix-* targets above, and then as a dependency here. -fix: fix-vanilla-compliance +fix: fix-vanilla-compliance fix-pngs @echo @echo "All fixable errors fixed." diff --git a/scripts/fix-deutex-pngs b/scripts/fix-deutex-pngs new file mode 100755 index 00000000..7eb660d9 --- /dev/null +++ b/scripts/fix-deutex-pngs @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# +# deutex-pngs - Use deutex to process the PNGs in order to standardize them. +# +# Often there are PNG files that are not in the preferred format (that are not +# indexed, or that are indexed with something other than PLAYPAL, that have +# chunks that are not supported, etc.). For example, gAMA and similar color +# correction chunks only cause the PNG to look different in image editors than +# they do in game engines, which is bad. It's also for the palette to have +# invalid colors, or to be missing colors, which makes the PNG hard to work +# with. The solution is to use deutex to extract the WADs produced by the build, +# and apply the PNGs exctracted to the build. +# +# This programs strives to be safe by only applying PNGs to the build when the +# PNG already exists in the build since otherwise the relationship between the +# extracted PNG and the build PNG is not clear. If -a, --all is specified then +# the PNG will be be copied regardless. + +# When multiple WADs are passed to the script the each WAD effectively +# overwrites the previous ones, so the preferred WAD should be passed last. + +# Normally this script is invoked by "make": +# make fix-deutex-pngs + +# Imports + +import argparse +import os +import shutil +import sys +import tempfile +import time + +# Globals. Alphabetical. + +# Command line arguments. +args = {} + +# A count of unexpected errors. These are errors other than what this script +# intends to check for, such as the file not being readable. These errors +# suggest that something is fundamentally wrong, and that the results should +# not be trusted. +unexpected_count = 0 + +# Parse the command line arguments and store the result in 'args'. +def parse_args(): + global args + + parser = argparse.ArgumentParser( + description="Use deutex to process the PNGs in order to standardize them.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + # The following is sorted by long argument. + + parser.add_argument( + "-a", + "--all", + action="store_true", + help="Copy all PNGs, not just the ones that already exist in the build.") + parser.add_argument( + "-d", + "--doom2", + default="bootstrap", + help="The '-doom2' deutex argument.") + + parser.add_argument( + "wads", + metavar="WAD", + nargs="+", + help="WAD files to apply. The preferred WAD should be last.") + + args = parser.parse_args() + + return args + +# Main enty point. +def main(): + parse_args() + process_wads() + summarize() + +# Process a WAD file. +def process_wad(wad_path): + global unexpected_count + + wad_dir = os.path.dirname(wad_path) + wad_name = os.path.basename(wad_path).split(".")[0] + print() + print("Processing WAD", wad_path) + pngs_copied = 0 + with tempfile.TemporaryDirectory(prefix=wad_name + "-") as tmp_dir: + print('created temporary directory', tmp_dir) + ec = os.system("deutex -doom2 " + args.doom2 + " -dir " + tmp_dir + + " -x " + wad_path) + if ec: + print("deutex failed with exit code", ec, "for WAD", wad_path, + file=sys.stderr) + unexpected_count += 1 + return + # Successful deutex from this point forward. + before = time.time() + for dirpath, dirnames, filenames in os.walk(tmp_dir): + for png_base in filenames: + if not png_base.lower().endswith(".png"): + continue + png_source_path = os.path.join(dirpath, png_base) + png_relative = os.path.relpath(png_source_path, tmp_dir) + if args.all or os.path.exists(png_relative): + shutil.copyfile(png_source_path, png_relative) + pngs_copied += 1 + print("Processed WAD", wad_path, "copied", pngs_copied, "PNG files.") + +# Process multiple WAD files. +def process_wads(): + for wad_path in args.wads: + process_wad(wad_path) + +# Summarize what happened, and then exit with the appropriate exit code. +def summarize(): + print() + print("Processed", len(args.wads), "WAD files.") + if unexpected_count: + print("There were", unexpected_count, "unexpected errors - see above. " + + "The output produced should not be trusted.", file=sys.stderr) + sys.exit(1) + +# So that this script may be accessed as a module. +if __name__ == "__main__": + main() diff --git a/scripts/map-color-index b/scripts/map-color-index new file mode 100755 index 00000000..ddfbd9de --- /dev/null +++ b/scripts/map-color-index @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# +# map-color-index - Map from one palette index to another +# +# Often there are PNG files that use an index that should not be used. For +# example, in the Doom palette index 255 is used for transparence in some +# legacy engines, so it can be helpful to avoid it. Color 255 can be mapped to +# 133 since they are visually similar. +# +# This programs strives to be the following: +# 1) Be fast in the common case of the PNG not needing to be changed. +# 2) Be as safe as possible by making a few extraneous changes as possible. +# All chunks other than the pixels (IDAT) are preserved. +# 3) Be as safe as possible by making only modifying relevant PNGs. For +# additonal safety a palette can be passed (-p option) which must be +# matched exactly for any change to be made. +# +# Normally this script is invoked by "make": +# make fix-legacy-transparency-pngs + +# Imports + +import argparse +import os +import png +import sys + +# Globals. Alphabetical. + +# Command line arguments. +args = {} + +# The palette in byte form (binary read of args.palette). +palette_bytes = None + +# The number of RGB entries in the args.palette. +palette_size = 0 + +# The number of PNGs that were actually changed. +pngs_changed_count = 0 + +# The total number of PNGs that were examined. +pngs_examined_count = 0 + +# Main entry point. +def main(): + parse_args() + parse_palette() + process_dir() + summarize() + +# Parse the command line arguments and store the result in 'args'. +def parse_args(): + global args + + parser = argparse.ArgumentParser( + description="Map from one palette index to another.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + # The following is sorted by long argument. + + parser.add_argument( + "-k", + "--keep", + action="store_true", + help="Keep temporary files.") + parser.add_argument( + "-d", + "--debug", + action="store_true", + help="Debug (lots of) output. Implies verbose.") + parser.add_argument( + "-p", + "--palette", + help="If specified then only PNGs with precisely this palette will be " + + "altered. This is tightly packed list of RGB three byte pairs, " + + "just like the PLTE chunk. \"lumps/playpal/playpal-base.lmp\" " + + "is a common value.") + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Verbose output. Typically one line per PNG.") + parser.add_argument( + "dir", + metavar="DIR", + help="Directory to process recursively.") + parser.add_argument( + "from_index", + metavar="FROM-INDEX", + type=int, + help="Change from this index.") + parser.add_argument( + "to_index", + metavar="TO-INDEX", + type=int, + help="Change to this index.") + + args = parser.parse_args() + + # Additonal custom argument parsing and checking. + + # TODO: Is writing to "args" ok? + # --debug implies --verbose. + if args.debug: + args.verbose = True + + return args + +# Reads args.palette, which a file containing RGBRGBRGB ... +def parse_palette(): + global palette_bytes + global palette_size + + if not args.palette: + return + with open(args.palette, "rb") as handle: + palette_bytes = handle.read() + if len(palette_bytes) % 3: + print("The length of", args.palette, "is not a multiple of three.", + file=sys.stderr) + sys.exit(1) + if len(palette_bytes) > 768: + print("The length of", args.palette, "exceeds 768.", file=sys.stderr) + sys.exit(1) + palette_size = len(palette_bytes) // 3 + if args.debug: + print("palette_bytes", palette_bytes) + print("palette_size", palette_size) + +# Process a directory recursively for PNG files. +def process_dir(): + global pngs_changed_count + global pngs_examined_count + + pngs_changed = 0 + for dirpath, dirnames, filenames in os.walk(args.dir): + for png_base in filenames: + if not png_base.lower().endswith(".png"): + continue + png_path = os.path.join(dirpath, png_base) + pngs_examined_count += 1 + if process_png(png_path): + pngs_changed_count += 1 + +# Process a PNG file in place. +def process_png(png_path): + # This method works by first seeing if the PNG needs to be modified, and if + # so a temporary PNG is buit with from-index replaced with to-index. The + # IDAT (pixels) are then taken from that temporary PNG and combined with + # the chunks from the original PNG in order to produce a PNG where only the + # IDAT is modified. + # + # Prefix naming conventions used in this method: + # old_ - Taken from png_path in its original form. + # tmp_ - A temporary file that has the correct new IDAT, but otherwise + # unwanted chunks. + # new_ - The new png_path. + # Some variables that lack a prefix apply to more than one of the above. + + # If verbose print the last 22 characters of the relative png_path followed + # by two spaces for a total width of 24. If not debug then end="" + # so that the resolution is on the same line with one line per PNG. + if args.verbose: + png_relpath = os.path.relpath(png_path, args.dir) + print("{0:22s} ".format(png_relpath[-22:]), end=("\n" if args.debug else "")) + + # Ignore temporary files. + if png_path.endswith("-tmp.png"): + if args.verbose: + print("temporary file") + return False + + # Read the existing old png_path as a flat array of pixels as well as some + # other information (kwargs). Note that png.Reader only allows images to be + # read once in a single pass, so it's sometimes necessary to open the same + # PNG multiple times. + old_reader = png.Reader(png_path) + old_flat = old_reader.read_flat() + + # Get the existing pixels. The kwargs is a map that describes the mode and + # style of the PNG, so it can be replicated. + old_pixels = old_flat[2] + kwargs = old_flat[3] + + if args.debug: + print("old_flat", old_flat) + + # Before doing any other processing return quickly if the image is not + # reasonable. + + # If it does not have a palette then it is not even indexed. + if not "palette" in kwargs: + if args.verbose: + print("no palette") + return False + + # Is there a from-index to change? + if not args.from_index in old_pixels: + if args.verbose: + print("from-index not found") + return False + + if args.palette: + # Check that the palette has the correct size first since it's fast. + if len(kwargs["palette"]) != palette_size: + if args.verbose: + print("palette incorrect size") + return False + + # Check that PLTE exactly matches the palette specified. + old_reader = png.Reader(png_path) + old_plte = old_reader.chunk("PLTE")[1] + if old_plte != palette_bytes: + if args.verbose: + print("palette does not match") + return False + + # Open png_path again to get the chunks. + old_reader = png.Reader(png_path) + old_chunks = old_reader.chunks() + + # Build a new list of pixels with the index replaced. + if args.debug: + print("old_pixels", old_pixels) + new_pixels = [args.to_index if p == args.from_index else p for p in old_pixels] + if args.debug: + print("new_pixels", new_pixels) + + # Remove keys from kwargs that are known to cause trouble with writing. + kwargs.pop("background", None) + + # Create a temporary PNG with the desired pixels, but not the desired + # chunks. Passing kwargs assures that the temporary PNG has the same mode + # as the original one. + tmp_writer = png.Writer(**kwargs) + tmp_path = png_path[:-4] + "-tmp.png" + with open(tmp_path, "wb") as tmp_hand: + tmp_writer.write_array(tmp_hand, new_pixels) + + # TODO: It would be nice if there was access to the internal API that + # generates the IDAT without creating temporary files. + tmp_reader = png.Reader(tmp_path) + tmp_idat = tmp_reader.chunk("IDAT") + if not args.keep: + os.remove(tmp_path) + if args.debug: + print("tmp_idat", tmp_idat) + + # Create the new image with the new pixels, but the old chunks. + new_chunks_array = [tmp_idat if chunk[0] == "IDAT" else chunk + for chunk in old_chunks] + if args.debug: + print("new_chunks_array", new_chunks_array) + + # Overwrite the existing png_path with the new chunks. + with open(png_path, "wb") as new_hand: + png.write_chunks(new_hand, new_chunks_array) + + if args.verbose: + print("changed") + if args.debug: + print() + + return True + +# Summarize what happened, and then exit with the appropriate exit code. +def summarize(): + print() + print("Processed directory", args.dir) + print("Examined", pngs_examined_count, "PNGs.") + print("Changed ", pngs_changed_count , "PNGs.") + +# So that this script may be accessed as a module. +if __name__ == "__main__": + main()